diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go
index 9fe39cc0bc..709756462d 100644
--- a/swarm/network/stream/snapshot_retrieval_test.go
+++ b/swarm/network/stream/snapshot_retrieval_test.go
@@ -15,33 +15,29 @@
// along with the go-ethereum library. If not, see .
package stream
-/*
-
import (
- "context"
+ //"context"
crand "crypto/rand"
"flag"
"fmt"
"io"
"math/rand"
-// "os"
+ // "os"
+ "github.com/ethereum/go-ethereum/log"
"testing"
"time"
-
- "github.com/ethereum/go-ethereum/log"
-// "github.com/ethereum/go-ethereum/node"
-// "github.com/ethereum/go-ethereum/pot"
+ // "github.com/ethereum/go-ethereum/node"
+ // "github.com/ethereum/go-ethereum/pot"
"github.com/ethereum/go-ethereum/p2p/discover"
- "github.com/ethereum/go-ethereum/p2p/simulations"
-// "github.com/ethereum/go-ethereum/p2p/simulations/adapters"
- "github.com/ethereum/go-ethereum/swarm/network"
+ //"github.com/ethereum/go-ethereum/p2p/simulations"
+ // "github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+ //"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
//streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
)
var rootHash storage.Key
-
func init() {
flag.Parse()
rand.Seed(time.Now().Unix())
@@ -49,47 +45,11 @@ func init() {
initRetrievalTest()
}
-
func initRetrievalTest() {
- //assign the toAddr func so NewStreamerService can build the addr
- toAddr = func(id discover.NodeID) *network.BzzAddr {
- addr := network.NewAddrFromNodeID(id)
- addr.OAddr[0] = byte(0)
- return addr
- }
-
- //nodeCount is needed to load a specific json snapshot file,
- //e.g. "snapshot_16.json"
- nodeCount = 16
- //is used to continuosly provide the current discoverID to NewStreamerService
- //while loading the snapshot
- currentId = 0
- //local stores
- stores = make(map[discover.NodeID]storage.ChunkStore)
- //data directories for each node and store
- datadirs = make(map[discover.NodeID]string)
- //deliveries for each node
- deliveries = make(map[discover.NodeID]*Delivery)
- //the list of the ids loaded from the snapshot
- ids = make([]discover.NodeID, nodeCount)
- //mapping of nearest node addresses for chunk hashes
- //chunksForAddressesMap = make(map[discover.NodeID][]storage.Key)
-
-
- waitPeerErrC = make(chan error)
- // peerCount function gives the number of peer connections for a nodeID
- // this is needed for the service run function to wait until
- // each protocol instance runs and the streamer peers are available
- peerCount = func(id discover.NodeID) int {
- if ids[0] == id || ids[len(ids)-1] == id {
- return 1
- }
- return 2
- }
}
+func TestRetrieval_4(t *testing.T) { retrievalTest(t, 4) }
-func TestRetrieval_4(t *testing.T) { retrievalTest(t, 4) }
/*
func TestRetrieval_1(t *testing.T) { retrievalTest(t, 1) }
func TestSyncing_4(t *testing.T) { testSyncing(t, 4) }
@@ -122,12 +82,11 @@ func benchmarkSyncing(b *testing.B, chunkCount int) {
}
*/
-/*
func retrievalTest(t *testing.T, chunkCount int) {
- err := runRetrievalTest(chunkCount)
- if err != nil {
- t.Fatal(err)
- }
+ err := runRetrievalTest(chunkCount)
+ if err != nil {
+ t.Fatal(err)
+ }
}
/*
@@ -135,172 +94,177 @@ The test generates the given number of chunks,
then uploads these to a random node.
Afterwards for every chunk generated, the nearest node addresses
are identified, syncing is started, and finally we verify
-that the nodes closer to the chunk addresses actually do have
+that the nodes closer to the chunk addresses actually do have
the chunks in their local stores.
-The test loads a snapshot file to construct the swarm network,
+The test loads a snapshot file to construct the swarm network,
assuming that the snapshot file identifies a healthy
kademlia network. The snapshot should have 'streamer' in its service list.
*/
-/*
func runRetrievalTest(chunkCount int) error {
+ /*
+ //First load the snapshot from the file
+ net, err := initNetWithSnapshot()
+ if err != nil {
+ return err
+ }
+ defer net.Shutdown()
- //reset global vars
- resetVars()
- //First load the snapshot from the file
- net,err := initNetWithSnapshot()
- if err != nil {
- return err
- }
- defer net.Shutdown()
-
- //get the nodes of the network
- nodes := net.GetNodes()
- //select one index at random...
- idx := rand.Intn(len(nodes))
- //...and get the the node at that index
- //this is the node selected for upload
- node := nodes[idx]
- //iterate over all nodes...
- for c:=0; c 0 {
- startPo = endPo + 1
- }
- } else if endPo < kadDepth || endPo == 0 {
- if po == 0 && kadDepth == 0 {
- startPo = endPo
- } else {
- startPo = endPo + 1
- }
- endPo = maxPO
+ log.Debug(fmt.Sprintf("Requesting subscription by: registry %s from peer %s for bin: %d", r.addr.ID(), conf.addrToIdMap[string(conn.Address())], po))
+
+ err = r.RequestSubscription(conf.addrToIdMap[string(conn.Address())], NewStream("SYNC", []byte{uint8(po)}, true), &Range{}, Top)
+ if err != nil {
+ log.Error(fmt.Sprintf("Error in RequestSubsciption! %v", err))
+ return false
}
-
- // now iterate and subscribe
- for bin := po - startPo; bin <= endPo; bin++ {
-
- f(func(val pot.Val, i int) bool {
- a := val.(network.OverlayPeer)
- log.Debug(fmt.Sprintf("Requesting subscription by: registry %s from peer %s for bin: %d", r.addr.ID(), conf.addrToIdMap[string(a.Address())], bin))
-
- err = r.RequestSubscription(conf.addrToIdMap[string(a.Address())], NewStream("SYNC", []byte{uint8(bin)}, true), &Range{}, Top)
- if err != nil {
- log.Error(fmt.Sprintf("Error in RequestSubsciption! %v", err))
- return false
- }
- return true
- })
- }
- i++
+ requestedSubscriptions += 1
+ //fmt.Println(requestedSubscriptions)
return true
- })
+ })
return nil
}
@@ -607,6 +562,38 @@ func uploadFileToSingleNodeStore(id discover.NodeID, chunkCount int) ([]storage.
return rootkeys, nil
}
+//Here we wait until all connections from the snapshot are up
+func waitForSnapshotConnsUp(ctx context.Context, net *simulations.Network, done chan struct{}, connCount int, errc chan error) {
+ arrivedConns := 0
+ events := make(chan *simulations.Event)
+ //subscribe to all events from the network
+ sub := net.Events().Subscribe(events)
+ defer sub.Unsubscribe()
+
+ for {
+ select {
+ case <-ctx.Done():
+ errc <- fmt.Errorf("Timeout waiting for Snapshot connections")
+ case event := <-events:
+ //if the event is of type connection, is a Live event and the connection is up
+ //NOTE; this will require that all connections are UP in the snapshot!
+ if event.Type == simulations.EventTypeConn && !event.Control && event.Conn.Up {
+ arrivedConns++
+ //the amount of expected connections has been reached, so we can stop waiting
+ if arrivedConns == connCount {
+ done <- struct{}{}
+ return
+ }
+ }
+ case err := <-sub.Err():
+ if err != nil {
+ errc <- err
+ }
+ }
+ }
+ return
+}
+
//initialize a network from a snapshot
func initNetWithSnapshot(nodeCount int) (*simulations.Network, error) {
@@ -651,10 +638,60 @@ func initNetWithSnapshot(nodeCount int) (*simulations.Network, error) {
if err != nil {
return nil, err
}
+
+ log.Info("Waiting for p2p connections to be established...")
+ errc := make(chan error)
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ connCount := len(snap.Conns)
+ done := make(chan struct{})
+ go waitForSnapshotConnsUp(ctx, net, done, connCount, errc)
+
err = net.Load(&snap)
if err != nil {
return nil, err
}
- log.Info("Snapshot loaded")
+ select {
+ case <-done:
+ close(done)
+ case err = <-errc:
+ return nil, err
+ }
+ log.Info("Snapshot loaded and connections established")
return net, nil
}
+
+func watchSubscriptionEvents(ctx context.Context, id discover.NodeID, client *rpc.Client, done chan struct{}, errc chan error) {
+ events := make(chan *p2p.PeerEvent)
+ sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
+ if err != nil {
+ errc <- fmt.Errorf("error getting peer events for node %v: %s", id, err)
+ return
+ }
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case e := <-events:
+ fmt.Println(e)
+ fmt.Println(*e.MsgCode)
+ if e.Type == p2p.PeerEventTypeMsgRecv && e.Protocol == "stream" && e.MsgCode != nil && *e.MsgCode == 1 {
+ fmt.Println(receivedSubscriptions)
+ fmt.Println(requestedSubscriptions)
+ receivedSubscriptions += 1
+ if receivedSubscriptions == requestedSubscriptions {
+ done <- struct{}{}
+ return
+ }
+ }
+ case err := <-sub.Err():
+ if err != nil {
+ errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err)
+ return
+ }
+ }
+ }
+ }()
+ return
+}
diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go
index 4121954f19..c3317e734f 100644
--- a/swarm/network/stream/syncer.go
+++ b/swarm/network/stream/syncer.go
@@ -193,9 +193,9 @@ func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) {
// NeedData
func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) {
- chunk, _ := s.db.GetOrCreateRequest(key)
+ chunk, need := s.db.GetOrCreateRequest(key)
// TODO: we may want to request from this peer anyway even if the request exists
- if chunk.ReqC == nil {
+ if chunk.ReqC == nil || need == false {
return nil
}
// create request and wait until the chunk data arrives and is stored