Merge pull request #206 from ethersphere/swarm-network-rewrite-syncer-refactor

Swarm network rewrite syncer refactor

p2p/simulations/adapters/inproc.go

@@ -112,7 +112,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) {
 			MaxPeers:        math.MaxInt32,
 			NoDiscovery:     true,
 			Dialer:          s,
-			EnableMsgEvents: true,
+			EnableMsgEvents: false,
 		},
 		NoUSB:  true,
 		Logger: log.New("node.id", id.String()),

swarm/network/discovery.go

@@ -25,9 +25,9 @@ import (
 
 // discovery bzz extension for requesting and relaying node address records
 
-// discPeer wraps bzzPeer and embeds an Overlay connectivity driver
+// discPeer wraps BzzPeer and embeds an Overlay connectivity driver
 type discPeer struct {
-	*bzzPeer
+	*BzzPeer
 	overlay   Overlay
 	sentPeers bool // whether we already sent peer closer to this address
 	mtx       sync.Mutex
@@ -36,10 +36,10 @@ type discPeer struct {
 }
 
 // NewDiscovery constructs a discovery peer
-func newDiscovery(p *bzzPeer, o Overlay) *discPeer {
+func newDiscovery(p *BzzPeer, o Overlay) *discPeer {
 	d := &discPeer{
 		overlay: o,
-		bzzPeer: p,
+		BzzPeer: p,
 		peers:   make(map[string]bool),
 	}
 	// record remote as seen so we never send a peer its own record

swarm/network/discovery_test.go

@@ -33,7 +33,7 @@ func TestDiscovery(t *testing.T) {
 	addr := RandomAddr()
 	to := NewKademlia(addr.OAddr, NewKadParams())
 
-	run := func(p *bzzPeer) error {
+	run := func(p *BzzPeer) error {
 		dp := newDiscovery(p, to)
 		to.On(p)
 		defer to.Off(p)

swarm/network/hive.go

@@ -159,7 +159,7 @@ func (h *Hive) connect() {
 }
 
 // Run protocol run function
-func (h *Hive) Run(p *bzzPeer) error {
+func (h *Hive) Run(p *BzzPeer) error {
 	dp := newDiscovery(p, h)
 	depth, changed := h.On(dp)
 	// if we want discovery, advertise changed depth of depth
@@ -191,7 +191,7 @@ func ToAddr(pa OverlayPeer) *BzzAddr {
 	if p, ok := pa.(*discPeer); ok {
 		return p.BzzAddr
 	}
-	return pa.(*bzzPeer).BzzAddr
+	return pa.(*BzzPeer).BzzAddr
 }
 
 // loadPeers, savePeer implement persistence callback/

swarm/network/kademlia_test.go

@@ -70,7 +70,7 @@ func newTestKademlia(b string) *testKademlia {
 }
 
 func (k *testKademlia) newTestKadPeer(s string) Peer {
-	return &testDropPeer{&bzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc}
+	return &testDropPeer{&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc}
 }
 
 func (k *testKademlia) On(ons ...string) *testKademlia {

swarm/network/light/lightnode.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.d
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,17 +14,18 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package network
+package light
 
 import (
 	"errors"
 
+	"github.com/ethereum/go-ethereum/swarm/network/stream"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
 // RemoteReader implements IncomingStreamer
 type RemoteSectionReader struct {
-	db            *DbAccess
+	db            *storage.DBAPI
 	start         uint64
 	end           uint64
 	hashes        chan []byte
@@ -35,7 +36,7 @@ type RemoteSectionReader struct {
 }
 
 // NewRemoteReader is the constructor for RemoteReader
-func NewRemoteSectionReader(root []byte, db *DbAccess) *RemoteSectionReader {
+func NewRemoteSectionReader(root []byte, db *storage.DBAPI) *RemoteSectionReader {
 	return &RemoteSectionReader{
 		db:     db,
 		root:   root,
@@ -45,7 +46,7 @@ func NewRemoteSectionReader(root []byte, db *DbAccess) *RemoteSectionReader {
 }
 
 func (r *RemoteSectionReader) NeedData(key []byte) func() {
-	chunk, created := r.db.getOrCreateRequest(storage.Key(key))
+	chunk, created := r.db.GetOrCreateRequest(storage.Key(key))
 	// TODO: we may want to request from this peer anyway even if the request exists
 	if chunk.ReqC == nil || !created {
 		return nil
@@ -58,7 +59,7 @@ func (r *RemoteSectionReader) NeedData(key []byte) func() {
 	}
 }
 
-func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) {
+func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*stream.TakeoverProof, error) {
 	r.hashes <- hashes
 	return nil
 }
@@ -75,9 +76,9 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) {
 		return l, nil
 	}
 	var end bool
-	for i := 0; !end && i < len(r.currentHashes); i += HashSize {
+	for i := 0; !end && i < len(r.currentHashes); i += stream.HashSize {
-		hash := r.currentHashes[i : i+HashSize]
+		hash := r.currentHashes[i : i+stream.HashSize]
-		chunk, err := r.db.get(hash)
+		chunk, err := r.db.Get(hash)
 		if err != nil {
 			return n, err
 		}
@@ -96,9 +97,9 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) {
 			return n, errors.New("aborted")
 		case hashes := <-r.hashes:
 			var i int
-			for ; !end && i < len(hashes); i += HashSize {
+			for ; !end && i < len(hashes); i += stream.HashSize {
-				hash := hashes[i : i+HashSize]
+				hash := hashes[i : i+stream.HashSize]
-				chunk, err := r.db.get(hash)
+				chunk, err := r.db.Get(hash)
 				if err != nil {
 					return n, err
 				}
@@ -120,12 +121,12 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) {
 type RemoteSectionServer struct {
 	// quit chan struct{}
 	root []byte
-	db   *DbAccess
+	db   *storage.DBAPI
 	r    *storage.LazyChunkReader
 }
 
 // NewRemoteReader is the constructor for RemoteReader
-func NewRemoteSectionServer(db *DbAccess, r *storage.LazyChunkReader) *RemoteSectionServer {
+func NewRemoteSectionServer(db *storage.DBAPI, r *storage.LazyChunkReader) *RemoteSectionServer {
 	return &RemoteSectionServer{
 		db: db,
 		r:  r,
@@ -134,7 +135,7 @@ func NewRemoteSectionServer(db *DbAccess, r *storage.LazyChunkReader) *RemoteSec
 
 // GetData retrieves the actual chunk from localstore
 func (s *RemoteSectionServer) GetData(key []byte) []byte {
-	chunk, err := s.db.get(storage.Key(key))
+	chunk, err := s.db.Get(storage.Key(key))
 	if err != nil {
 		return nil
 	}
@@ -142,26 +143,26 @@ func (s *RemoteSectionServer) GetData(key []byte) []byte {
 }
 
 // GetBatch retrieves the next batch of hashes from the dbstore
-func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
+func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *stream.HandoverProof, error) {
-	if to > from+batchSize {
+	if to > from+stream.BatchSize {
-		to = from + batchSize
+		to = from + stream.BatchSize
 	}
-	batch := make([]byte, (to-from)*HashSize)
+	batch := make([]byte, (to-from)*stream.HashSize)
 	s.r.ReadAt(batch, int64(from))
 	return batch, from, to, nil, nil
 }
 
 // RegisterRemoteSectionReader registers RemoteSectionReader on light downstream node
-func RegisterRemoteSectionReader(s *Streamer, db *DbAccess) {
+func RegisterRemoteSectionReader(s *stream.Registry, db *storage.DBAPI) {
-	s.RegisterIncomingStreamer("REMOTE_SECTION", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
+	s.RegisterClientFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Client, error) {
 		return NewRemoteSectionReader(t, db), nil
 	})
 }
 
 // RegisterRemoteSectionServer registers RemoteSectionServer outgoing streamer on
 // upstream light server node
-func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) {
+func RegisterRemoteSectionServer(s *stream.Registry, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) {
-	s.RegisterOutgoingStreamer("REMOTE_SECTION", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) {
+	s.RegisterServerFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Server, error) {
 		r := rf(t)
 		return NewRemoteSectionServer(db, r), nil
 	})
@@ -169,16 +170,16 @@ func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *sto
 
 // RegisterRemoteDownloader registers RemoteDownloader incoming streamer
 // on downstream light  node
-// func RegisterRemoteDownloader(s *Streamer, db *DbAccess) {
+// func RegisterRemoteDownloader(s *Streamer, db *storage.DBAPI) {
-// 	s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
+// 	s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (IncomingStreamer, error) {
 // 		return NewRemoteDownloader(t, db), nil
 // 	})
 // }
 //
 // // RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on
 // // upstream light server node
-// func RegisterRemoteDownloadServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) {
+// func RegisterRemoteDownloadServer(s *Streamer, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) {
-// 	s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) {
+// 	s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (OutgoingStreamer, error) {
 // 		r := rf(t)
 // 		return NewRemoteDownloadServer(db, r), nil
 // 	})

swarm/network/protocol.go

@@ -103,7 +103,6 @@ type BzzConfig struct {
 
 // Bzz is the swarm protocol bundle
 type Bzz struct {
-	Streamer *Streamer
 	*Hive
 	localAddr  *BzzAddr
 	mtx        sync.Mutex
@@ -115,9 +114,8 @@ type Bzz struct {
 // * bzz config
 // * overlay driver
 // * peer store
-func NewBzz(config *BzzConfig, kad Overlay, store StateStore, streamer *Streamer) *Bzz {
+func NewBzz(config *BzzConfig, kad Overlay, store StateStore) *Bzz {
 	return &Bzz{
-		Streamer:   streamer,
 		Hive:       NewHive(config.HiveParams, kad, store),
 		localAddr:  &BzzAddr{config.OverlayAddr, config.UnderlayAddr},
 		handshakes: make(map[discover.NodeID]*HandshakeMsg),
@@ -143,7 +141,7 @@ func (b *Bzz) NodeInfo() interface{} {
 // * handshake/hive
 // * discovery
 func (b *Bzz) Protocols() []p2p.Protocol {
-	protocols := []p2p.Protocol{
+	return []p2p.Protocol{
 		{
 			Name:     BzzSpec.Name,
 			Version:  BzzSpec.Version,
@@ -160,17 +158,6 @@ func (b *Bzz) Protocols() []p2p.Protocol {
 			PeerInfo: b.Hive.PeerInfo,
 		},
 	}
-	if b.Streamer != nil {
-		protocols = append(protocols, p2p.Protocol{
-			Name:     StreamerSpec.Name,
-			Version:  StreamerSpec.Version,
-			Length:   StreamerSpec.Length(),
-			Run:      b.RunProtocol(StreamerSpec, b.Streamer.Run),
-			NodeInfo: b.Streamer.NodeInfo,
-			PeerInfo: b.Streamer.PeerInfo,
-		})
-	}
-	return protocols
 }
 
 // APIs returns the APIs offered by bzz
@@ -188,12 +175,12 @@ func (b *Bzz) APIs() []rpc.API {
 // returns a p2p protocol run function that can be assigned to p2p.Protocol#Run field
 // arguments:
 // * p2p protocol spec
-// * run function taking bzzPeer as argument
+// * run function taking BzzPeer as argument
 //   this run function is meant to block for the duration of the protocol session
 //   on return the session is terminated and the peer is disconnected
 // the protocol waits for the bzz handshake is negotiated
-// the overlay address on the bzzPeer is set from the remote handshake
+// the overlay address on the BzzPeer is set from the remote handshake
-func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error {
+func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error {
 	return func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
 		// wait for the bzz protocol to perform the handshake
 		handshake, _ := b.GetHandshake(p.ID())
@@ -206,8 +193,8 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*
 		if handshake.err != nil {
 			return fmt.Errorf("%08x: %s protocol closed: %v", b.BaseAddr()[:4], spec.Name, handshake.err)
 		}
-		// the handshake has succeeded so construct the bzzPeer and run the protocol
+		// the handshake has succeeded so construct the BzzPeer and run the protocol
-		peer := &bzzPeer{
+		peer := &BzzPeer{
 			Peer:       protocols.NewPeer(p, rw, spec),
 			localAddr:  b.localAddr,
 			BzzAddr:    handshake.peerAddr,
@@ -257,22 +244,30 @@ func (b *Bzz) runBzz(p *p2p.Peer, rw p2p.MsgReadWriter) error {
 	return errors.New("received multiple handshakes")
 }
 
-// bzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer)
+// BzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer)
 // implements the Peer interface and all interfaces Peer implements: Addr, OverlayPeer
-type bzzPeer struct {
+type BzzPeer struct {
 	*protocols.Peer           // represents the connection for online peers
 	localAddr       *BzzAddr  // local Peers address
 	*BzzAddr                  // remote address -> implements Addr interface = protocols.Peer
 	lastActive      time.Time // time is updated whenever mutexes are releasing
 }
 
+func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer {
+	return &BzzPeer{
+		Peer:      p,
+		localAddr: addr,
+		BzzAddr:   NewAddrFromNodeID(p.ID()),
+	}
+}
+
 // Off returns the overlay peer record for offline persistance
-func (p *bzzPeer) Off() OverlayAddr {
+func (p *BzzPeer) Off() OverlayAddr {
 	return p.BzzAddr
 }
 
 // LastActive returns the time the peer was last active
-func (p *bzzPeer) LastActive() time.Time {
+func (p *BzzPeer) LastActive() time.Time {
 	return p.lastActive
 }
 

swarm/network/protocol_test.go

@@ -17,16 +17,29 @@
 package network
 
 import (
+	"flag"
 	"fmt"
+	"os"
 	"sync"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/protocols"
 	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
 )
 
+var (
+	adapter  = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker")
+	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
+)
+
+func init() {
+	flag.Parse()
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+}
+
 type testStore struct {
 	sync.Mutex
 
@@ -78,16 +91,16 @@ func HandshakeMsgExchange(lhs, rhs *HandshakeMsg, id discover.NodeID) []p2ptest.
 	}
 }
 
-func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*bzzPeer) error) *bzzTester {
+func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*BzzPeer) error) *bzzTester {
 	cs := make(map[string]chan bool)
 
-	srv := func(p *bzzPeer) error {
+	srv := func(p *BzzPeer) error {
 		defer close(cs[p.ID().String()])
 		return run(p)
 	}
 
 	protocall := func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
-		return srv(&bzzPeer{
+		return srv(&BzzPeer{
 			Peer:      protocols.NewPeer(p, rw, spec),
 			localAddr: addr,
 			BzzAddr:   NewAddrFromNodeID(p.ID()),
@@ -115,7 +128,7 @@ type bzzTester struct {
 
 func newBzzTester(t *testing.T, n int, addr *BzzAddr, pp *p2ptest.TestPeerPool, spec *protocols.Spec, services func(Peer) error) *bzzTester {
 
-	extraservices := func(p *bzzPeer) error {
+	extraservices := func(p *BzzPeer) error {
 		pp.Add(p)
 		defer pp.Remove(p)
 		if services == nil {

swarm/network/request_test.go

@@ -1,444 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"bytes"
-	"context"
-	crand "crypto/rand"
-	"fmt"
-	"io"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/protocols"
-	"github.com/ethereum/go-ethereum/p2p/simulations"
-	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
-	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-func TestStreamerRetrieveRequest(t *testing.T) {
-	tester, streamer, _, teardown, err := newStreamerTester(t)
-	defer teardown()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	peerID := tester.IDs[0]
-
-	streamer.delivery.RequestFromPeers(hash0[:], true)
-
-	err = tester.TestExchanges(p2ptest.Exchange{
-		Label: "RetrieveRequestMsg",
-		Expects: []p2ptest.Expect{
-			p2ptest.Expect{
-				Code: 5,
-				Msg: &RetrieveRequestMsg{
-					Key:       hash0[:],
-					SkipCheck: true,
-				},
-				Peer: peerID,
-			},
-		},
-	})
-
-	if err != nil {
-		t.Fatalf("Expected no error, got %v", err)
-	}
-}
-
-func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
-	tester, streamer, _, teardown, err := newStreamerTester(t)
-	defer teardown()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	peerID := tester.IDs[0]
-
-	chunk := storage.NewChunk(storage.Key(hash0[:]), nil)
-
-	peer := streamer.getPeer(peerID)
-
-	peer.handleSubscribeMsg(&SubscribeMsg{
-		Stream:   retrieveRequestStream,
-		Key:      nil,
-		From:     0,
-		To:       0,
-		Priority: Top,
-	})
-
-	err = tester.TestExchanges(p2ptest.Exchange{
-		Label: "RetrieveRequestMsg",
-		Triggers: []p2ptest.Trigger{
-			p2ptest.Trigger{
-				Code: 5,
-				Msg: &RetrieveRequestMsg{
-					Key: chunk.Key[:],
-				},
-				Peer: peerID,
-			},
-		},
-		Expects: []p2ptest.Expect{
-			p2ptest.Expect{
-				Code: 1,
-				Msg: &OfferedHashesMsg{
-					HandoverProof: nil,
-					Hashes:        nil,
-					From:          0,
-					To:            0,
-				},
-				Peer: peerID,
-			},
-		},
-	})
-
-	expectedError := "exchange 0: 'RetrieveRequestMsg' timed out"
-	if err == nil || err.Error() != expectedError {
-		t.Fatalf("Expected error %v, got %v", expectedError, err)
-	}
-}
-
-// upstream request server receives a retrieve Request and responds with
-// offered hashes or delivery if skipHash is set to true
-func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
-	tester, streamer, localStore, teardown, err := newStreamerTester(t)
-	defer teardown()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	peerID := tester.IDs[0]
-	peer := streamer.getPeer(peerID)
-
-	peer.handleSubscribeMsg(&SubscribeMsg{
-		Stream:   retrieveRequestStream,
-		Key:      nil,
-		From:     0,
-		To:       0,
-		Priority: Top,
-	})
-
-	hash := storage.Key(hash0[:])
-	chunk := storage.NewChunk(hash, nil)
-	chunk.SData = hash
-	localStore.Put(chunk)
-	chunk.WaitToStore()
-
-	err = tester.TestExchanges(p2ptest.Exchange{
-		Label: "RetrieveRequestMsg",
-		Triggers: []p2ptest.Trigger{
-			p2ptest.Trigger{
-				Code: 5,
-				Msg: &RetrieveRequestMsg{
-					Key: hash,
-				},
-				Peer: peerID,
-			},
-		},
-		Expects: []p2ptest.Expect{
-			p2ptest.Expect{
-				Code: 1,
-				Msg: &OfferedHashesMsg{
-					HandoverProof: &HandoverProof{
-						Handover: &Handover{},
-					},
-					Hashes: hash,
-					From:   0,
-					// TODO: why is this 32???
-					To:     32,
-					Key:    []byte{},
-					Stream: retrieveRequestStream,
-				},
-				Peer: peerID,
-			},
-		},
-	})
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hash = storage.Key(hash1[:])
-	chunk = storage.NewChunk(hash, nil)
-	chunk.SData = hash1[:]
-	localStore.Put(chunk)
-	chunk.WaitToStore()
-
-	err = tester.TestExchanges(p2ptest.Exchange{
-		Label: "RetrieveRequestMsg",
-		Triggers: []p2ptest.Trigger{
-			p2ptest.Trigger{
-				Code: 5,
-				Msg: &RetrieveRequestMsg{
-					Key:       hash,
-					SkipCheck: true,
-				},
-				Peer: peerID,
-			},
-		},
-		Expects: []p2ptest.Expect{
-			p2ptest.Expect{
-				Code: 6,
-				Msg: &ChunkDeliveryMsg{
-					Key:   hash,
-					SData: hash,
-				},
-				Peer: peerID,
-			},
-		},
-	})
-
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
-	tester, streamer, localStore, teardown, err := newStreamerTester(t)
-	defer teardown()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
-		return &testIncomingStreamer{
-			t: t,
-		}, nil
-	})
-
-	peerID := tester.IDs[0]
-
-	err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true)
-	if err != nil {
-		t.Fatalf("Expected no error, got %v", err)
-	}
-
-	chunkKey := hash0[:]
-	chunkData := hash1[:]
-	chunk, created := localStore.GetOrCreateRequest(chunkKey)
-
-	if !created {
-		t.Fatal("chunk already exists")
-	}
-	select {
-	case <-chunk.ReqC:
-		t.Fatal("chunk is already received")
-	default:
-	}
-
-	err = tester.TestExchanges(p2ptest.Exchange{
-		Label: "Subscribe message",
-		Expects: []p2ptest.Expect{
-			p2ptest.Expect{
-				Code: 4,
-				Msg: &SubscribeMsg{
-					Stream:   "foo",
-					Key:      []byte{},
-					From:     5,
-					To:       8,
-					Priority: Top,
-				},
-				Peer: peerID,
-			},
-		},
-	},
-		p2ptest.Exchange{
-			Label: "ChunkDeliveryRequest message",
-			Triggers: []p2ptest.Trigger{
-				p2ptest.Trigger{
-					Code: 6,
-					Msg: &ChunkDeliveryMsg{
-						Key:   chunkKey,
-						SData: chunkData,
-					},
-					Peer: peerID,
-				},
-			},
-		})
-
-	if err != nil {
-		t.Fatalf("Expected no error, got %v", err)
-	}
-
-	timeout := time.NewTimer(1 * time.Second)
-
-	select {
-	case <-timeout.C:
-		t.Fatal("timeout receiving chunk")
-	case <-chunk.ReqC:
-	}
-
-	storedChunk, err := localStore.Get(chunkKey)
-	if err != nil {
-		t.Fatalf("Expected no error, got %v", err)
-	}
-
-	if !bytes.Equal(storedChunk.SData, chunkData) {
-		t.Fatal("Retrieved chunk has different data than original")
-	}
-
-}
-
-func TestDeliveryFromNodes(t *testing.T) {
-	testSimulation(t, testDeliveryFromNodes(2, 1, 8100, true))
-	testSimulation(t, testDeliveryFromNodes(2, 1, 8100, false))
-	testSimulation(t, testDeliveryFromNodes(3, 1, 8100, true))
-	testSimulation(t, testDeliveryFromNodes(3, 1, 8100, false))
-}
-
-func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) {
-	return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) {
-		trigger := func(net *simulations.Network) chan discover.NodeID {
-			triggerC := make(chan discover.NodeID)
-			ticker := time.NewTicker(500 * time.Millisecond)
-			go func() {
-				defer ticker.Stop()
-				// we are only testing the pivot node (net.Nodes[0])
-				for range ticker.C {
-					triggerC <- net.Nodes[0].ID()
-				}
-			}()
-			return triggerC
-		}
-
-		action := func(net *simulations.Network) func(context.Context) error {
-			// here we distribute chunks of a random file into localstores of nodes 1 to nodes
-			rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams())
-			rrdpa.Start()
-			// create a retriever dpa for the pivot node
-			dpacs := storage.NewNetStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) })
-			dpa := storage.NewDPA(dpacs, storage.NewChunkerParams())
-			dpa.Start()
-			return func(context.Context) error {
-				defer rrdpa.Stop()
-				// upload an actual random file of size size
-				hash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size))
-				if err != nil {
-					return err
-				}
-				// wait until all chunks stored
-				// TODO: is wait() necessary?
-				wait()
-				// assign the fileHash to a global so that it is available for the check function
-				fileHash = hash
-				go func() {
-					defer dpa.Stop()
-					log.Debug(fmt.Sprintf("retrieve %v", fileHash))
-					// start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks
-					// we must wait for the peer connections to have started before requesting
-					time.Sleep(2 * time.Second)
-					n, err := mustReadAll(dpa, fileHash)
-					log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err)
-				}()
-				return nil
-			}
-		}
-
-		check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) {
-			return func(ctx context.Context, id discover.NodeID) (bool, error) {
-				if id != net.Nodes[0].ID() {
-					return true, nil
-				}
-				select {
-				case <-ctx.Done():
-					return false, ctx.Err()
-				default:
-				}
-				// try to locally retrieve the file to check if retrieve requests have been successful
-				total, err := mustReadAll(dpa, fileHash)
-				log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err))
-				if err != nil || total != size {
-					return false, nil
-				}
-				return true, nil
-				// node := net.GetNode(id)
-				// if node == nil {
-				// 	return false, fmt.Errorf("unknown node: %s", id)
-				// }
-				// client, err := node.Client()
-				// if err != nil {
-				// 	return false, fmt.Errorf("error getting node client: %s", err)
-				// }
-				// var response int
-				// if err := client.Call(&response, "test_haslocal", hash); err != nil {
-				// 	return false, fmt.Errorf("error getting bzz_has response: %s", err)
-				// }
-				// log.Debug(fmt.Sprintf("node has: %v\n%v", id, response))
-				// return response == 0, nil
-			}
-		}
-
-		result, err := runSimulation(nodes, conns, "delivery", NewAddrFromNodeID, action, trigger, check, adapter)
-		if err != nil {
-			return nil, fmt.Errorf("Setting up simulation failed: %v", err)
-		}
-		if result.Error != nil {
-			return nil, fmt.Errorf("Simulation failed: %s", result.Error)
-		}
-		return result, err
-	}
-}
-
-// newDeliveryService
-func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) {
-	id := ctx.Config.ID
-	addr := NewAddrFromNodeID(id)
-	kad := NewKademlia(addr.Over(), NewKadParams())
-	localStore := localStores[nodeCount]
-	dbAccess := NewDbAccess(localStore.(*storage.LocalStore))
-	streamer := NewStreamer(NewDelivery(kad, dbAccess))
-	if nodeCount == 0 {
-		// the delivery service for the pivot node is assigned globally
-		// so that the simulation action call can use it for the
-		// swarm enabled dpa
-		delivery = streamer.delivery
-	}
-	self := &testStreamerService{
-		addr:     addr,
-		streamer: streamer,
-	}
-	self.run = self.runDelivery
-	nodeCount++
-	return self, nil
-}
-
-func (b *testStreamerService) runDelivery(p *p2p.Peer, rw p2p.MsgReadWriter) error {
-	bzzPeer := &bzzPeer{
-		Peer:      protocols.NewPeer(p, rw, StreamerSpec),
-		localAddr: b.addr,
-		BzzAddr:   NewAddrFromNodeID(p.ID()),
-	}
-	b.streamer.delivery.overlay.On(bzzPeer)
-	defer b.streamer.delivery.overlay.Off(bzzPeer)
-	go func() {
-		// each node Subscribes to each other's retrieveRequestStream
-		// need to wait till an aynchronous process registers the peers in streamer.peers
-		// that is used by Subscribe
-		time.Sleep(1 * time.Second)
-		err := b.streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true)
-		if err != nil {
-			log.Warn("error in subscribe", "err", err)
-		}
-	}()
-	return b.streamer.Run(bzzPeer)
-}

swarm/network/stream/common_test.go

@@ -0,0 +1,151 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stream
+
+import (
+	"errors"
+	"flag"
+	"io/ioutil"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+var (
+	adapter  = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker")
+	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
+)
+
+var (
+	defaultSkipCheck bool
+	waitPeerErrC     chan error
+	chunkSize        = 4096
+)
+
+var services = adapters.Services{
+	"streamer": NewStreamerService,
+}
+
+func init() {
+	flag.Parse()
+	// register the Delivery service which will run as a devp2p
+	// protocol when using the exec adapter
+	adapters.RegisterServices(services)
+
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
+
+}
+
+// NewStreamerService
+func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
+	id := ctx.Config.ID
+	addr := toAddr(id)
+	kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+	store := stores[id]
+	db := storage.NewDBAPI(store.(*storage.LocalStore))
+	delivery := NewDelivery(kad, db)
+	deliveries[id] = delivery
+	r := NewRegistry(addr, delivery, store, defaultSkipCheck)
+	RegisterSwarmSyncerServer(r, db)
+	RegisterSwarmSyncerClient(r, db)
+	go func() {
+		waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id))
+	}()
+	return r, nil
+}
+
+func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
+	// setup
+	addr := network.RandomAddr() // tested peers peer address
+	to := network.NewKademlia(addr.OAddr, network.NewKadParams())
+
+	// temp datadir
+	datadir, err := ioutil.TempDir("", "streamer")
+	if err != nil {
+		return nil, nil, nil, func() {}, err
+	}
+	teardown := func() {
+		os.RemoveAll(datadir)
+	}
+
+	localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over())
+	if err != nil {
+		return nil, nil, nil, teardown, err
+	}
+
+	db := storage.NewDBAPI(localStore)
+	delivery := NewDelivery(to, db)
+	streamer := NewRegistry(addr, delivery, localStore, defaultSkipCheck)
+	protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol)
+
+	err = waitForPeers(streamer, 1*time.Second, 1)
+	if err != nil {
+		return nil, nil, nil, nil, errors.New("timeout: peer is not created")
+	}
+
+	return protocolTester, streamer, localStore, teardown, nil
+}
+
+func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error {
+	ticker := time.NewTicker(10 * time.Millisecond)
+	timeoutTimer := time.NewTimer(timeout)
+	for {
+		select {
+		case <-ticker.C:
+			if streamer.peersCount() >= expectedPeers {
+				return nil
+			}
+		case <-timeoutTimer.C:
+			return errors.New("timeout")
+		}
+	}
+}
+
+type roundRobinStore struct {
+	index  uint32
+	stores []storage.ChunkStore
+}
+
+func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore {
+	return &roundRobinStore{
+		stores: stores,
+	}
+}
+
+func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) {
+	return nil, errors.New("get not well defined on round robin store")
+}
+
+func (rrs *roundRobinStore) Put(chunk *storage.Chunk) {
+	i := atomic.AddUint32(&rrs.index, 1)
+	idx := int(i) % len(rrs.stores)
+	rrs.stores[idx].Put(chunk)
+}
+
+func (rrs *roundRobinStore) Close() {
+	for _, store := range rrs.stores {
+		store.Close()
+	}
+}

swarm/network/stream/delivery.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,60 +14,63 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package network
+package stream
 
 import (
 	"errors"
-	"fmt"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-const retrieveRequestStream = "RETRIEVE_REQUEST"
+const (
+	swarmChunkServerStreamName = "RETRIEVE_REQUEST"
+	deliveryCap                = 32
+)
 
 type Delivery struct {
-	dbAccess *DbAccess
+	db       *storage.DBAPI
-	overlay  Overlay
+	overlay  network.Overlay
 	receiveC chan *ChunkDeliveryMsg
-	getPeer  func(discover.NodeID) *StreamerPeer
+	getPeer  func(discover.NodeID) *Peer
 	quit     chan struct{}
 }
 
-func NewDelivery(overlay Overlay, dbAccess *DbAccess) *Delivery {
+func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery {
-	self := &Delivery{
+	d := &Delivery{
-		dbAccess: dbAccess,
+		db:       db,
 		overlay:  overlay,
-		receiveC: make(chan *ChunkDeliveryMsg, 10),
+		receiveC: make(chan *ChunkDeliveryMsg, deliveryCap),
 	}
 
-	go self.processReceivedChunks()
+	go d.processReceivedChunks()
-	return self
+	return d
 }
 
-// RetrieveRequestStreamer implements OutgoingStreamer
+// SwarmChunkServer implements OutgoingStreamer
-type RetrieveRequestStreamer struct {
+type SwarmChunkServer struct {
 	deliveryC  chan []byte
 	batchC     chan []byte
-	dbAccess   *DbAccess
+	db         *storage.DBAPI
 	currentLen uint64
 }
 
-// NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor
+// NewSwarmChunkServer is SwarmChunkServer constructor
-func NewRetrieveRequestStreamer(dbAccess *DbAccess) *RetrieveRequestStreamer {
+func NewSwarmChunkServer(db *storage.DBAPI) *SwarmChunkServer {
-	s := &RetrieveRequestStreamer{
+	s := &SwarmChunkServer{
-		deliveryC: make(chan []byte),
+		deliveryC: make(chan []byte, deliveryCap),
 		batchC:    make(chan []byte),
-		dbAccess:  dbAccess,
+		db:        db,
 	}
 	go s.processDeliveries()
 	return s
 }
 
 // processDeliveries handles delivered chunk hashes
-func (s *RetrieveRequestStreamer) processDeliveries() {
+func (s *SwarmChunkServer) processDeliveries() {
 	var hashes []byte
 	var batchC chan []byte
 	for {
@@ -83,7 +86,7 @@ func (s *RetrieveRequestStreamer) processDeliveries() {
 }
 
 // SetNextBatch
-func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) {
+func (s *SwarmChunkServer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) {
 	hashes = <-s.batchC
 	from = s.currentLen
 	s.currentLen += uint64(len(hashes))
@@ -92,8 +95,8 @@ func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from
 }
 
 // GetData retrives chunk data from db store
-func (s *RetrieveRequestStreamer) GetData(key []byte) []byte {
+func (s *SwarmChunkServer) GetData(key []byte) []byte {
-	chunk, _ := s.dbAccess.get(storage.Key(key))
+	chunk, _ := s.db.Get(storage.Key(key))
 	return chunk.SData
 }
 
@@ -103,16 +106,18 @@ type RetrieveRequestMsg struct {
 	SkipCheck bool
 }
 
-func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRequestMsg) error {
+func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) error {
-	s, err := sp.getOutgoingStreamer(retrieveRequestStream)
+	log.Debug("received request", "peer", sp.ID(), "hash", req.Key)
+	s, err := sp.getServer(swarmChunkServerStreamName)
 	if err != nil {
 		return err
 	}
-	streamer := s.OutgoingStreamer.(*RetrieveRequestStreamer)
+	streamer := s.Server.(*SwarmChunkServer)
-	chunk, created := self.dbAccess.getOrCreateRequest(req.Key)
+	chunk, created := d.db.GetOrCreateRequest(req.Key)
 	if chunk.ReqC != nil {
 		if created {
-			if err := self.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil {
+			if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil {
+				log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err)
 				return nil
 			}
 		}
@@ -122,15 +127,17 @@ func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRe
 
 			select {
 			case <-chunk.ReqC:
-			case <-self.quit:
+			case <-d.quit:
 				return
 			case <-t.C:
 				return
 			}
 
 			if req.SkipCheck {
-				sp.Deliver(chunk, s.priority)
+				err := sp.Deliver(chunk, s.priority)
-				return
+				if err != nil {
+					sp.Drop(err)
+				}
 			}
 			streamer.deliveryC <- chunk.Key[:]
 		}()
@@ -138,6 +145,7 @@ func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRe
 	}
 	// TODO: call the retrieve function of the outgoing syncer
 	if req.SkipCheck {
+		log.Trace("deliver", "peer", sp.ID(), "hash", chunk.Key)
 		return sp.Deliver(chunk, s.priority)
 	}
 	streamer.deliveryC <- chunk.Key[:]
@@ -149,57 +157,66 @@ type ChunkDeliveryMsg struct {
 	SData []byte // the stored chunk Data (incl size)
 }
 
-func (self *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error {
+func (d *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error {
-	chunk, err := self.dbAccess.get(req.Key)
+	d.receiveC <- req
-	if err != nil {
-		return err
-	}
-
-	self.receiveC <- req
-
-	log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, self))
 	return nil
 }
 
-func (self *Delivery) processReceivedChunks() {
+func (d *Delivery) processReceivedChunks() {
-	for req := range self.receiveC {
+R:
-		chunk, err := self.dbAccess.get(req.Key)
+	for req := range d.receiveC {
+		// this should be has locally
+		chunk, err := d.db.Get(req.Key)
 		if err != nil {
-			continue
+			log.Error("not in db? ", "key", req.Key, "chunk", chunk)
+			continue R
+		}
+		if chunk.ReqC == nil {
+			continue R
 		}
-		chunk.SData = req.SData
 		select {
 		case <-chunk.ReqC:
+			continue R
 		default:
-			self.dbAccess.put(chunk)
-			close(chunk.ReqC)
 		}
+		chunk.SData = req.SData
+		d.db.Put(chunk)
+		log.Warn("reecived delivery", "hash", chunk.Key)
+		chunk.WaitToStore()
+		log.Warn("received delivery stored", "hash", chunk.Key)
+		close(chunk.ReqC)
+		log.Warn("received delivery requesters notified", "hash", chunk.Key)
 	}
 }
 
 // RequestFromPeers sends a chunk retrieve request to
-func (self *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error {
+func (d *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error {
 	var success bool
-	self.overlay.EachConn(hash, 255, func(p OverlayConn, po int, nn bool) bool {
+	var err error
-		spId := p.(Peer).ID()
+	log.Warn("request", "hash", hash)
+	d.overlay.EachConn(hash, 255, func(p network.OverlayConn, po int, nn bool) bool {
+		spId := p.(*network.BzzPeer).ID()
 		for _, p := range peersToSkip {
 			if p == spId {
+				log.Warn("skip peer", "peer", spId)
 				return true
 			}
 		}
-		sp := self.getPeer(spId)
+		sp := d.getPeer(spId)
+		if sp == nil {
+			log.Warn("peer not found", "id", spId)
+			return true
+		}
 		// TODO: skip light nodes that do not accept retrieve requests
-		err := sp.SendPriority(&RetrieveRequestMsg{
+		err = sp.SendPriority(&RetrieveRequestMsg{
 			Key:       hash,
 			SkipCheck: skipCheck,
 		}, Top)
-		if err == nil {
 		success = true
-		}
 		return false
 	})
 	if success {
-		return nil
+		return err
 	}
 	return errors.New("no peer found")
 }

swarm/network/stream/delivery_test.go

@@ -0,0 +1,670 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stream
+
+import (
+	"bytes"
+	"context"
+	crand "crypto/rand"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+var (
+	deliveries map[discover.NodeID]*Delivery
+	stores     map[discover.NodeID]storage.ChunkStore
+	toAddr     func(discover.NodeID) *network.BzzAddr
+	peerCount  func(discover.NodeID) int
+)
+
+func TestStreamerRetrieveRequest(t *testing.T) {
+	tester, streamer, _, teardown, err := newStreamerTester(t)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	peerID := tester.IDs[0]
+
+	streamer.delivery.RequestFromPeers(hash0[:], true)
+
+	err = tester.TestExchanges(p2ptest.Exchange{
+		Label: "RetrieveRequestMsg",
+		Expects: []p2ptest.Expect{
+			p2ptest.Expect{
+				Code: 5,
+				Msg: &RetrieveRequestMsg{
+					Key:       hash0[:],
+					SkipCheck: true,
+				},
+				Peer: peerID,
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatalf("Expected no error, got %v", err)
+	}
+}
+
+func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
+	tester, streamer, _, teardown, err := newStreamerTester(t)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	peerID := tester.IDs[0]
+
+	chunk := storage.NewChunk(storage.Key(hash0[:]), nil)
+
+	peer := streamer.getPeer(peerID)
+
+	peer.handleSubscribeMsg(&SubscribeMsg{
+		Stream:   swarmChunkServerStreamName,
+		Key:      nil,
+		From:     0,
+		To:       0,
+		Priority: Top,
+	})
+
+	err = tester.TestExchanges(p2ptest.Exchange{
+		Label: "RetrieveRequestMsg",
+		Triggers: []p2ptest.Trigger{
+			p2ptest.Trigger{
+				Code: 5,
+				Msg: &RetrieveRequestMsg{
+					Key: chunk.Key[:],
+				},
+				Peer: peerID,
+			},
+		},
+		Expects: []p2ptest.Expect{
+			p2ptest.Expect{
+				Code: 1,
+				Msg: &OfferedHashesMsg{
+					HandoverProof: nil,
+					Hashes:        nil,
+					From:          0,
+					To:            0,
+				},
+				Peer: peerID,
+			},
+		},
+	})
+
+	expectedError := "exchange 0: 'RetrieveRequestMsg' timed out"
+	if err == nil || err.Error() != expectedError {
+		t.Fatalf("Expected error %v, got %v", expectedError, err)
+	}
+}
+
+// upstream request server receives a retrieve Request and responds with
+// offered hashes or delivery if skipHash is set to true
+func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
+	tester, streamer, localStore, teardown, err := newStreamerTester(t)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	peerID := tester.IDs[0]
+	peer := streamer.getPeer(peerID)
+
+	peer.handleSubscribeMsg(&SubscribeMsg{
+		Stream:   swarmChunkServerStreamName,
+		Key:      nil,
+		From:     0,
+		To:       0,
+		Priority: Top,
+	})
+
+	hash := storage.Key(hash0[:])
+	chunk := storage.NewChunk(hash, nil)
+	chunk.SData = hash
+	localStore.Put(chunk)
+	chunk.WaitToStore()
+
+	err = tester.TestExchanges(p2ptest.Exchange{
+		Label: "RetrieveRequestMsg",
+		Triggers: []p2ptest.Trigger{
+			p2ptest.Trigger{
+				Code: 5,
+				Msg: &RetrieveRequestMsg{
+					Key: hash,
+				},
+				Peer: peerID,
+			},
+		},
+		Expects: []p2ptest.Expect{
+			p2ptest.Expect{
+				Code: 1,
+				Msg: &OfferedHashesMsg{
+					HandoverProof: &HandoverProof{
+						Handover: &Handover{},
+					},
+					Hashes: hash,
+					From:   0,
+					// TODO: why is this 32???
+					To:     32,
+					Key:    []byte{},
+					Stream: swarmChunkServerStreamName,
+				},
+				Peer: peerID,
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hash = storage.Key(hash1[:])
+	chunk = storage.NewChunk(hash, nil)
+	chunk.SData = hash1[:]
+	localStore.Put(chunk)
+	chunk.WaitToStore()
+
+	err = tester.TestExchanges(p2ptest.Exchange{
+		Label: "RetrieveRequestMsg",
+		Triggers: []p2ptest.Trigger{
+			p2ptest.Trigger{
+				Code: 5,
+				Msg: &RetrieveRequestMsg{
+					Key:       hash,
+					SkipCheck: true,
+				},
+				Peer: peerID,
+			},
+		},
+		Expects: []p2ptest.Expect{
+			p2ptest.Expect{
+				Code: 6,
+				Msg: &ChunkDeliveryMsg{
+					Key:   hash,
+					SData: hash,
+				},
+				Peer: peerID,
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
+	tester, streamer, localStore, teardown, err := newStreamerTester(t)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) {
+		return &testClient{
+			t: t,
+		}, nil
+	})
+
+	peerID := tester.IDs[0]
+
+	err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true)
+	if err != nil {
+		t.Fatalf("Expected no error, got %v", err)
+	}
+
+	chunkKey := hash0[:]
+	chunkData := hash1[:]
+	chunk, created := localStore.GetOrCreateRequest(chunkKey)
+
+	if !created {
+		t.Fatal("chunk already exists")
+	}
+	select {
+	case <-chunk.ReqC:
+		t.Fatal("chunk is already received")
+	default:
+	}
+
+	err = tester.TestExchanges(p2ptest.Exchange{
+		Label: "Subscribe message",
+		Expects: []p2ptest.Expect{
+			p2ptest.Expect{
+				Code: 4,
+				Msg: &SubscribeMsg{
+					Stream:   "foo",
+					Key:      []byte{},
+					From:     5,
+					To:       8,
+					Priority: Top,
+				},
+				Peer: peerID,
+			},
+		},
+	},
+		p2ptest.Exchange{
+			Label: "ChunkDeliveryRequest message",
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 6,
+					Msg: &ChunkDeliveryMsg{
+						Key:   chunkKey,
+						SData: chunkData,
+					},
+					Peer: peerID,
+				},
+			},
+		})
+
+	if err != nil {
+		t.Fatalf("Expected no error, got %v", err)
+	}
+
+	timeout := time.NewTimer(1 * time.Second)
+
+	select {
+	case <-timeout.C:
+		t.Fatal("timeout receiving chunk")
+	case <-chunk.ReqC:
+	}
+
+	storedChunk, err := localStore.Get(chunkKey)
+	if err != nil {
+		t.Fatalf("Expected no error, got %v", err)
+	}
+
+	if !bytes.Equal(storedChunk.SData, chunkData) {
+		t.Fatal("Retrieved chunk has different data than original")
+	}
+
+}
+
+func TestDeliveryFromNodes(t *testing.T) {
+	testDeliveryFromNodes(t, 2, 1, dataChunkCount, true)
+	testDeliveryFromNodes(t, 2, 1, dataChunkCount, false)
+	testDeliveryFromNodes(t, 4, 1, dataChunkCount, true)
+	testDeliveryFromNodes(t, 4, 1, dataChunkCount, false)
+	testDeliveryFromNodes(t, 8, 1, dataChunkCount, true)
+	testDeliveryFromNodes(t, 8, 1, dataChunkCount, false)
+	testDeliveryFromNodes(t, 16, 1, dataChunkCount, true)
+	testDeliveryFromNodes(t, 16, 1, dataChunkCount, false)
+}
+
+func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) {
+	defaultSkipCheck = skipCheck
+	toAddr = network.NewAddrFromNodeID
+	conf := &streamTesting.RunConfig{
+		Adapter:   *adapter,
+		NodeCount: nodes,
+		ConnLevel: conns,
+		ToAddr:    toAddr,
+		Services:  services,
+	}
+
+	sim, teardown, err := streamTesting.NewSimulation(conf)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	stores = make(map[discover.NodeID]storage.ChunkStore)
+	deliveries = make(map[discover.NodeID]*Delivery)
+	for i, id := range sim.IDs {
+		stores[id] = sim.Stores[i]
+	}
+	peerCount = func(id discover.NodeID) int {
+		if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
+			return 1
+		}
+		return 2
+	}
+
+	// here we distribute chunks of a random file into Stores of nodes 1 to nodes
+	rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams())
+	rrdpa.Start()
+	size := chunkCount * chunkSize
+	fileHash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size))
+	// wait until all chunks stored
+	wait()
+	defer rrdpa.Stop()
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	errc := make(chan error, 1)
+	waitPeerErrC = make(chan error)
+	quitC := make(chan struct{})
+
+	action := func(ctx context.Context) error {
+		// each node Subscribes to each other's swarmChunkServerStreamName
+		// need to wait till an aynchronous process registers the peers in streamer.peers
+		// that is used by Subscribe
+		// using a global err channel to share betweem action and node service
+		i := 0
+		for err := range waitPeerErrC {
+			if err != nil {
+				return fmt.Errorf("error waiting for peers: %s", err)
+			}
+			i++
+			if i == nodes {
+				break
+			}
+		}
+
+		// each node subscribes to the upstream swarm chunk server stream
+		// which responds to chunk retrieve requests all but the last node in the chain does not
+		var j int
+		err := sim.CallClient(func(client *rpc.Client) error {
+			err := streamTesting.WatchDisconnections(sim.IDs[j], client, errc, quitC)
+			if err != nil {
+				return err
+			}
+			ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
+			defer cancel()
+			j++
+			sid := sim.IDs[j]
+			return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false)
+		}, sim.IDs[0:nodes-1]...)
+		if err != nil {
+			return err
+		}
+
+		// create a retriever dpa for the pivot node
+		delivery := deliveries[sim.IDs[0]]
+		retrieveFunc := func(chunk *storage.Chunk) error {
+			return delivery.RequestFromPeers(chunk.Key[:], skipCheck)
+		}
+		netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc)
+		dpa := storage.NewDPA(netStore, storage.NewChunkerParams())
+		dpa.Start()
+
+		go func() {
+			defer dpa.Stop()
+			// start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks
+			// we must wait for the peer connections to have started before requesting
+			n, err := readAll(dpa, fileHash)
+			log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err)
+			if err != nil {
+				errc <- fmt.Errorf("requesting chunks action error: %v", err)
+			}
+		}()
+		return nil
+	}
+	checkC := make(chan struct{})
+	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
+		defer func() { checkC <- struct{}{} }()
+		select {
+		case err := <-errc:
+			return false, err
+		case <-ctx.Done():
+			return false, ctx.Err()
+		default:
+		}
+		var total int64
+		err := sim.CallClient(func(client *rpc.Client) error {
+			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			defer cancel()
+			return client.CallContext(ctx, &total, "stream_readAll", common.BytesToHash(fileHash))
+		}, id)
+		log.Info(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err))
+		if err != nil || total != int64(size) {
+			return false, nil
+		}
+		close(quitC)
+		return true, nil
+	}
+
+	conf.Step = &simulations.Step{
+		Action:  action,
+		Trigger: streamTesting.PivotTrigger(10*time.Millisecond, checkC, sim.IDs[0]),
+		// we are only testing the pivot node (net.Nodes[0])
+		Expect: &simulations.Expectation{
+			Nodes: sim.IDs[0:1],
+			Check: check,
+		},
+	}
+	startedAt := time.Now()
+	result, err := sim.Run(conf)
+	finishedAt := time.Now()
+	if err != nil {
+		t.Fatalf("Setting up simulation failed: %v", err)
+	}
+	if result.Error != nil {
+		t.Fatalf("Simulation failed: %s", result.Error)
+	}
+	streamTesting.CheckResult(t, result, startedAt, finishedAt)
+}
+
+func BenchmarkDeliveryFromNodesWithoutCheck(b *testing.B) {
+	for chunks := 32; chunks <= 128; chunks *= 2 {
+		for i := 2; i < 32; i *= 2 {
+			b.Run(
+				fmt.Sprintf("nodes=%v,chunks=%v", i, chunks),
+				func(b *testing.B) {
+					benchmarkDeliveryFromNodes(b, i, 1, chunks, true)
+				},
+			)
+		}
+	}
+}
+
+func BenchmarkDeliveryFromNodesWithCheck(b *testing.B) {
+	for chunks := 32; chunks <= 128; chunks *= 2 {
+		for i := 2; i < 32; i *= 2 {
+			b.Run(
+				fmt.Sprintf("nodes=%v,chunks=%v", i, chunks),
+				func(b *testing.B) {
+					benchmarkDeliveryFromNodes(b, i, 1, chunks, false)
+				},
+			)
+		}
+	}
+}
+
+func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skipCheck bool) {
+	toAddr = network.NewAddrFromNodeID
+	conf := &streamTesting.RunConfig{
+		Adapter:   *adapter,
+		NodeCount: nodes,
+		ConnLevel: conns,
+		ToAddr:    toAddr,
+		Services:  services,
+	}
+	defaultSkipCheck = skipCheck
+	sim, teardown, err := streamTesting.NewSimulation(conf)
+	defer teardown()
+	if err != nil {
+		b.Fatal(err.Error())
+	}
+	stores = make(map[discover.NodeID]storage.ChunkStore)
+	deliveries = make(map[discover.NodeID]*Delivery)
+	for i, id := range sim.IDs {
+		stores[id] = sim.Stores[i]
+	}
+	peerCount = func(id discover.NodeID) int {
+		if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
+			return 1
+		}
+		return 2
+	}
+	// create a dpa for the last node in the chain which we are gonna write to
+	remoteDpa := storage.NewDPA(sim.Stores[nodes-1], storage.NewChunkerParams())
+	remoteDpa.Start()
+	defer remoteDpa.Stop()
+
+	// wait channel for all nodes all peer connections to set up
+	waitPeerErrC = make(chan error)
+	// channel to signal simulation initialisation with action call complete
+	// or node disconnections
+	simErrC := make(chan error)
+	quitC := make(chan struct{})
+
+	action := func(ctx context.Context) error {
+		// each node Subscribes to each other's swarmChunkServerStreamName
+		// need to wait till an aynchronous process registers the peers in streamer.peers
+		// that is used by Subscribe
+		// waitPeerErrC using a global err channel to share betweem action and node service
+		i := 0
+		for err := range waitPeerErrC {
+			if err != nil {
+				return fmt.Errorf("error waiting for peers: %s", err)
+			}
+			i++
+			if i == nodes {
+				break
+			}
+		}
+
+		// each node except the last one subscribes to the upstream swarm chunk server stream
+		// which responds to chunk retrieve requests
+		var j int
+		simErrC <- sim.CallClient(func(client *rpc.Client) error {
+			err := streamTesting.WatchDisconnections(sim.IDs[j], client, simErrC, quitC)
+			if err != nil {
+				return err
+			}
+			ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
+			defer cancel()
+			j++
+			sid := sim.IDs[j] // the upstream peer's id
+			return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false)
+		}, sim.IDs[0:nodes-1]...)
+		// signal to the benchmark that setup is complete
+		return err
+	}
+
+	// the check function is only triggered when the benchmark finishes
+	checkC := make(chan error)
+	trigger := make(chan discover.NodeID)
+	check := func(ctx context.Context, id discover.NodeID) (_ bool, err error) {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+		case err = <-checkC:
+		}
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+
+	conf.Step = &simulations.Step{
+		Action:  action,
+		Trigger: trigger,
+		// we are only testing the pivot node (net.Nodes[0])
+		Expect: &simulations.Expectation{
+			Nodes: sim.IDs[0:1],
+			Check: check,
+		},
+	}
+
+	// run the simulation in the background
+	errc := make(chan error)
+	go func() {
+		_, err := sim.Run(conf)
+		errc <- err
+	}()
+
+	// wait for simulation action to complete stream subscriptions
+	err = <-simErrC
+	if err != nil {
+		b.Fatalf("simulation failed to initialise. expected no error. got %v", err)
+	}
+	go func() {
+		for {
+			var err error
+			select {
+			case err = <-simErrC:
+			case <-quitC:
+			}
+			trigger <- sim.IDs[0]
+			checkC <- err
+		}
+	}()
+
+	// create a retriever dpa for the pivot node
+	// by now deliveries are set for each node by the streamer service
+	delivery := deliveries[sim.IDs[0]]
+	retrieveFunc := func(chunk *storage.Chunk) error {
+		return delivery.RequestFromPeers(chunk.Key[:], skipCheck)
+	}
+	netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc)
+
+	// benchmark loop
+	b.ResetTimer()
+	b.StopTimer()
+	for i := 0; i < b.N; i++ {
+		// uploading chunkCount random chunks to the last node
+		hashes := make([]storage.Key, chunkCount)
+		for i := 0; i < chunkCount; i++ {
+			// create actual size real chunks
+			hash, wait, err := remoteDpa.Store(io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize))
+			// wait until all chunks stored
+			wait()
+			if err != nil {
+				b.Fatalf("expected no error. got %v", err)
+			}
+			// collect the hashes
+			hashes[i] = hash
+		}
+		// now benchmark the actual retrieval
+		// netstore.Get is called for each hash in a go routine and errors are collected
+		b.StartTimer()
+		errs := make(chan error)
+		for _, hash := range hashes {
+			go func(h storage.Key) {
+				_, err := netStore.Get(h)
+				log.Warn("test check netstore get", "hash", h, "err", err)
+				errs <- err
+			}(hash)
+		}
+		// count and report retrieval errors
+		// if there are misses then chunk timeout is too low for the distance and volume (?)
+		var total, misses int
+		for err := range errs {
+			if err != nil {
+				log.Warn(err.Error())
+				misses++
+			}
+			total++
+			if total == chunkCount {
+				break
+			}
+		}
+		b.StopTimer()
+		if misses > 0 {
+			simErrC <- fmt.Errorf("%v chunk not found out of %v", misses, total)
+		}
+	}
+	// benchmark over, trigger the check function to conclude the simulation
+	close(quitC)
+	err = <-errc
+	if err != nil {
+		b.Fatalf("expected no error. got %v", err)
+	}
+}

swarm/network/stream/messages.go

@@ -0,0 +1,231 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stream
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/log"
+	bv "github.com/ethereum/go-ethereum/swarm/network/bitvector"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// Handover represents a statement that the upstream peer hands over the stream section
+type Handover struct {
+	Stream     string // name of stream
+	Start, End uint64 // index of hashes
+	Root       []byte // Root hash for indexed segment inclusion proofs
+}
+
+// HandoverProof represents a signed statement that the upstream peer handed over the stream section
+type HandoverProof struct {
+	Sig []byte // Sign(Hash(Serialisation(Handover)))
+	*Handover
+}
+
+// Takeover represents a statement that downstream peer took over (stored all data)
+// handed over
+type Takeover Handover
+
+//  TakeoverProof represents a signed statement that the downstream peer took over
+// the stream section
+type TakeoverProof struct {
+	Sig []byte // Sign(Hash(Serialisation(Takeover)))
+	*Takeover
+}
+
+// TakeoverProofMsg is the protocol msg sent by downstream peer
+type TakeoverProofMsg TakeoverProof
+
+// String pretty prints TakeoverProofMsg
+func (m TakeoverProofMsg) String() string {
+	return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", m.Stream, m.Start, m.End, m.Root, m.Sig)
+}
+
+// SubcribeMsg is the protocol msg for requesting a stream(section)
+type SubscribeMsg struct {
+	Stream   string
+	Key      []byte
+	From, To uint64
+	Priority uint8 // delivered on priority channel
+}
+
+func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) error {
+	f, err := p.streamer.GetServerFunc(req.Stream)
+	if err != nil {
+		return err
+	}
+	s, err := f(p, req.Key)
+	if err != nil {
+		return err
+	}
+	os, err := p.setServer(req.Stream, req.Key, s, req.Priority)
+	if err != nil {
+		return nil
+	}
+	log.Debug("received subscription", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
+	go func() {
+		if err := p.SendOfferedHashes(os, req.From, req.To); err != nil {
+			p.Drop(err)
+		}
+	}()
+	return nil
+}
+
+// OfferedHashesMsg is the protocol msg for offering to hand over a
+// stream section
+type OfferedHashesMsg struct {
+	Stream         string // name of Stream
+	Key            []byte // subtype or key
+	From, To       uint64 // peer and db-specific entry count
+	Hashes         []byte // stream of hashes (128)
+	*HandoverProof        // HandoverProof
+}
+
+// String pretty prints OfferedHashesMsg
+func (m OfferedHashesMsg) String() string {
+	return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", m.Stream, m.From, m.To, len(m.Hashes)/HashSize)
+}
+
+// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface
+// Filter method
+func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error {
+	sk := req.Stream
+	sk += keyToString(req.Key)
+	s, err := p.getClient(sk)
+	if err != nil {
+		return err
+	}
+	hashes := req.Hashes
+	want, err := bv.New(len(hashes) / HashSize)
+	if err != nil {
+		return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err)
+	}
+	wg := sync.WaitGroup{}
+	for i := 0; i < len(hashes); i += HashSize {
+		hash := hashes[i : i+HashSize]
+		if wait := s.NeedData(hash); wait != nil {
+			want.Set(i/HashSize, true)
+			wg.Add(1)
+			// create request and wait until the chunk data arrives and is stored
+			go func(w func()) {
+				w()
+				wg.Done()
+			}(wait)
+		}
+	}
+	go func() {
+		wg.Wait()
+		s.next <- s.batchDone(p, req, hashes)
+	}()
+	// only send wantedKeysMsg if all missing chunks of the previous batch arrived
+	// except
+	if s.live {
+		s.sessionAt = req.From
+	}
+	from, to := s.nextBatch(req.To)
+	log.Trace("received offered batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
+	if from == to {
+		return nil
+	}
+
+	msg := &WantedHashesMsg{
+		Stream: req.Stream,
+		Key:    req.Key,
+		Want:   want.Bytes(),
+		From:   from,
+		To:     to,
+	}
+	go func() {
+		select {
+		case err := <-s.next:
+			if err != nil {
+				p.Drop(err)
+				return
+			}
+		case <-s.quit:
+			return
+		}
+		log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To)
+		err := p.SendPriority(msg, s.priority)
+		if err != nil {
+			p.Drop(err)
+		}
+	}()
+	return nil
+}
+
+// WantedHashesMsg is the protocol msg data for signaling which hashes
+// offered in OfferedHashesMsg downstream peer actually wants sent over
+type WantedHashesMsg struct {
+	Stream   string // name of stream
+	Key      []byte // subtype or key
+	Want     []byte // bitvector indicating which keys of the batch needed
+	From, To uint64 // next interval offset - empty if not to be continued
+}
+
+// String pretty prints WantedHashesMsg
+func (m WantedHashesMsg) String() string {
+	return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", m.Stream, m.Want, m.From, m.To)
+}
+
+// handleWantedHashesMsg protocol msg handler
+// * sends the next batch of unsynced keys
+// * sends the actual data chunks as per WantedHashesMsg
+func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error {
+	log.Trace("received wanted batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
+	s, err := p.getServer(req.Stream + keyToString(req.Key))
+	if err != nil {
+		return err
+	}
+	hashes := s.currentBatch
+	// launch in go routine since GetBatch blocks until new hashes arrive
+	go p.SendOfferedHashes(s, req.From, req.To)
+	l := len(hashes) / HashSize
+	want, err := bv.NewFromBytes(req.Want, l)
+	if err != nil {
+		return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err)
+	}
+	for i := 0; i < l; i++ {
+		if want.Get(i) {
+			hash := hashes[i*HashSize : (i+1)*HashSize]
+			data := s.GetData(hash)
+			if data == nil {
+				return errors.New("not found")
+			}
+			chunk := storage.NewChunk(hash, nil)
+			chunk.SData = data
+			if err := p.Deliver(chunk, s.priority); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (p *Peer) handleTakeoverProofMsg(req *TakeoverProofMsg) error {
+	_, err := p.getServer(req.Stream)
+	if err != nil {
+		return err
+	}
+	// store the strongest takeoverproof for the stream in streamer
+	return nil
+}
+
+type UnsubscribeMsg struct{}

swarm/network/stream/peer.go

@@ -0,0 +1,169 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stream
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/protocols"
+	pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+var sendTimeout = 5 * time.Second
+
+// Peer is the Peer extention for the streaming protocol
+type Peer struct {
+	*protocols.Peer
+	streamer *Registry
+	pq       *pq.PriorityQueue
+	serverMu sync.RWMutex
+	clientMu sync.RWMutex
+	servers  map[string]*server
+	clients  map[string]*client
+	quit     chan struct{}
+}
+
+// NewPeer is the constructor for Peer
+func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer {
+	p := &Peer{
+		Peer:     peer,
+		pq:       pq.New(int(PriorityQueue), PriorityQueueCap),
+		streamer: streamer,
+		servers:  make(map[string]*server),
+		clients:  make(map[string]*client),
+		quit:     make(chan struct{}),
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	go p.pq.Run(ctx, func(i interface{}) { p.Send(i) })
+	go func() {
+		<-p.quit
+		cancel()
+	}()
+	return p
+}
+
+// Deliver sends a storeRequestMsg protocol message to the peer
+func (p *Peer) Deliver(chunk *storage.Chunk, priority uint8) error {
+	msg := &ChunkDeliveryMsg{
+		Key:   chunk.Key,
+		SData: chunk.SData,
+	}
+	return p.SendPriority(msg, priority)
+}
+
+// SendPriority sends message to the peer using the outgoing priority queue
+func (p *Peer) SendPriority(msg interface{}, priority uint8) error {
+	ctx, cancel := context.WithTimeout(context.Background(), sendTimeout)
+	defer cancel()
+	return p.pq.Push(ctx, msg, int(priority))
+}
+
+// SendOfferedHashes sends OfferedHashesMsg protocol msg
+func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error {
+	hashes, from, to, proof, err := s.SetNextBatch(f, t)
+	if err != nil {
+		return err
+	}
+	if proof == nil {
+		proof = &HandoverProof{
+			Handover: &Handover{},
+		}
+	}
+	s.currentBatch = hashes
+	msg := &OfferedHashesMsg{
+		HandoverProof: proof,
+		Hashes:        hashes,
+		From:          from,
+		To:            to,
+		Stream:        s.stream,
+		Key:           s.key,
+	}
+	log.Warn("Swarm syncer offer batch", "peer", p.ID(), "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to)
+	return p.SendPriority(msg, s.priority)
+}
+
+func (p *Peer) getServer(s string) (*server, error) {
+	p.serverMu.RLock()
+	defer p.serverMu.RUnlock()
+
+	server := p.servers[s]
+	if server == nil {
+		return nil, fmt.Errorf("server '%v' not provided to peer %v", s, p.ID())
+	}
+	return server, nil
+}
+
+func (p *Peer) getClient(s string) (*client, error) {
+	p.clientMu.RLock()
+	defer p.clientMu.RUnlock()
+
+	client := p.clients[s]
+	if client == nil {
+		return nil, fmt.Errorf("client '%v' not provided to peer %v", s, p.ID())
+	}
+	return client, nil
+}
+
+func (p *Peer) setServer(s string, key []byte, o Server, priority uint8) (*server, error) {
+	p.serverMu.Lock()
+	defer p.serverMu.Unlock()
+
+	sk := s + keyToString(key)
+	if p.servers[sk] != nil {
+		return nil, fmt.Errorf("server %v already registered", sk)
+	}
+	os := &server{
+		Server:   o,
+		priority: priority,
+		stream:   s,
+		key:      key,
+	}
+	p.servers[sk] = os
+	return os, nil
+}
+
+func (p *Peer) setClient(s string, key []byte, i Client, priority uint8, live bool) error {
+	p.clientMu.Lock()
+	defer p.clientMu.Unlock()
+
+	sk := s + keyToString(key)
+	if p.clients[sk] != nil {
+		return fmt.Errorf("client %v already registered", sk)
+	}
+	next := make(chan error, 1)
+	// var intervals *Intervals
+	// if !live {
+	// key := s + p.ID().String()
+	// intervals = NewIntervals(key, p.streamer)
+	// }
+	p.clients[sk] = &client{
+		Client: i,
+		// intervals:        intervals,
+		live:     live,
+		priority: priority,
+		next:     next,
+		stream:   s,
+		key:      key,
+	}
+	next <- nil // this is to allow wantedKeysMsg before first batch arrives
+	return nil
+}

swarm/network/stream/stream.go

@@ -0,0 +1,406 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stream
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/protocols"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const (
+	Low uint8 = iota
+	Mid
+	High
+	Top
+	PriorityQueue         // number of queues
+	PriorityQueueCap = 32 // queue capacity
+	HashSize         = 32
+)
+
+// Registry registry for outgoing and incoming streamer constructors
+type Registry struct {
+	api         *API
+	addr        *network.BzzAddr
+	skipCheck   bool
+	clientMu    sync.RWMutex
+	serverMu    sync.RWMutex
+	peersMu     sync.RWMutex
+	serverFuncs map[string]func(*Peer, []byte) (Server, error)
+	clientFuncs map[string]func(*Peer, []byte) (Client, error)
+	peers       map[discover.NodeID]*Peer
+	delivery    *Delivery
+	store       storage.ChunkStore
+}
+
+// NewRegistry is Streamer constructor
+func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkStore, skipCheck bool) *Registry {
+	streamer := &Registry{
+		addr:        addr,
+		skipCheck:   skipCheck,
+		store:       store,
+		serverFuncs: make(map[string]func(*Peer, []byte) (Server, error)),
+		clientFuncs: make(map[string]func(*Peer, []byte) (Client, error)),
+		peers:       make(map[discover.NodeID]*Peer),
+		delivery:    delivery,
+	}
+	streamer.api = NewAPI(streamer, streamer.store)
+	delivery.getPeer = streamer.getPeer
+	streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, t []byte) (Server, error) {
+		return NewSwarmChunkServer(delivery.db), nil
+	})
+	streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t []byte) (Client, error) {
+		return NewSwarmSyncerClient(p, delivery.db, nil)
+	})
+	return streamer
+}
+
+// RegisterClient registers an incoming streamer constructor
+func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, []byte) (Client, error)) {
+	r.clientMu.Lock()
+	defer r.clientMu.Unlock()
+
+	r.clientFuncs[stream] = f
+}
+
+// RegisterServer registers an outgoing streamer constructor
+func (r *Registry) RegisterServerFunc(stream string, f func(*Peer, []byte) (Server, error)) {
+	r.serverMu.Lock()
+	defer r.serverMu.Unlock()
+
+	r.serverFuncs[stream] = f
+}
+
+// GetClient accessor for incoming streamer constructors
+func (r *Registry) GetClientFunc(stream string) (func(*Peer, []byte) (Client, error), error) {
+	r.clientMu.RLock()
+	defer r.clientMu.RUnlock()
+
+	f := r.clientFuncs[stream]
+	if f == nil {
+		return nil, fmt.Errorf("stream %v not registered", stream)
+	}
+	return f, nil
+}
+
+// GetServer accessor for incoming streamer constructors
+func (r *Registry) GetServerFunc(stream string) (func(*Peer, []byte) (Server, error), error) {
+	r.serverMu.RLock()
+	defer r.serverMu.RUnlock()
+
+	f := r.serverFuncs[stream]
+	if f == nil {
+		return nil, fmt.Errorf("stream %v not registered", stream)
+	}
+	return f, nil
+}
+
+// Subscribe initiates the streamer
+func (r *Registry) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error {
+	f, err := r.GetClientFunc(s)
+	if err != nil {
+		return err
+	}
+
+	peer := r.getPeer(peerId)
+	if peer == nil {
+		return fmt.Errorf("peer not found %v", peerId)
+	}
+
+	is, err := f(peer, t)
+	if err != nil {
+		return err
+	}
+	err = peer.setClient(s, t, is, priority, live)
+	if err != nil {
+		return err
+	}
+
+	msg := &SubscribeMsg{
+		Stream: s,
+		Key:    t,
+		// Live:     live,
+		From:     from,
+		To:       to,
+		Priority: priority,
+	}
+	log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to)
+
+	peer.SendPriority(msg, priority)
+	return nil
+}
+
+func (r *Registry) Retrieve(chunk *storage.Chunk) error {
+	return r.delivery.RequestFromPeers(chunk.Key[:], r.skipCheck)
+}
+
+func (r *Registry) NodeInfo() interface{} {
+	return nil
+}
+
+func (r *Registry) PeerInfo(id discover.NodeID) interface{} {
+	return nil
+}
+
+func (r *Registry) getPeer(peerId discover.NodeID) *Peer {
+	r.peersMu.RLock()
+	defer r.peersMu.RUnlock()
+
+	return r.peers[peerId]
+}
+
+func (r *Registry) setPeer(peer *Peer) {
+	r.peersMu.Lock()
+	r.peers[peer.ID()] = peer
+	r.peersMu.Unlock()
+}
+
+func (r *Registry) deletePeer(peer *Peer) {
+	r.peersMu.Lock()
+	delete(r.peers, peer.ID())
+	r.peersMu.Unlock()
+}
+
+func (r *Registry) peersCount() (c int) {
+	r.peersMu.Lock()
+	c = len(r.peers)
+	r.peersMu.Unlock()
+	return
+}
+
+// Run protocol run function
+func (r *Registry) run(p *protocols.Peer) error {
+	sp := NewPeer(p, r)
+	r.setPeer(sp)
+	defer r.deletePeer(sp)
+	defer close(sp.quit)
+	return sp.Run(sp.HandleMsg)
+}
+
+func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error {
+	peer := protocols.NewPeer(p, rw, Spec)
+	bzzPeer := network.NewBzzTestPeer(peer, r.addr)
+	r.delivery.overlay.On(bzzPeer)
+	defer r.delivery.overlay.Off(bzzPeer)
+	return r.run(peer)
+}
+
+// HandleMsg is the message handler that delegates incoming messages
+func (p *Peer) HandleMsg(msg interface{}) error {
+	switch msg := msg.(type) {
+
+	case *SubscribeMsg:
+		return p.handleSubscribeMsg(msg)
+
+	case *OfferedHashesMsg:
+		return p.handleOfferedHashesMsg(msg)
+
+	case *TakeoverProofMsg:
+		return p.handleTakeoverProofMsg(msg)
+
+	case *WantedHashesMsg:
+		return p.handleWantedHashesMsg(msg)
+
+	case *ChunkDeliveryMsg:
+		return p.streamer.delivery.handleChunkDeliveryMsg(msg)
+
+	case *RetrieveRequestMsg:
+		return p.streamer.delivery.handleRetrieveRequestMsg(p, msg)
+
+	default:
+		return fmt.Errorf("unknown message type: %T", msg)
+	}
+}
+
+func keyToString(key []byte) string {
+	l := len(key)
+	if l == 0 {
+		return ""
+	}
+	return fmt.Sprintf("%s-%d", string(key[:l-1]), uint8(key[l-1]))
+}
+
+type server struct {
+	Server
+	priority     uint8
+	currentBatch []byte
+	stream       string
+	key          []byte
+}
+
+// Server interface for outgoing peer Streamer
+type Server interface {
+	SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error)
+	GetData([]byte) []byte
+}
+
+type client struct {
+	Client
+	priority  uint8
+	sessionAt uint64
+	live      bool
+	stream    string
+	key       []byte
+	quit      chan struct{}
+	next      chan error
+}
+
+// Client interface for incoming peer Streamer
+type Client interface {
+	NeedData([]byte) func()
+	BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error)
+}
+
+// nextBatch adjusts the indexes by inspecting the intervals
+func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) {
+	var intervals []uint64
+	if c.live {
+		if len(intervals) == 0 {
+			intervals = []uint64{c.sessionAt, from}
+		} else {
+			intervals[1] = from
+		}
+		nextFrom = from
+	} else if from >= c.sessionAt { // history sync complete
+		intervals = nil
+		nextFrom = from
+		nextTo = math.MaxUint64
+	} else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals
+		intervals = append(intervals[:1], intervals[3:]...)
+		nextFrom = intervals[1]
+		if len(intervals) > 2 {
+			nextTo = intervals[2]
+		} else {
+			nextTo = c.sessionAt
+		}
+	} else {
+		nextFrom = from
+		intervals[1] = from
+		nextTo = c.sessionAt
+	}
+	// b.intervals.set(intervals)
+	return nextFrom, nextTo
+}
+
+func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error {
+	if tf := c.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil {
+		tp, err := tf()
+		if err != nil {
+			return err
+		}
+		return p.SendPriority(tp, c.priority)
+	}
+	return nil
+}
+
+// Spec is the spec of the streamer protocol
+var Spec = &protocols.Spec{
+	Name:       "stream",
+	Version:    1,
+	MaxMsgSize: 10 * 1024 * 1024,
+	Messages: []interface{}{
+		UnsubscribeMsg{},
+		OfferedHashesMsg{},
+		WantedHashesMsg{},
+		TakeoverProofMsg{},
+		SubscribeMsg{},
+		RetrieveRequestMsg{},
+		ChunkDeliveryMsg{},
+	},
+}
+
+func (r *Registry) Protocols() []p2p.Protocol {
+	return []p2p.Protocol{
+		{
+			Name:    Spec.Name,
+			Version: Spec.Version,
+			Length:  Spec.Length(),
+			Run:     r.runProtocol,
+			// NodeInfo: ,
+			// PeerInfo: ,
+		},
+	}
+}
+
+func (r *Registry) APIs() []rpc.API {
+	return []rpc.API{
+		{
+			Namespace: "stream",
+			Version:   "0.1",
+			Service:   r.api,
+			Public:    true,
+		},
+	}
+}
+
+func (r *Registry) Start(server *p2p.Server) error {
+	r.api.dpa.Start()
+	return nil
+}
+
+func (r *Registry) Stop() error {
+	r.api.dpa.Stop()
+	return nil
+}
+
+type API struct {
+	streamer *Registry
+	dpa      *storage.DPA
+}
+
+func NewAPI(r *Registry, store storage.ChunkStore) *API {
+	dpa := storage.NewDPA(store, storage.NewChunkerParams())
+	return &API{
+		streamer: r,
+		dpa:      dpa,
+	}
+}
+
+func readAll(dpa *storage.DPA, hash []byte) (int64, error) {
+	r := dpa.Retrieve(hash)
+	buf := make([]byte, 1024)
+	var n int
+	var total int64
+	var err error
+	for (total == 0 || n > 0) && err == nil {
+		n, err = r.ReadAt(buf, total)
+		total += int64(n)
+	}
+	if err != nil && err != io.EOF {
+		return total, err
+	}
+	return total, nil
+}
+
+func (api *API) ReadAll(hash common.Hash) (int64, error) {
+	return readAll(api.dpa, hash[:])
+}
+
+func (api *API) SubscribeStream(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error {
+	return api.streamer.Subscribe(peerId, s, t, from, to, priority, live)
+}

swarm/network/stream/streamer_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package network
+package stream
 
 import (
 	"bytes"
@@ -50,15 +50,15 @@ var (
 	batchDone                        = make(chan bool)
 )
 
-type testIncomingStreamer struct {
+type testClient struct {
 	t []byte
 }
 
-type testOutgoingStreamer struct {
+type testServer struct {
 	t []byte
 }
 
-func (self *testIncomingStreamer) NeedData(hash []byte) func() {
+func (self *testClient) NeedData(hash []byte) func() {
 	receivedHashes[string(hash)] = hash
 	if bytes.Equal(hash, hash0[:]) {
 		return func() {
@@ -72,16 +72,16 @@ func (self *testIncomingStreamer) NeedData(hash []byte) func() {
 	return nil
 }
 
-func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) {
+func (self *testClient) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) {
 	close(batchDone)
 	return nil
 }
 
-func (self *testOutgoingStreamer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
+func (self *testServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
 	return make([]byte, HashSize), from + 1, to + 1, nil, nil
 }
 
-func (self *testOutgoingStreamer) GetData([]byte) []byte {
+func (self *testServer) GetData([]byte) []byte {
 	return nil
 }
 
@@ -92,8 +92,8 @@ func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
+	streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) {
-		return &testIncomingStreamer{
+		return &testClient{
 			t: t,
 		}, nil
 	})
@@ -134,8 +134,8 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) {
+	streamer.RegisterServerFunc("foo", func(p *Peer, t []byte) (Server, error) {
-		return &testOutgoingStreamer{
+		return &testServer{
 			t: t,
 		}, nil
 	})
@@ -188,8 +188,8 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
+	streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) {
-		return &testIncomingStreamer{
+		return &testClient{
 			t: t,
 		}, nil
 	})

swarm/network/stream/syncer.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package network
+package stream
 
 import (
 	"bytes"
@@ -29,88 +29,52 @@ import (
 )
 
 const (
-	batchSize = 2
+	// BatchSize = 2
-	// batchSize = 128
+	BatchSize = 128
 )
 
-// wrapper of db-s to provide mockable custom local chunk store access to syncer
+// SwarmSyncerServer implements an OutgoingStreamer for history syncing on bins
-type DbAccess struct {
-	db  *storage.DbStore
-	loc *storage.LocalStore
-}
-
-func NewDbAccess(loc *storage.LocalStore) *DbAccess {
-	return &DbAccess{loc.DbStore.(*storage.DbStore), loc}
-}
-
-// to obtain the chunks from key or request db entry only
-func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) {
-	return self.loc.Get(key)
-}
-
-// current storage counter of chunk db
-func (self *DbAccess) currentBucketStorageIndex(po uint8) uint64 {
-	return self.db.CurrentBucketStorageIndex(po)
-}
-
-// iteration storage counter and proximity order
-func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage.Key, uint64) bool) error {
-	return self.db.SyncIterator(from, to, po, f)
-}
-
-// to obtain the chunks from key or request db entry only
-func (self *DbAccess) getOrCreateRequest(key storage.Key) (*storage.Chunk, bool) {
-	return self.loc.GetOrCreateRequest(key)
-}
-
-// to obtain the chunks from key or request db entry only
-func (self *DbAccess) put(chunk *storage.Chunk) {
-	self.loc.Put(chunk)
-}
-
-// OutgoingSwarmSyncer implements an OutgoingStreamer for history syncing on bins
 // offered streams:
 // * live request delivery with or without checkback
 // * (live/non-live historical) chunk syncing per proximity bin
-type OutgoingSwarmSyncer struct {
+type SwarmSyncerServer struct {
 	po        uint8
-	db        *DbAccess
+	db        *storage.DBAPI
 	sessionAt uint64
 	start     uint64
 }
 
-// NewOutgoingSwarmSyncer is contructor for OutgoingSwarmSyncer
+// NewSwarmSyncerServer is contructor for SwarmSyncerServer
-func NewOutgoingSwarmSyncer(live bool, po uint8, db *DbAccess) (*OutgoingSwarmSyncer, error) {
+func NewSwarmSyncerServer(live bool, po uint8, db *storage.DBAPI) (*SwarmSyncerServer, error) {
-	sessionAt := db.currentBucketStorageIndex(po)
+	sessionAt := db.CurrentBucketStorageIndex(po)
 	var start uint64
 	if live {
 		start = sessionAt
 	}
-	self := &OutgoingSwarmSyncer{
+	return &SwarmSyncerServer{
 		po:        po,
 		db:        db,
 		sessionAt: sessionAt,
 		start:     start,
-	}
+	}, nil
-	return self, nil
 }
 
 const maxPO = 32
 
-func RegisterOutgoingSyncer(streamer *Streamer, db *DbAccess) {
+func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) {
-	streamer.RegisterOutgoingStreamer("SYNC", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) {
+	streamer.RegisterServerFunc("SYNC", func(p *Peer, t []byte) (Server, error) {
 		po := uint8(t[0])
 		// TODO: make this work for HISTORY too
-		return NewOutgoingSwarmSyncer(false, po, db)
+		return NewSwarmSyncerServer(false, po, db)
 	})
-	// streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) {
+	// streamer.RegisterOutgoingStreamer(stream, func(p *Peer) (OutgoingStreamer, error) {
 	// 	return NewOutgoingProvableSwarmSyncer(po, db)
 	// })
 }
 
 // GetSection retrieves the actual chunk from localstore
-func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte {
+func (s *SwarmSyncerServer) GetData(key []byte) []byte {
-	chunk, err := self.db.get(storage.Key(key))
+	chunk, err := s.db.Get(storage.Key(key))
 	if err != nil {
 		return nil
 	}
@@ -118,23 +82,23 @@ func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte {
 }
 
 // GetBatch retrieves the next batch of hashes from the dbstore
-func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
+func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
 	var batch []byte
 	i := 0
 	if from == 0 {
-		from = self.start
+		from = s.start
 	}
-	if to <= from || from >= self.sessionAt {
+	if to <= from || from >= s.sessionAt {
 		to = math.MaxUint64
 	}
 	ticker := time.NewTicker(10 * time.Millisecond)
 	defer ticker.Stop()
 	for range ticker.C {
-		err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool {
+		err := s.db.Iterator(from, to, s.po, func(key storage.Key, idx uint64) bool {
 			batch = append(batch, key[:]...)
 			i++
 			to = idx
-			return i < batchSize
+			return i < BatchSize
 		})
 		if err != nil {
 			return nil, 0, 0, nil, err
@@ -144,41 +108,40 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64,
 		}
 	}
 
-	log.Debug("Swarm syncer offer batch", "po", self.po, "len", i, "from", from, "to", to, "current store count", self.db.currentBucketStorageIndex(self.po))
+	log.Debug("Swarm syncer offer batch", "po", s.po, "len", i, "from", from, "to", to, "current store count", s.db.CurrentBucketStorageIndex(s.po))
 	return batch, from, to + 1, nil, nil
 }
 
-// IncomingSwarmSyncer
+// SwarmSyncerClient
-type IncomingSwarmSyncer struct {
+type SwarmSyncerClient struct {
 	sessionAt     uint64
 	nextC         chan struct{}
 	sessionRoot   storage.Key
 	sessionReader storage.LazySectionReader
 	retrieveC     chan *storage.Chunk
 	storeC        chan *storage.Chunk
-	dbAccess      *DbAccess
+	db            *storage.DBAPI
 	chunker       storage.Chunker
 	currentRoot   storage.Key
 	requestFunc   func(chunk *storage.Chunk)
 	end, start    uint64
 }
 
-// NewIncomingSwarmSyncer is a contructor for provable data exchange syncer
+// NewSwarmSyncerClient is a contructor for provable data exchange syncer
-func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) {
+func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI, chunker storage.Chunker) (*SwarmSyncerClient, error) {
-	self := &IncomingSwarmSyncer{
+	return &SwarmSyncerClient{
-		dbAccess: dbAccess,
+		db:      db,
 		chunker: chunker,
-	}
+	}, nil
-	return self, nil
 }
 
 // // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer
-// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *IncomingSwarmSyncer {
+// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *SwarmSyncerClient {
 // 	retrieveC := make(storage.Chunk, chunksCap)
 // 	RunChunkRequestor(p, retrieveC)
 // 	storeC := make(storage.Chunk, chunksCap)
 // 	RunChunkStorer(store, storeC)
-// 	self := &IncomingSwarmSyncer{
+// 	s := &SwarmSyncerClient{
 // 		po:            po,
 // 		priority:      priority,
 // 		sessionAt:     sessionAt,
@@ -191,10 +154,10 @@ func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker)
 // 		retrieveC:     retrieveC,
 // 		storeC:        storeC,
 // 	}
-// 	return self
+// 	return s
 // }
 
-// // StartSyncing is called on the StreamerPeer to start the syncing process
+// // StartSyncing is called on the Peer to start the syncing process
 // // the idea is that it is called only after kademlia is close to healthy
 // func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) {
 // 	lastPO := po
@@ -208,47 +171,53 @@ func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker)
 // 	}
 // }
 
-func RegisterIncomingSyncer(streamer *Streamer, db *DbAccess) {
+// RegisterSwarmSyncerClient registers the client constructor function for
-	streamer.RegisterIncomingStreamer("SYNC", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
+// to handle incoming sync streams
-		return NewIncomingSwarmSyncer(p, db, nil)
+func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) {
+	streamer.RegisterClientFunc("SYNC", func(p *Peer, t []byte) (Client, error) {
+		return NewSwarmSyncerClient(p, db, nil)
 	})
 }
 
 // NeedData
-func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) {
+func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) {
-	chunk, _ := self.dbAccess.getOrCreateRequest(key)
+	chunk, _ := s.db.GetOrCreateRequest(key)
+	log.Warn("created request", "key", chunk.Key)
 	// TODO: we may want to request from this peer anyway even if the request exists
 	if chunk.ReqC == nil {
 		return nil
 	}
 	// create request and wait until the chunk data arrives and is stored
-	return chunk.WaitToStore
+	return func() {
+		chunk.WaitToStore()
+		log.Warn("stored", "key", chunk.Key)
+	}
 }
 
 // BatchDone
-func (self *IncomingSwarmSyncer) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) {
+func (s *SwarmSyncerClient) BatchDone(streamName string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) {
-	if self.chunker != nil {
+	if s.chunker != nil {
-		return func() (*TakeoverProof, error) { return self.TakeoverProof(s, from, hashes, root) }
+		return func() (*TakeoverProof, error) { return s.TakeoverProof(streamName, from, hashes, root) }
 	}
 	return nil
 }
 
-func (self *IncomingSwarmSyncer) TakeoverProof(s string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) {
+func (s *SwarmSyncerClient) TakeoverProof(streamName string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) {
 	// for provable syncer currentRoot is non-zero length
-	if self.chunker != nil {
+	if s.chunker != nil {
-		if from > self.sessionAt { // for live syncing currentRoot is always updated
+		if from > s.sessionAt { // for live syncing currentRoot is always updated
-			//expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC, self.storeC)
+			//expRoot, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC, s.storeC)
-			expRoot, _, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC)
+			expRoot, _, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC)
 			if err != nil {
 				return nil, err
 			}
 			if !bytes.Equal(root, expRoot) {
 				return nil, fmt.Errorf("HandoverProof mismatch")
 			}
-			self.currentRoot = root
+			s.currentRoot = root
 		} else {
 			expHashes := make([]byte, len(hashes))
-			_, err := self.sessionReader.ReadAt(expHashes, int64(self.end*HashSize))
+			_, err := s.sessionReader.ReadAt(expHashes, int64(s.end*HashSize))
 			if err != nil && err != io.EOF {
 				return nil, err
 			}
@@ -258,12 +227,12 @@ func (self *IncomingSwarmSyncer) TakeoverProof(s string, from uint64, hashes []b
 		}
 		return nil, nil
 	}
-	self.end += uint64(len(hashes)) / HashSize
+	s.end += uint64(len(hashes)) / HashSize
 	takeover := &Takeover{
-		Stream: s,
+		Stream: streamName,
-		// Key:    self.Key,
+		// Key:    s.Key,
-		Start: self.start,
+		Start: s.start,
-		End:   self.end,
+		End:   s.end,
 		Root:  root,
 	}
 	// serialise and sign

swarm/network/stream/syncer_test.go

@@ -0,0 +1,173 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stream
+
+import (
+	"context"
+	crand "crypto/rand"
+	"fmt"
+	"io"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const dataChunkCount = 500
+
+func TestSyncerSimulation(t *testing.T) {
+	testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1)
+	// testSyncBetweenNodes(t, 2, 1, dataChunkCount, false, 1)
+	testSyncBetweenNodes(t, 4, 1, dataChunkCount, true, 1)
+	// testSyncBetweenNodes(t, 4, 1, dataChunkCount, false, 1)
+	testSyncBetweenNodes(t, 8, 1, dataChunkCount, true, 1)
+	// testSyncBetweenNodes(t, 8, 1, dataChunkCount, false, 1)
+	testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1)
+	// testSyncBetweenNodes(t, 16, 1, dataChunkCount, false, 1)
+}
+
+func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) {
+	defaultSkipCheck = skipCheck
+	toAddr = func(id discover.NodeID) *network.BzzAddr {
+		addr := network.NewAddrFromNodeID(id)
+		addr.OAddr[0] = byte(0)
+		return addr
+	}
+	conf := &streamTesting.RunConfig{
+		Adapter:   *adapter,
+		NodeCount: nodes,
+		ConnLevel: conns,
+		ToAddr:    toAddr,
+		Services:  services,
+	}
+
+	sim, teardown, err := streamTesting.NewSimulation(conf)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	stores = make(map[discover.NodeID]storage.ChunkStore)
+	deliveries = make(map[discover.NodeID]*Delivery)
+	for i, id := range sim.IDs {
+		stores[id] = sim.Stores[i]
+	}
+	peerCount = func(id discover.NodeID) int {
+		if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
+			return 1
+		}
+		return 2
+	}
+	// here we distribute chunks of a random file into Stores of nodes 1 to nodes
+	rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams())
+	rrdpa.Start()
+	size := chunkCount * chunkSize
+	_, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size))
+	// need to wait cos we then immediately collect the relevant bin content
+	wait()
+	defer rrdpa.Stop()
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// collect hashes in po 1 from all nodes
+	var hashes []storage.Key
+	dbs := make([]*storage.DBAPI, nodes)
+	for i := 0; i < nodes; i++ {
+		dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore))
+	}
+	for i := 1; i < nodes; i++ {
+		dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool {
+			hashes = append(hashes, key)
+			return true
+		})
+	}
+
+	waitPeerErrC = make(chan error)
+	action := func(ctx context.Context) error {
+		// need to wait till an aynchronous process registers the peers in streamer.peers
+		// that is used by Subscribe
+		// the global peerCount function tells how many connections each node has
+		// TODO: this is to be reimplemented with peerEvent watcher without global var
+		i := 0
+		for err := range waitPeerErrC {
+			if err != nil {
+				return fmt.Errorf("error waiting for peers: %s", err)
+			}
+			i++
+			if i == nodes {
+				break
+			}
+		}
+		// each node Subscribes to each other's swarmChunkServerStreamName
+		j := 0
+		return sim.CallClient(func(client *rpc.Client) error {
+			ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
+			defer cancel()
+			j++
+			return client.CallContext(ctx, nil, "stream_subscribeStream", sim.IDs[j], "SYNC", []byte{1}, 0, 0, Top, false)
+		}, sim.IDs[0:nodes-1]...)
+	}
+
+	// this makes sure check is not called before the previous call finishes
+	checkC := make(chan struct{})
+	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
+		defer func() { checkC <- struct{}{} }()
+
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		default:
+		}
+
+		var found int
+		total := len(hashes)
+		for _, key := range hashes {
+			_, err := dbs[0].Get(key)
+			if err == nil {
+				found++
+			}
+		}
+		log.Debug("sync check", "bin", po, "found", found, "total", total)
+		return found == total, nil
+	}
+
+	conf.Step = &simulations.Step{
+		Action:  action,
+		Trigger: streamTesting.PivotTrigger(10*time.Millisecond, checkC, sim.IDs[0]),
+		Expect: &simulations.Expectation{
+			Nodes: sim.IDs[0:1],
+			Check: check,
+		},
+	}
+	startedAt := time.Now()
+	result, err := sim.Run(conf)
+	finishedAt := time.Now()
+	if err != nil {
+		t.Fatalf("Setting up simulation failed: %v", err)
+	}
+	if result.Error != nil {
+		t.Fatalf("Simulation failed: %s", result.Error)
+	}
+	streamTesting.CheckResult(t, result, startedAt, finishedAt)
+}

swarm/network/stream/testing/testing.go

@@ -0,0 +1,268 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package testing
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+type Simulation struct {
+	Net    *simulations.Network
+	Stores []storage.ChunkStore
+	Addrs  []network.Addr
+	IDs    []discover.NodeID
+}
+
+func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) {
+	var datadirs []string
+	stores := make([]storage.ChunkStore, len(addrs))
+	var err error
+	for i, addr := range addrs {
+		var datadir string
+		datadir, err = ioutil.TempDir("", "streamer")
+		if err != nil {
+			break
+		}
+		var store storage.ChunkStore
+		store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
+		if err != nil {
+			break
+		}
+		datadirs = append(datadirs, datadir)
+		stores[i] = store
+	}
+	teardown := func() {
+		for i, datadir := range datadirs {
+			stores[i].Close()
+			os.RemoveAll(datadir)
+		}
+	}
+	return stores, teardown, err
+}
+
+func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) {
+	teardown = func() {}
+	switch adapterType {
+	case "sim":
+		adapter = adapters.NewSimAdapter(services)
+	case "socket":
+		adapter = adapters.NewSocketAdapter(services)
+	case "exec":
+		baseDir, err0 := ioutil.TempDir("", "swarm-test")
+		if err0 != nil {
+			return nil, teardown, err0
+		}
+		teardown = func() { os.RemoveAll(baseDir) }
+		adapter = adapters.NewExecAdapter(baseDir)
+	case "docker":
+		adapter, err = adapters.NewDockerAdapter()
+		if err != nil {
+			return nil, teardown, err
+		}
+	default:
+		return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker")
+	}
+	return adapter, teardown, nil
+}
+
+func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finishedAt time.Time) {
+	t.Logf("Simulation passed in %s", result.FinishedAt.Sub(result.StartedAt))
+	if len(result.Passes) > 1 {
+		var min, max time.Duration
+		var sum int
+		for _, pass := range result.Passes {
+			duration := pass.Sub(result.StartedAt)
+			if sum == 0 || duration < min {
+				min = duration
+			}
+			if duration > max {
+				max = duration
+			}
+			sum += int(duration.Nanoseconds())
+		}
+		t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond)
+	}
+	t.Logf("Setup: %s, Shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt))
+}
+
+type RunConfig struct {
+	Adapter   string
+	Step      *simulations.Step
+	NodeCount int
+	ConnLevel int
+	ToAddr    func(discover.NodeID) *network.BzzAddr
+	Services  adapters.Services
+}
+
+func NewSimulation(conf *RunConfig) (*Simulation, func(), error) {
+	// create network
+	nodes := conf.NodeCount
+	adapter, adapterTeardown, err := NewAdapter(conf.Adapter, conf.Services)
+	if err != nil {
+		return nil, adapterTeardown, err
+	}
+	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
+		ID:             "0",
+		DefaultService: "streamer",
+	})
+	teardown := func() {
+		adapterTeardown()
+		net.Shutdown()
+	}
+	ids := make([]discover.NodeID, nodes)
+	addrs := make([]network.Addr, nodes)
+	// start nodes
+	for i := 0; i < nodes; i++ {
+		node, err := net.NewNode()
+		if err != nil {
+			return nil, teardown, fmt.Errorf("error creating node: %s", err)
+		}
+		ids[i] = node.ID()
+		addrs[i] = conf.ToAddr(ids[i])
+	}
+	// set nodes number of Stores available
+	stores, storeTeardown, err := SetStores(addrs...)
+	teardown = func() {
+		storeTeardown()
+		adapterTeardown()
+		net.Shutdown()
+	}
+	if err != nil {
+		return nil, teardown, err
+	}
+	s := &Simulation{
+		Net:    net,
+		Stores: stores,
+		IDs:    ids,
+		Addrs:  addrs,
+	}
+	return s, teardown, nil
+}
+
+func (s *Simulation) Run(conf *RunConfig) (*simulations.StepResult, error) {
+	// bring up nodes, launch the servive
+	nodes := conf.NodeCount
+	conns := conf.ConnLevel
+	for i := 0; i < nodes; i++ {
+		if err := s.Net.Start(s.IDs[i]); err != nil {
+			return nil, fmt.Errorf("error starting node %s: %s", s.IDs[i].TerminalString(), err)
+		}
+	}
+	// run a simulation which connects the 10 nodes in a chain
+	wg := sync.WaitGroup{}
+	for i := range s.IDs {
+		// collect the overlay addresses, to
+		for j := 0; j < conns; j++ {
+			var k int
+			if j == 0 {
+				k = i - 1
+			} else {
+				k = rand.Intn(len(s.IDs))
+			}
+			if i > 0 {
+				wg.Add(1)
+				go func(i, k int) {
+					defer wg.Done()
+					s.Net.Connect(s.IDs[i], s.IDs[k])
+				}(i, k)
+			}
+		}
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("simulation with %v nodes", len(s.Addrs)))
+
+	// create an only locally retrieving dpa for the pivot node to test
+	// if retriee requests have arrived
+	timeout := 300 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step)
+	return result, nil
+}
+
+func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) error {
+	events := make(chan *p2p.PeerEvent)
+	sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
+	if err != nil {
+		return fmt.Errorf("error getting peer events for node %v: %s", id, err)
+	}
+	go func() {
+		defer sub.Unsubscribe()
+		select {
+		case <-quitC:
+			return
+		case e := <-events:
+			errc <- fmt.Errorf("peerEvent for node %v: %v", id, e)
+		case err := <-sub.Err():
+			if err != nil {
+				errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err)
+			}
+		}
+	}()
+	return nil
+}
+
+func PivotTrigger(d time.Duration, checkC chan struct{}, ids ...discover.NodeID) chan discover.NodeID {
+	trigger := make(chan discover.NodeID)
+	go func() {
+		ticker := time.NewTicker(d)
+		defer ticker.Stop()
+		// we are only testing the pivot node (net.Nodes[0])
+		for range ticker.C {
+			for _, id := range ids {
+				trigger <- id
+			}
+			<-checkC
+		}
+	}()
+	return trigger
+}
+
+func (sim *Simulation) CallClient(f func(*rpc.Client) error, ids ...discover.NodeID) error {
+	for _, id := range ids {
+		node := sim.Net.GetNode(id)
+		if node == nil {
+			return fmt.Errorf("unknown node: %s", id)
+		}
+		client, err := node.Client()
+		if err != nil {
+			return fmt.Errorf("error getting node client: %s", err)
+		}
+		err = f(client)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

swarm/network/streamer.go

@@ -1,630 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"math"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/protocols"
-	bv "github.com/ethereum/go-ethereum/swarm/network/bitvector"
-	pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-const (
-	Low uint8 = iota
-	Mid
-	High
-	Top
-	PriorityQueue        // number of queues
-	PriorityQueueCap = 3 // queue capacity
-	HashSize         = 32
-)
-
-// Handover represents a statement that the upstream peer hands over the stream section
-type Handover struct {
-	Stream     string // name of stream
-	Start, End uint64 // index of hashes
-	Root       []byte // Root hash for indexed segment inclusion proofs
-}
-
-// HandoverProof represents a signed statement that the upstream peer handed over the stream section
-type HandoverProof struct {
-	Sig []byte // Sign(Hash(Serialisation(Handover)))
-	*Handover
-}
-
-// Takeover represents a statement that downstream peer took over (stored all data)
-// handed over
-type Takeover Handover
-
-//  TakeoverProof represents a signed statement that the downstream peer took over
-// the stream section
-type TakeoverProof struct {
-	Sig []byte // Sign(Hash(Serialisation(Takeover)))
-	*Takeover
-}
-
-// TakeoverProofMsg is the protocol msg sent by downstream peer
-type TakeoverProofMsg TakeoverProof
-
-// String pretty prints TakeoverProofMsg
-func (self TakeoverProofMsg) String() string {
-	return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", self.Stream, self.Start, self.End, self.Root, self.Sig)
-}
-
-// SubcribeMsg is the protocol msg for requesting a stream(section)
-type SubscribeMsg struct {
-	Stream   string
-	Key      []byte
-	From, To uint64
-	Priority uint8 // delivered on priority channel
-}
-
-// OfferedHashesMsg is the protocol msg for offering to hand over a
-// stream section
-type OfferedHashesMsg struct {
-	Stream         string // name of Stream
-	Key            []byte // subtype or key
-	From, To       uint64 // peer and db-specific entry count
-	Hashes         []byte // stream of hashes (128)
-	*HandoverProof        // HandoverProof
-}
-
-// String pretty prints OfferedHashesMsg
-func (self OfferedHashesMsg) String() string {
-	return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", self.Stream, self.From, self.To, len(self.Hashes)/HashSize)
-}
-
-// WantedHashesMsg is the protocol msg data for signaling which hashes
-// offered in OfferedHashesMsg downstream peer actually wants sent over
-type WantedHashesMsg struct {
-	Stream   string // name of stream
-	Key      []byte // subtype or key
-	Want     []byte // bitvector indicating which keys of the batch needed
-	From, To uint64 // next interval offset - empty if not to be continued
-}
-
-// String pretty prints WantedHashesMsg
-func (self WantedHashesMsg) String() string {
-	return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", self.Stream, self.Want, self.From, self.To)
-}
-
-func keyToString(key []byte) string {
-	l := len(key)
-	if l == 0 {
-		return ""
-	}
-	return fmt.Sprintf("%s-%d", string(key[:l-1]), uint8(key[l-1]))
-}
-
-// Streamer registry for outgoing and incoming streamer constructors
-type Streamer struct {
-	incomingLock sync.RWMutex
-	outgoingLock sync.RWMutex
-	peersLock    sync.RWMutex
-	outgoing     map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error)
-	incoming     map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error)
-	peers        map[discover.NodeID]*StreamerPeer
-	delivery     *Delivery
-}
-
-// NewStreamer is Streamer constructor
-func NewStreamer(delivery *Delivery) *Streamer {
-	streamer := &Streamer{
-		outgoing: make(map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error)),
-		incoming: make(map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error)),
-		peers:    make(map[discover.NodeID]*StreamerPeer),
-		delivery: delivery,
-	}
-	delivery.getPeer = streamer.getPeer
-	streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) {
-		return NewRetrieveRequestStreamer(delivery.dbAccess), nil
-	})
-	streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) {
-		return NewIncomingSwarmSyncer(p, delivery.dbAccess, nil)
-	})
-	return streamer
-}
-
-func (self *Streamer) Retrieve(chunk *storage.Chunk) error {
-	return self.delivery.RequestFromPeers(chunk.Key[:], false)
-}
-
-// RegisterIncomingStreamer registers an incoming streamer constructor
-func (self *Streamer) RegisterIncomingStreamer(stream string, f func(*StreamerPeer, []byte) (IncomingStreamer, error)) {
-	self.incomingLock.Lock()
-	defer self.incomingLock.Unlock()
-	self.incoming[stream] = f
-}
-
-// RegisterOutgoingStreamer registers an outgoing streamer constructor
-func (self *Streamer) RegisterOutgoingStreamer(stream string, f func(*StreamerPeer, []byte) (OutgoingStreamer, error)) {
-	self.outgoingLock.Lock()
-	defer self.outgoingLock.Unlock()
-	self.outgoing[stream] = f
-}
-
-// GetIncomingStreamer accessor for incoming streamer constructors
-func (self *Streamer) GetIncomingStreamer(stream string) (func(*StreamerPeer, []byte) (IncomingStreamer, error), error) {
-	self.incomingLock.RLock()
-	defer self.incomingLock.RUnlock()
-	f := self.incoming[stream]
-	if f == nil {
-		return nil, fmt.Errorf("stream %v not registered", stream)
-	}
-	return f, nil
-}
-
-// GetOutgoingStreamer accessor for incoming streamer constructors
-func (self *Streamer) GetOutgoingStreamer(stream string) (func(*StreamerPeer, []byte) (OutgoingStreamer, error), error) {
-	self.outgoingLock.RLock()
-	defer self.outgoingLock.RUnlock()
-	f := self.outgoing[stream]
-	if f == nil {
-		return nil, fmt.Errorf("stream %v not registered", stream)
-	}
-	return f, nil
-}
-
-func (self *Streamer) NodeInfo() interface{} {
-	return nil
-}
-
-func (self *Streamer) PeerInfo(id discover.NodeID) interface{} {
-	return nil
-}
-
-type outgoingStreamer struct {
-	OutgoingStreamer
-	priority     uint8
-	currentBatch []byte
-	stream       string
-	key          []byte
-}
-
-// OutgoingStreamer interface for outgoing peer Streamer
-type OutgoingStreamer interface {
-	SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error)
-	GetData([]byte) []byte
-}
-
-type incomingStreamer struct {
-	IncomingStreamer
-	priority  uint8
-	sessionAt uint64
-	live      bool
-	stream    string
-	key       []byte
-	quit      chan struct{}
-	next      chan struct{}
-}
-
-// IncomingStreamer interface for incoming peer Streamer
-type IncomingStreamer interface {
-	NeedData([]byte) func()
-	BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error)
-}
-
-// StreamerPeer is the Peer extention for the streaming protocol
-type StreamerPeer struct {
-	Peer
-	streamer *Streamer
-	pq       *pq.PriorityQueue
-	//netStore     storage.ChunkStore
-	outgoingLock sync.RWMutex
-	incomingLock sync.RWMutex
-	outgoing     map[string]*outgoingStreamer
-	incoming     map[string]*incomingStreamer
-	quit         chan struct{}
-}
-
-// NewStreamerPeer is the constructor for StreamerPeer
-func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer {
-	self := &StreamerPeer{
-		Peer:     p,
-		pq:       pq.New(int(PriorityQueue), PriorityQueueCap),
-		streamer: streamer,
-		outgoing: make(map[string]*outgoingStreamer),
-		incoming: make(map[string]*incomingStreamer),
-		quit:     make(chan struct{}),
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	go self.pq.Run(ctx, func(i interface{}) { p.Send(i) })
-	go func() {
-		<-self.quit
-		cancel()
-	}()
-	return self
-}
-
-func (self *Streamer) getPeer(peerId discover.NodeID) *StreamerPeer {
-	self.peersLock.RLock()
-	defer self.peersLock.RUnlock()
-	return self.peers[peerId]
-}
-
-func (self *Streamer) setPeer(peer *StreamerPeer) {
-	self.peersLock.Lock()
-	self.peers[peer.ID()] = peer
-	self.peersLock.Unlock()
-}
-
-func (self *Streamer) deletePeer(peer *StreamerPeer) {
-	self.peersLock.Lock()
-	delete(self.peers, peer.ID())
-	self.peersLock.Unlock()
-}
-
-func (self *StreamerPeer) getOutgoingStreamer(s string) (*outgoingStreamer, error) {
-	self.outgoingLock.RLock()
-	defer self.outgoingLock.RUnlock()
-	streamer := self.outgoing[s]
-	if streamer == nil {
-		return nil, fmt.Errorf("outgoing stream '%v' not provided to peer %v", s, self.ID())
-	}
-	return streamer, nil
-}
-
-func (self *StreamerPeer) getIncomingStreamer(s string) (*incomingStreamer, error) {
-	self.incomingLock.RLock()
-	defer self.incomingLock.RUnlock()
-	streamer := self.incoming[s]
-	if streamer == nil {
-		return nil, fmt.Errorf("incoming stream '%v' not provided to peer %v", s, self.ID())
-	}
-	return streamer, nil
-}
-
-func (self *StreamerPeer) setOutgoingStreamer(s string, key []byte, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) {
-	self.outgoingLock.Lock()
-	defer self.outgoingLock.Unlock()
-	sk := s + keyToString(key)
-	if self.outgoing[sk] != nil {
-		return nil, fmt.Errorf("stream %v already registered", sk)
-	}
-	os := &outgoingStreamer{
-		OutgoingStreamer: o,
-		priority:         priority,
-		stream:           s,
-		key:              key,
-	}
-	self.outgoing[sk] = os
-	return os, nil
-}
-
-func (self *StreamerPeer) setIncomingStreamer(s string, key []byte, i IncomingStreamer, priority uint8, live bool) error {
-	self.incomingLock.Lock()
-	defer self.incomingLock.Unlock()
-
-	sk := s + keyToString(key)
-	if self.incoming[sk] != nil {
-		return fmt.Errorf("stream %v already registered", sk)
-	}
-	next := make(chan struct{}, 1)
-	// var intervals *Intervals
-	// if !live {
-	// key := s + self.ID().String()
-	// intervals = NewIntervals(key, self.streamer)
-	// }
-	self.incoming[sk] = &incomingStreamer{
-		IncomingStreamer: i,
-		// intervals:        intervals,
-		live:     live,
-		priority: priority,
-		next:     next,
-		stream:   s,
-		key:      key,
-	}
-	next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives
-	return nil
-}
-
-// NextBatch adjusts the indexes by inspecting the intervals
-func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) {
-	var intervals []uint64
-	if self.live {
-		if len(intervals) == 0 {
-			intervals = []uint64{self.sessionAt, from}
-		} else {
-			intervals[1] = from
-		}
-		nextFrom = from
-	} else if from >= self.sessionAt { // history sync complete
-		intervals = nil
-		nextFrom = from
-		nextTo = math.MaxUint64
-	} else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals
-		intervals = append(intervals[:1], intervals[3:]...)
-		nextFrom = intervals[1]
-		if len(intervals) > 2 {
-			nextTo = intervals[2]
-		} else {
-			nextTo = self.sessionAt
-		}
-	} else {
-		nextFrom = from
-		intervals[1] = from
-		nextTo = self.sessionAt
-	}
-	// self.intervals.set(intervals)
-	return nextFrom, nextTo
-}
-
-// Subscribe initiates the streamer
-func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error {
-	f, err := self.GetIncomingStreamer(s)
-	if err != nil {
-		return err
-	}
-
-	peer := self.getPeer(peerId)
-	if peer == nil {
-		return fmt.Errorf("peer not found %v", peerId)
-	}
-
-	is, err := f(peer, t)
-	if err != nil {
-		return err
-	}
-	err = peer.setIncomingStreamer(s, t, is, priority, live)
-	if err != nil {
-		return err
-	}
-
-	msg := &SubscribeMsg{
-		Stream: s,
-		Key:    t,
-		// Live:     live,
-		From:     from,
-		To:       to,
-		Priority: priority,
-	}
-	log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to)
-
-	peer.SendPriority(msg, priority)
-	return nil
-}
-
-func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error {
-	f, err := self.streamer.GetOutgoingStreamer(req.Stream)
-	if err != nil {
-		return err
-	}
-	s, err := f(self, req.Key)
-	if err != nil {
-		return err
-	}
-	os, err := self.setOutgoingStreamer(req.Stream, req.Key, s, req.Priority)
-	if err != nil {
-		return nil
-	}
-	log.Debug("received subscription", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
-	go self.SendOfferedHashes(os, req.From, req.To)
-	return nil
-}
-
-// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface
-// Filter method
-func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error {
-	sk := req.Stream
-	sk += keyToString(req.Key)
-	s, err := self.getIncomingStreamer(sk)
-	if err != nil {
-		return err
-	}
-	hashes := req.Hashes
-	want, err := bv.New(len(hashes) / HashSize)
-	if err != nil {
-		return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err)
-	}
-	wg := sync.WaitGroup{}
-	for i := 0; i < len(hashes); i += HashSize {
-		hash := hashes[i : i+HashSize]
-		if wait := s.NeedData(hash); wait != nil {
-			want.Set(i/HashSize, true)
-			wg.Add(1)
-			// create request and wait until the chunk data arrives and is stored
-			go func(w func()) {
-				w()
-				wg.Done()
-			}(wait)
-		}
-	}
-	go func() {
-		wg.Wait()
-		if tf := s.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil {
-			tp, err := tf()
-			if err != nil {
-				return
-			}
-			self.SendPriority(tp, s.priority)
-		}
-		s.next <- struct{}{}
-	}()
-	// only send wantedKeysMsg if all missing chunks of the previous batch arrived
-	// except
-	if s.live {
-		s.sessionAt = req.From
-	}
-	from, to := s.nextBatch(req.To)
-	log.Debug("received batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
-	if from == to {
-		return nil
-	}
-
-	msg := &WantedHashesMsg{
-		Stream: req.Stream,
-		Key:    req.Key,
-		Want:   want.Bytes(),
-		From:   from,
-		To:     to,
-	}
-	go func() {
-		select {
-		case <-s.next:
-		case <-s.quit:
-			return
-		}
-		log.Debug("want batch", "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To)
-		self.SendPriority(msg, s.priority)
-	}()
-	return nil
-}
-
-// handleWantedHashesMsg protocol msg handler
-// * sends the next batch of unsynced keys
-// * sends the actual data chunks as per WantedHashesMsg
-func (self *StreamerPeer) handleWantedHashesMsg(req *WantedHashesMsg) error {
-	log.Debug("received wanted batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
-	s, err := self.getOutgoingStreamer(req.Stream + keyToString(req.Key))
-	if err != nil {
-		log.Debug(err.Error())
-		return err
-	}
-	hashes := s.currentBatch
-	// launch in go routine since GetBatch blocks until new hashes arrive
-	go self.SendOfferedHashes(s, req.From, req.To)
-	l := len(hashes) / HashSize
-	want, err := bv.NewFromBytes(req.Want, l)
-	if err != nil {
-		return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err)
-	}
-	for i := 0; i < l; i++ {
-		if want.Get(i) {
-			hash := hashes[i*HashSize : (i+1)*HashSize]
-			data := s.GetData(hash)
-			if data == nil {
-				return errors.New("not found")
-			}
-			chunk := storage.NewChunk(hash, nil)
-			chunk.SData = data
-			if err := self.Deliver(chunk, s.priority); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (self *StreamerPeer) handleTakeoverProofMsg(req *TakeoverProofMsg) error {
-	_, err := self.getOutgoingStreamer(req.Stream)
-	if err != nil {
-		return err
-	}
-	// store the strongest takeoverproof for the stream in streamer
-	return nil
-}
-
-// Deliver sends a storeRequestMsg protocol message to the peer
-func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority uint8) error {
-	msg := &ChunkDeliveryMsg{
-		Key:   chunk.Key,
-		SData: chunk.SData,
-	}
-	return self.pq.Push(nil, msg, int(priority))
-}
-
-// Deliver sends a storeRequestMsg protocol message to the peer
-func (self *StreamerPeer) SendPriority(msg interface{}, priority uint8) error {
-	return self.pq.Push(nil, msg, int(priority))
-}
-
-// SendOfferedHashes sends OfferedHashesMsg protocol msg
-func (self *StreamerPeer) SendOfferedHashes(s *outgoingStreamer, f, t uint64) error {
-	hashes, from, to, proof, err := s.SetNextBatch(f, t)
-	if err != nil {
-		return err
-	}
-	if proof == nil {
-		proof = &HandoverProof{
-			Handover: &Handover{},
-		}
-	}
-	s.currentBatch = hashes
-	msg := &OfferedHashesMsg{
-		HandoverProof: proof,
-		Hashes:        hashes,
-		From:          from,
-		To:            to,
-		Stream:        s.stream,
-		Key:           s.key,
-	}
-	log.Debug("Swarm syncer offer batch", "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to)
-	return self.SendPriority(msg, s.priority)
-}
-
-// StreamerSpec is the spec of the streamer protocol.
-var StreamerSpec = &protocols.Spec{
-	Name:       "stream",
-	Version:    1,
-	MaxMsgSize: 10 * 1024 * 1024,
-	Messages: []interface{}{
-		HandshakeMsg{},
-		OfferedHashesMsg{},
-		WantedHashesMsg{},
-		TakeoverProofMsg{},
-		SubscribeMsg{},
-		RetrieveRequestMsg{},
-		ChunkDeliveryMsg{},
-	},
-}
-
-// Run protocol run function
-func (s *Streamer) Run(p *bzzPeer) error {
-	sp := NewStreamerPeer(p, s)
-	// load saved intervals
-
-	s.setPeer(sp)
-
-	defer s.deletePeer(sp)
-	defer close(sp.quit)
-	return sp.Run(sp.HandleMsg)
-}
-
-// HandleMsg is the message handler that delegates incoming messages
-func (self *StreamerPeer) HandleMsg(msg interface{}) error {
-	switch msg := msg.(type) {
-
-	case *SubscribeMsg:
-		return self.handleSubscribeMsg(msg)
-
-	case *OfferedHashesMsg:
-		return self.handleOfferedHashesMsg(msg)
-
-	case *TakeoverProofMsg:
-		return self.handleTakeoverProofMsg(msg)
-
-	case *WantedHashesMsg:
-		return self.handleWantedHashesMsg(msg)
-
-	case *ChunkDeliveryMsg:
-		return self.streamer.delivery.handleChunkDeliveryMsg(msg)
-
-	case *RetrieveRequestMsg:
-		return self.streamer.delivery.handleRetrieveRequestMsg(self, msg)
-
-	default:
-		return fmt.Errorf("unknown message type: %T", msg)
-	}
-}

swarm/network/streamer_common_test.go

@@ -1,349 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/protocols"
-	"github.com/ethereum/go-ethereum/p2p/simulations"
-	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
-	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-var services = adapters.Services{
-	"delivery": newDeliveryService,
-	"syncer":   newSyncerService,
-}
-
-var (
-	adapter  = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker")
-	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
-)
-
-func init() {
-	flag.Parse()
-	// register the Delivery service which will run as a devp2p
-	// protocol when using the exec adapter
-	adapters.RegisterServices(services)
-
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
-}
-
-var (
-	delivery    *Delivery
-	localStores []storage.ChunkStore
-	addrs       []Addr
-	fileHash    storage.Key
-	nodeCount   int
-)
-
-func setLocalStores(addrs ...Addr) (func(), error) {
-	var datadirs []string
-	localStores = make([]storage.ChunkStore, len(addrs))
-	var err error
-	for i, addr := range addrs {
-		// TODO: remove temp datadir after test
-		var datadir string
-		datadir, err = ioutil.TempDir("", "streamer")
-		if err != nil {
-			break
-		}
-		var localStore *storage.LocalStore
-		localStore, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
-		if err != nil {
-			break
-		}
-		datadirs = append(datadirs, datadir)
-		localStores[i] = localStore
-	}
-	teardown := func() {
-		for _, datadir := range datadirs {
-			os.RemoveAll(datadir)
-		}
-	}
-	return teardown, err
-}
-
-func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) {
-	r := dpa.Retrieve(fileHash)
-	buf := make([]byte, 1024)
-	var n, total int
-	var err error
-	for (total == 0 || n > 0) && err == nil {
-		n, err = r.ReadAt(buf, int64(total))
-		total += n
-	}
-	if err != nil && err != io.EOF {
-		return total, err
-	}
-	return total, nil
-}
-
-func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) {
-	var err error
-	var result *simulations.StepResult
-	startedAt := time.Now()
-
-	switch *adapter {
-	case "sim":
-		t.Logf("simadapter")
-		result, err = simf(adapters.NewSimAdapter(services))
-	case "socket":
-		result, err = simf(adapters.NewSocketAdapter(services))
-	case "exec":
-		baseDir, err0 := ioutil.TempDir("", "swarm-test")
-		if err0 != nil {
-			t.Fatal(err0)
-		}
-		defer os.RemoveAll(baseDir)
-		result, err = simf(adapters.NewExecAdapter(baseDir))
-	case "docker":
-		adapter, err0 := adapters.NewDockerAdapter()
-		if err0 != nil {
-			t.Fatal(err0)
-		}
-		result, err = simf(adapter)
-	default:
-		t.Fatal("adapter needs to be one of sim, socket, exec, docker")
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Logf("Simulation with %d nodes passed in %s", len(result.Passes), result.FinishedAt.Sub(result.StartedAt))
-	var min, max time.Duration
-	var sum int
-	for _, pass := range result.Passes {
-		duration := pass.Sub(result.StartedAt)
-		if sum == 0 || duration < min {
-			min = duration
-		}
-		if duration > max {
-			max = duration
-		}
-		sum += int(duration.Nanoseconds())
-	}
-	t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond)
-	finishedAt := time.Now()
-	t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt))
-}
-
-func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.NodeID) *BzzAddr, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) {
-	// create network
-	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
-		ID:             "0",
-		DefaultService: serviceName,
-	})
-	defer net.Shutdown()
-	ids := make([]discover.NodeID, nodes)
-	nodeCount = 0
-	addrs = make([]Addr, nodes)
-	// start nodes
-	for i := 0; i < nodes; i++ {
-		node, err := net.NewNode()
-		if err != nil {
-			return nil, fmt.Errorf("error creating node: %s", err)
-		}
-		ids[i] = node.ID()
-		addrs[i] = toAddr(ids[i])
-	}
-	// set nodes number of localstores globally available
-	teardown, err := setLocalStores(addrs...)
-	defer teardown()
-	if err != nil {
-		return nil, err
-	}
-
-	for i := 0; i < nodes; i++ {
-		if err := net.Start(ids[i]); err != nil {
-			return nil, fmt.Errorf("error starting node %s: %s", ids[i].TerminalString(), err)
-		}
-	}
-
-	// run a simulation which connects the 10 nodes in a chain
-	wg := sync.WaitGroup{}
-	for i := range ids {
-		// collect the overlay addresses, to
-		for j := 0; j < conns; j++ {
-			var k int
-			if j == 0 {
-				k = i - 1
-			} else {
-				k = rand.Intn(len(ids))
-			}
-			if i > 0 {
-				wg.Add(1)
-				go func(i, k int) {
-					defer wg.Done()
-					net.Connect(ids[i], ids[k])
-				}(i, k)
-			}
-		}
-	}
-	wg.Wait()
-
-	log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
-
-	// create an only locally retrieving dpa for the pivot node to test
-	// if retriee requests have arrived
-	dpa := storage.NewDPA(localStores[0], storage.NewChunkerParams())
-	dpa.Start()
-	defer dpa.Stop()
-	timeout := 300 * time.Second
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
-		Action:  action(net),
-		Trigger: trigger(net),
-		Expect: &simulations.Expectation{
-			Nodes: ids[0:1],
-			Check: check(net, dpa),
-		},
-	})
-	return result, nil
-}
-
-func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) {
-	// setup
-	addr := RandomAddr() // tested peers peer address
-	to := NewKademlia(addr.OAddr, NewKadParams())
-
-	// temp datadir
-	datadir, err := ioutil.TempDir("", "streamer")
-	if err != nil {
-		return nil, nil, nil, func() {}, err
-	}
-	teardown := func() {
-		os.RemoveAll(datadir)
-	}
-
-	localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over())
-	if err != nil {
-		return nil, nil, nil, teardown, err
-	}
-
-	dbAccess := NewDbAccess(localStore)
-	delivery := NewDelivery(to, dbAccess)
-	streamer := NewStreamer(delivery)
-	run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
-		bzzPeer := &bzzPeer{
-			Peer:      protocols.NewPeer(p, rw, StreamerSpec),
-			localAddr: addr,
-			BzzAddr:   NewAddrFromNodeID(p.ID()),
-		}
-		to.On(bzzPeer)
-		return streamer.Run(bzzPeer)
-	}
-	protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run)
-
-	err = waitForPeers(streamer, 1*time.Second)
-	if err != nil {
-		return nil, nil, nil, nil, errors.New("timeout: peer is not created")
-	}
-
-	return protocolTester, streamer, localStore, teardown, nil
-}
-
-type roundRobinStore struct {
-	index  uint32
-	stores []storage.ChunkStore
-}
-
-func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore {
-	return &roundRobinStore{
-		stores: stores,
-	}
-}
-
-func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) {
-	return nil, errors.New("get not well defined on round robin store")
-}
-
-func (rrs *roundRobinStore) Put(chunk *storage.Chunk) {
-	i := atomic.AddUint32(&rrs.index, 1)
-	idx := int(i) % len(rrs.stores)
-	rrs.stores[idx].Put(chunk)
-}
-
-func (rrs *roundRobinStore) Close() {
-	for _, store := range rrs.stores {
-		store.Close()
-	}
-}
-
-func waitForPeers(streamer *Streamer, timeout time.Duration) error {
-	ticker := time.NewTicker(10 * time.Millisecond)
-	timeoutTimer := time.NewTimer(timeout)
-	for {
-		select {
-		case <-ticker.C:
-			if len(streamer.peers) > 0 {
-				return nil
-			}
-		case <-timeoutTimer.C:
-			return errors.New("timeout")
-		}
-	}
-}
-
-type testStreamerService struct {
-	index    int
-	addr     *BzzAddr
-	streamer *Streamer
-	run      func(p *p2p.Peer, rw p2p.MsgReadWriter) error
-}
-
-func (tds *testStreamerService) Protocols() []p2p.Protocol {
-	return []p2p.Protocol{
-		{
-			Name:    StreamerSpec.Name,
-			Version: StreamerSpec.Version,
-			Length:  StreamerSpec.Length(),
-			Run:     tds.run,
-			// NodeInfo: ,
-			// PeerInfo: ,
-		},
-	}
-}
-
-func (b *testStreamerService) APIs() []rpc.API {
-	return []rpc.API{}
-}
-
-func (b *testStreamerService) Start(server *p2p.Server) error {
-	return nil
-}
-
-func (b *testStreamerService) Stop() error {
-	return nil
-}

swarm/network/syncer_test.go

@@ -1,168 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"context"
-	crand "crypto/rand"
-	"fmt"
-	"io"
-	"math"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/protocols"
-	"github.com/ethereum/go-ethereum/p2p/simulations"
-	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-func TestSyncerSimulation(t *testing.T) {
-	testSimulation(t, testSyncBetweenNodes(2, 1, 81000, true, 1))
-	testSimulation(t, testSyncBetweenNodes(3, 1, 81000, true, 1))
-}
-
-func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) {
-	return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) {
-		trigger := func(net *simulations.Network) chan discover.NodeID {
-			triggerC := make(chan discover.NodeID)
-			ticker := time.NewTicker(500 * time.Millisecond)
-			go func() {
-				defer ticker.Stop()
-				// we are only testing the pivot node (net.Nodes[0])
-				for range ticker.C {
-					triggerC <- net.Nodes[0].ID()
-				}
-			}()
-			return triggerC
-		}
-
-		action := func(net *simulations.Network) func(context.Context) error {
-			// here we distribute chunks of a random file into localstores of nodes 1 to nodes
-			rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams())
-			rrdpa.Start()
-			// create a retriever dpa for the pivot node
-			return func(context.Context) error {
-				defer rrdpa.Stop()
-				// upload an actual random file of size size
-				_, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size))
-				if err != nil {
-					return err
-				}
-				// wait until all chunks stored
-				wait()
-				return nil
-			}
-		}
-
-		check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) {
-			dbAccesses := make([]*DbAccess, nodes)
-
-			for i := 0; i < nodes; i++ {
-				dbAccesses[i] = NewDbAccess(localStores[i].(*storage.LocalStore))
-			}
-			return func(ctx context.Context, id discover.NodeID) (bool, error) {
-				if id != net.Nodes[0].ID() {
-					return true, nil
-				}
-				select {
-				case <-ctx.Done():
-					return false, ctx.Err()
-				default:
-				}
-
-				var found, total int
-				for i := 1; i < nodes; i++ {
-					dbAccesses[i].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool {
-						_, err := dbAccesses[0].get(key)
-						if err == nil {
-							found++
-						}
-						total++
-						return true
-					})
-				}
-				log.Debug("sync check", "bin", po, "found", found, "total", total)
-				return found == total, nil
-			}
-		}
-		toAddr := func(id discover.NodeID) *BzzAddr {
-			addr := NewAddrFromNodeID(id)
-			addr.OAddr[0] = byte(0)
-			return addr
-		}
-
-		result, err := runSimulation(nodes, conns, "syncer", toAddr, action, trigger, check, adapter)
-		if err != nil {
-			return nil, fmt.Errorf("Setting up simulation failed: %v", err)
-		}
-		if result.Error != nil {
-			return nil, fmt.Errorf("Simulation failed: %s", result.Error)
-		}
-		return result, err
-	}
-}
-
-func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) {
-	id := ctx.Config.ID
-	addr := NewAddrFromNodeID(id)
-	// for the test we make all peers share 8 bits so that syncing full bins make sense
-	addr.OAddr[0] = byte(0)
-	kad := NewKademlia(addr.Over(), NewKadParams())
-	localStore := localStores[nodeCount]
-	dbAccess := NewDbAccess(localStore.(*storage.LocalStore))
-	streamer := NewStreamer(NewDelivery(kad, dbAccess))
-	RegisterIncomingSyncer(streamer, dbAccess)
-	RegisterOutgoingSyncer(streamer, dbAccess)
-
-	self := &testStreamerService{
-		index:    nodeCount,
-		addr:     addr,
-		streamer: streamer,
-	}
-	self.run = self.runSyncer
-	nodeCount++
-	return self, nil
-}
-
-func (b *testStreamerService) runSyncer(p *p2p.Peer, rw p2p.MsgReadWriter) error {
-	addr := NewAddrFromNodeID(p.ID())
-	addr.OAddr[0] = byte(0)
-	bzzPeer := &bzzPeer{
-		Peer:      protocols.NewPeer(p, rw, StreamerSpec),
-		localAddr: b.addr,
-		BzzAddr:   addr,
-	}
-	b.streamer.delivery.overlay.On(bzzPeer)
-	defer b.streamer.delivery.overlay.Off(bzzPeer)
-	// if len(addr) > b.index+1 && bytes.Equal(addrs[b.index+1], addr) {
-	go func() {
-		// each node Subscribes to each other's retrieveRequestStream
-		// need to wait till an aynchronous process registers the peers in streamer.peers
-		// that is used by Subscribe
-		time.Sleep(1 * time.Second)
-		if err := b.streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil {
-			log.Warn("error in subscribe", "err", err)
-		}
-	}()
-	// }
-	return b.streamer.Run(bzzPeer)
-}

swarm/storage/chunker.go

@@ -13,7 +13,6 @@
 //
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package storage
 
 import (
@@ -463,8 +462,7 @@ func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
 	case <-chunk.C: // bells are ringing, data have been delivered
 	}
 	if len(chunk.SData) == 0 {
-		return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
+		return nil
-
 	}
 	return chunk
 }

swarm/storage/dbapi.go

@@ -0,0 +1,52 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+// wrapper of db-s to provide mockable custom local chunk store access to syncer
+type DBAPI struct {
+	db  *DbStore
+	loc *LocalStore
+}
+
+func NewDBAPI(loc *LocalStore) *DBAPI {
+	return &DBAPI{loc.DbStore.(*DbStore), loc}
+}
+
+// to obtain the chunks from key or request db entry only
+func (self *DBAPI) Get(key Key) (*Chunk, error) {
+	return self.loc.Get(key)
+}
+
+// current storage counter of chunk db
+func (self *DBAPI) CurrentBucketStorageIndex(po uint8) uint64 {
+	return self.db.CurrentBucketStorageIndex(po)
+}
+
+// iteration storage counter and proximity order
+func (self *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Key, uint64) bool) error {
+	return self.db.SyncIterator(from, to, po, f)
+}
+
+// to obtain the chunks from key or request db entry only
+func (self *DBAPI) GetOrCreateRequest(key Key) (*Chunk, bool) {
+	return self.loc.GetOrCreateRequest(key)
+}
+
+// to obtain the chunks from key or request db entry only
+func (self *DBAPI) Put(chunk *Chunk) {
+	self.loc.Put(chunk)
+}

swarm/storage/dbstore.go

@@ -599,8 +599,9 @@ func (s *DbStore) writeBatches() {
 		s.batchC = make(chan bool)
 		s.batch = new(leveldb.Batch)
 		s.lock.Unlock()
-		log.Trace(fmt.Sprintf("DbStore: spawn batch write (%d chunks) ", b.Len()))
+		err := s.writeBatch(b, e, d, a)
-		s.writeBatch(b, e, d, a)
+		// TODO: set this error on the batch, then tell the chunk
+		log.Trace(fmt.Sprintf("DbStore: spawn batch write (%d chunks): %v", b.Len(), err))
 		close(c)
 		if e >= s.capacity {
 			log.Trace(fmt.Sprintf("DbStore: collecting garbage...(%d chunks)", e))
@@ -611,15 +612,16 @@ func (s *DbStore) writeBatches() {
 }
 
 // must be called non concurrently
-func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) {
+func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) error {
 	b.Put(keyEntryCnt, U64ToBytes(entryCnt))
 	b.Put(keyDataIdx, U64ToBytes(dataIdx))
 	b.Put(keyAccessCnt, U64ToBytes(accessCnt))
 	l := b.Len()
 	if err := s.db.Write(b); err != nil {
-		log.Error(fmt.Sprintf("unable to write batch: %v", err))
+		return fmt.Errorf("unable to write batch: %v", err)
 	}
 	log.Trace(fmt.Sprintf("DbStore: batch write (%d chunks) complete", l))
+	return nil
 }
 
 // newMockEncodeDataFunc returns a function that stores the chunk data

swarm/storage/memstore.go

@@ -240,7 +240,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
 
 func (s *MemStore) removeOldest() {
 	node := s.memtree
-
+	log.Warn("purge memstore")
 	for node.entry == nil {
 
 		aidx := uint(0)
@@ -284,9 +284,11 @@ func (s *MemStore) removeOldest() {
 	<-node.entry.dbStored
 	log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()))
 
-	if node.entry.SData != nil {
+	if node.entry.ReqC == nil {
 		node.entry = nil
 		s.entryCnt--
+	} else {
+		return
 	}
 
 	node.access[0] = 0

swarm/storage/netstore.go

@@ -18,10 +18,7 @@ package storage
 
 import (
 	"encoding/binary"
-	"fmt"
 	"time"
-
-	"github.com/ethereum/go-ethereum/log"
 )
 
 // NetStore implements the ChunkStore interface,
@@ -43,7 +40,6 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
 	var created bool
 	chunk, created = self.localStore.GetOrCreateRequest(key)
 	if chunk.ReqC == nil {
-		log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData)))
 		return
 	}
 
@@ -57,7 +53,6 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
 
 	select {
 	case <-t.C:
-		log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log()))
 		return nil, notFound
 	case <-chunk.ReqC:
 	}

swarm/swarm.go

@@ -39,6 +39,7 @@ import (
 	httpapi "github.com/ethereum/go-ethereum/swarm/api/http"
 	"github.com/ethereum/go-ethereum/swarm/fuse"
 	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/network/stream"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/mock"
@@ -53,7 +54,7 @@ type Swarm struct {
 	//storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends
 	dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support
 	//depo        network.StorageHandler // remote request handler, interface between bzz protocol and the storage
-	streamer *network.Streamer
+	streamer *stream.Registry
 	//cloud       storage.CloudStore // procurement, cloud storage backend (can multi-cloud)
 	bzz         *network.Bzz       // the logistic manager
 	backend     chequebook.Backend // simple blockchain Backend
@@ -129,13 +130,13 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
 		HiveParams:   config.HiveParams,
 	}
 
-	dbAccess := network.NewDbAccess(self.lstore)
+	db := storage.NewDBAPI(self.lstore)
-	delivery := network.NewDelivery(to, dbAccess)
+	delivery := stream.NewDelivery(to, db)
-	self.streamer = network.NewStreamer(delivery)
+	self.streamer = stream.NewRegistry(addr, delivery)
-	network.RegisterOutgoingSyncer(self.streamer, dbAccess)
+	stream.RegisterSwarmSyncerServer(self.streamer, db)
-	network.RegisterIncomingSyncer(self.streamer, dbAccess)
+	stream.RegisterSwarmSyncerClient(self.streamer, db)
 
-	self.bzz = network.NewBzz(bzzconfig, to, nil, self.streamer)
+	self.bzz = network.NewBzz(bzzconfig, to, nil)
 
 	// set up DPA, the cloud storage local access layer
 	dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve)
@@ -271,6 +272,11 @@ func (self *Swarm) Protocols() (protos []p2p.Protocol) {
 			protos = append(protos, p)
 		}
 	}
+	if self.streamer != nil {
+		for _, p := range self.streamer.Protocols() {
+			protos = append(protos, p)
+		}
+	}
 	return
 }
 
@@ -283,7 +289,7 @@ func (self *Swarm) RegisterPssProtocol(spec *protocols.Spec, targetprotocol *p2p
 }
 
 // implements node.Service
-// Apis returns the RPC Api descriptors the Swarm implementation offers
+// APIs returns the RPC Api descriptors the Swarm implementation offers
 func (self *Swarm) APIs() []rpc.API {
 
 	apis := []rpc.API{