p2p: network testing framework and protocol abstraction

subpackages:

* adapters:
  * msgpipes for simulated test connections
  * rlpx the RLPx adapter for normal non-test use
  * inproc simulated in-process network adapter
  * docker placeholder for docker cluster remote adapter
* protocols: easy-to-setup modular protocols
* simulations:
  * generic network model
  * journal, events, snapshots
  * cytoscape visualisation plugin
  * resourceful controller suite + REST API server
  * example: connectivity UX backend
* testing: test resource drivers
  * exchange: trigger/expect style driver for single node and its peers
  * sessions:   for unit testing protocols and protocol modules
  * (network: for network testing, benchmarking, stats, correctness, fault tolerance)
  * test peerpool

see more in the README-s in each subpackage

* p2p/server : conn/disconn hooks
* reporter remote client skeleton

p2p/adapters/docker.go

@@ -0,0 +1,53 @@
+package adapters
+
+import (
+	"fmt"
+	"net"
+	// "net/http"
+	// "time"
+)
+
+func NewRemoteNode(id *NodeId, n Network, m Messenger) *RemoteNode {
+	return &RemoteNode{
+		ID:      id,
+		Network: n,
+	}
+}
+
+// RemoteNode is the network adapter that
+type RemoteNode struct {
+	ID   *NodeId
+	addr net.Addr
+	Network
+}
+
+func Name(id []byte) string {
+	return fmt.Sprintf("test-%08x", id)
+}
+
+// inject(s) sends an RPC command remotely via ssh to the particular dockernode
+func (self *RemoteNode) inject(string) error {
+	return nil
+}
+
+func (self *RemoteNode) LocalAddr() []byte {
+	return []byte(self.addr.String())
+}
+
+func (self *RemoteNode) ParseAddr(p []byte, s string) ([]byte, error) {
+	return p, nil
+}
+
+func (self *RemoteNode) Disconnect(rid []byte) error {
+	// ssh+ipc -> drop
+	// assumes the remote node is running the p2p module as part of the protocol
+	cmd := fmt.Sprintf(`p2p.Drop("%v")`, string(rid))
+	return self.inject(cmd)
+}
+
+func (self *RemoteNode) Connect(rid []byte) error {
+	// ssh+ipc -> connect
+	//
+	cmd := fmt.Sprintf(`admin.addPeer("%v")`, string(rid))
+	return self.inject(cmd)
+}

p2p/adapters/inproc.go

@@ -0,0 +1,182 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package adapters
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+func newPeer(rw p2p.MsgReadWriter) *Peer {
+	return &Peer{
+		RW:     rw,
+		Errc:   make(chan error, 1),
+		Flushc: make(chan bool),
+	}
+}
+
+type Peer struct {
+	RW     p2p.MsgReadWriter
+	Errc   chan error
+	Flushc chan bool
+}
+
+// Network interface to retrieve protocol runner to launch upon peer
+// connection
+type Network interface {
+	GetNodeAdapter(id *NodeId) NodeAdapter
+	Reporter
+}
+
+// SimNode is the network adapter that
+type SimNode struct {
+	lock      sync.RWMutex
+	Id        *NodeId
+	network   Network
+	messenger Messenger
+	peerMap   map[discover.NodeID]int
+	peers     []*Peer
+	Run       ProtoCall
+}
+
+func (self *SimNode) Messenger() Messenger {
+	return self.messenger
+}
+
+func NewSimNode(id *NodeId, n Network, m Messenger) *SimNode {
+	return &SimNode{
+		Id:        id,
+		network:   n,
+		messenger: m,
+		peerMap:   make(map[discover.NodeID]int),
+	}
+}
+
+func (self *SimNode) LocalAddr() []byte {
+	return self.Id.Bytes()
+}
+
+func (self *SimNode) ParseAddr(p []byte, s string) ([]byte, error) {
+	return p, nil
+}
+
+func (self *SimNode) GetPeer(id *NodeId) *Peer {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.getPeer(id)
+}
+
+func (self *SimNode) getPeer(id *NodeId) *Peer {
+	i, found := self.peerMap[id.NodeID]
+	if !found {
+		return nil
+	}
+	return self.peers[i]
+}
+
+func (self *SimNode) SetPeer(id *NodeId, rw p2p.MsgReadWriter) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.setPeer(id, rw)
+}
+
+func (self *SimNode) setPeer(id *NodeId, rw p2p.MsgReadWriter) *Peer {
+	i, found := self.peerMap[id.NodeID]
+	if !found {
+		i = len(self.peers)
+		self.peerMap[id.NodeID] = i
+		p := newPeer(rw)
+		self.peers = append(self.peers, p)
+		return p
+	}
+	if self.peers[i] != nil && rw != nil {
+		panic(fmt.Sprintf("pipe for %v already set", id))
+	}
+	// legit reconnect reset disconnection error,
+	p := self.peers[i]
+	p.RW = rw
+	return p
+}
+
+func (self *SimNode) Disconnect(rid []byte) error {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	id := NewNodeId(rid)
+	peer := self.getPeer(id)
+	if peer == nil || peer.RW == nil {
+		return fmt.Errorf("already disconnected")
+	}
+	peer.RW.(*p2p.MsgPipeRW).Close()
+	peer.RW = nil
+	// na := self.network.GetNodeAdapter(id)
+	// peer = na.(*SimNode).GetPeer(self.Id)
+	// peer.RW = nil
+	glog.V(6).Infof("dropped peer %v", id)
+	return self.network.DidDisconnect(self.Id, id)
+}
+
+func (self *SimNode) Connect(rid []byte) error {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	id := NewNodeId(rid)
+	na := self.network.GetNodeAdapter(id)
+	if na == nil {
+		return fmt.Errorf("node adapter for %v is missing", id)
+	}
+	rw, rrw := p2p.MsgPipe()
+	runc := make(chan bool)
+	defer close(runc)
+	// run protocol on remote node with self as peer
+
+	err := na.(*SimNode).runProtocol(self.Id, rrw, rw, runc)
+	if err != nil {
+		return fmt.Errorf("cannot run protocol (%v -> %v) %v", self.Id, id, err)
+	}
+	// run protocol on remote node with self as peer
+	err = self.runProtocol(id, rw, rrw, runc)
+	if err != nil {
+		return fmt.Errorf("cannot run protocol (%v -> %v): %v", id, self.Id, err)
+	}
+	self.network.DidConnect(self.Id, id)
+	return nil
+}
+
+func (self *SimNode) runProtocol(id *NodeId, rw, rrw p2p.MsgReadWriter, runc chan bool) error {
+	if self.Run == nil {
+		glog.V(6).Infof("no protocol starting on peer %v (connection with %v)", self.Id, id)
+		return nil
+	}
+	glog.V(6).Infof("protocol starting on peer %v (connection with %v)", self.Id, id)
+	peer := self.getPeer(id)
+	if peer != nil && peer.RW != nil {
+		return fmt.Errorf("already connected %v to peer %v", self.Id, id)
+	}
+	peer = self.setPeer(id, rrw)
+	p := p2p.NewPeer(id.NodeID, Name(id.Bytes()), []p2p.Cap{})
+	go func() {
+		err := self.Run(p, rw)
+		glog.V(6).Infof("protocol quit on peer %v (connection with %v broken)", self.Id, id)
+		<-runc
+		self.Disconnect(id.Bytes())
+		peer.Errc <- err
+	}()
+	return nil
+}

p2p/adapters/msgpipes.go

@@ -0,0 +1,50 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package adapters
+
+import (
+	"github.com/ethereum/go-ethereum/p2p"
+)
+
+//network adapter's messenger interace
+// NewPipe() (p2p.MsgReadWriter, p2p.MsgReadWriter)
+// ClosePipe(rw p2p.MsgReadWriter)
+
+// protocol Messenger interface
+// SendMsg(p2p.MsgWriter, uint64, interface{}) error
+// ReadMsg(p2p.MsgReader) (p2p.Msg, error)
+
+// peer session test
+// ExpectMsg(p2p.MsgReader, uint64, interface{}) error
+// SendMsg(p2p.MsgWriter, uint64, interface{}) error
+type SimPipe struct{}
+
+func (*SimPipe) SendMsg(w p2p.MsgWriter, code uint64, msg interface{}) error {
+	return p2p.Send(w, code, msg)
+}
+
+func (*SimPipe) ReadMsg(r p2p.MsgReader) (p2p.Msg, error) {
+	return r.ReadMsg()
+}
+
+func (*SimPipe) TriggerMsg(w p2p.MsgWriter, code uint64, msg interface{}) error {
+	return p2p.Send(w, code, msg)
+}
+
+func (*SimPipe) ExpectMsg(r p2p.MsgReader, code uint64, msg interface{}) error {
+	return p2p.ExpectMsg(r, code, msg)
+}

p2p/adapters/reporter.go

@@ -0,0 +1,44 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package adapters
+
+import (
+// "fmt"
+// "net"
+
+// "github.com/ethereum/go-ethereum/p2p"
+// "github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+type RemoteReporter struct {
+}
+
+func NewRemoteReporter(url string) *RemoteReporter {
+	return &RemoteReporter{}
+}
+
+func (self *RemoteReporter) DidConnect(source, target *NodeId) {
+	self.post(true)
+}
+
+func (self *RemoteReporter) DidDisconnect(source, target *NodeId) {
+	self.post(true)
+}
+
+func (self *RemoteReporter) post(interface{}) {
+
+}

p2p/adapters/rlpx.go

@@ -0,0 +1,116 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package adapters
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+// devp2p RLPx underlay support
+
+type RLPx struct {
+	id   *NodeId
+	net  *p2p.Server
+	addr []byte
+	m    Messenger
+	r    Reporter
+}
+
+type RLPxMessenger struct {
+}
+
+func NewRLPx(addr []byte, srv *p2p.Server, m Messenger) *RLPx {
+	if m == nil {
+		m = &RLPxMessenger{}
+	}
+	return &RLPx{
+		net:  srv,
+		addr: addr,
+		m:    m,
+	}
+}
+
+func NewReportingRLPx(addr []byte, srv *p2p.Server, m Messenger, r Reporter) *RLPx {
+	rlpx := NewRLPx(addr, srv, m)
+	rlpx.r = r
+	srv.PeerConnHook = func(p *p2p.Peer) {
+		r.DidConnect(rlpx.id, &NodeId{p.ID()})
+	}
+	srv.PeerDisconnHook = func(p *p2p.Peer) {
+		r.DidDisconnect(rlpx.id, &NodeId{p.ID()})
+	}
+	return rlpx
+}
+
+func (*RLPxMessenger) SendMsg(w p2p.MsgWriter, code uint64, msg interface{}) error {
+	return p2p.Send(w, code, msg)
+}
+
+func (*RLPxMessenger) ReadMsg(r p2p.MsgReader) (p2p.Msg, error) {
+	return r.ReadMsg()
+}
+
+func (self *RLPx) LocalAddr() []byte {
+	return self.addr
+}
+
+func (self *RLPx) Connect(enode []byte) error {
+	// TCP/UDP node address encoded with enode url scheme
+	// <node-id>@<ip-address>:<tcp-port>(?udp=<udp-port>)
+	node, err := discover.ParseNode(string(enode))
+	if err != nil {
+		return fmt.Errorf("invalid node URL: %v", err)
+	}
+	self.net.AddPeer(node)
+	return nil
+}
+
+func (self *RLPx) Messenger() Messenger {
+	return self.m
+}
+
+func (self *RLPx) Disconnect(p *p2p.Peer, rw p2p.MsgReadWriter) error {
+	p.Disconnect(p2p.DiscSubprotocolError)
+	return nil
+}
+
+// ParseAddr take two arguments, advertised in handshake and the one set on the peer struct
+// and constructs the remote address object
+func (self *RLPx) ParseAddr(s []byte, remoteAddr string) ([]byte, error) {
+
+	// returns self advertised node connection info (listening address w enodes)
+	// IP will get repaired on the other end if missing
+	// or resolved via ID by discovery at dialout
+	n, err := discover.ParseNode(string(s))
+	if err != nil {
+		return nil, err
+	}
+
+	// repair reported address if IP missing
+	if n.IP.IsUnspecified() {
+		host, _, err := net.SplitHostPort(remoteAddr)
+		if err != nil {
+			return nil, err
+		}
+		n.IP = net.ParseIP(host)
+	}
+	return []byte(n.String()), nil
+}

p2p/adapters/types.go

@@ -0,0 +1,87 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package adapters
+
+import (
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+const lablen = 4
+
+type NodeId struct {
+	discover.NodeID
+}
+
+func NewNodeId(id []byte) *NodeId {
+	var n discover.NodeID
+	copy(n[:], id)
+	return &NodeId{n}
+}
+
+func NewNodeIdFromHex(s string) *NodeId {
+	id := discover.MustHexID(s)
+	return &NodeId{id}
+}
+
+type ProtoCall func(*p2p.Peer, p2p.MsgReadWriter) error
+
+func (self *NodeId) Bytes() []byte {
+	return self.NodeID[:]
+}
+
+func (self *NodeId) MarshalJSON() (out []byte, err error) {
+	return []byte(`"` + self.String() + `"`), nil
+}
+
+func (self *NodeId) UnmarshalJSON(value []byte) error {
+	s := string(value)
+	h, err := discover.HexID(s)
+	if err != nil {
+		return err
+	}
+	*self = NodeId{h}
+	return nil
+}
+
+func (self *NodeId) Label() string {
+	return self.String()[:lablen]
+}
+
+type Messenger interface {
+	SendMsg(p2p.MsgWriter, uint64, interface{}) error
+	ReadMsg(p2p.MsgReader) (p2p.Msg, error)
+}
+
+type NodeAdapter interface {
+	Connect([]byte) error
+	Disconnect([]byte) error
+	// Disconnect(*p2p.Peer, p2p.MsgReadWriter)
+	LocalAddr() []byte
+	ParseAddr([]byte, string) ([]byte, error)
+	Messenger() Messenger
+}
+
+type StartAdapter interface {
+	Start() error
+	Stop() error
+}
+
+type Reporter interface {
+	DidConnect(*NodeId, *NodeId) error
+	DidDisconnect(*NodeId, *NodeId) error
+}

p2p/protocols/README.md

@@ -0,0 +1,27 @@
+p2p/protocols: devp2p subprotocol abstraction
+
+The protocols subpackage is an extension to p2p. It offers a simple and user friendly simple way
+to define devp2p subprotocols by abstracting away code that implementations would typically share.
+
+The package provides a protocol peer object of type protocols.Peer initialised from
+
+* a p2p.Peer, a p2p.MsgReadWriter (the arguments passed to p2p.Protocol#Run),
+* a protocols.CodeMap, this encodes the msg code and msg type associations
+* messenger interface (with methods SendMsg and ReadMsg) that abstracts out sending and receiving a msg
+* disconnect function
+
+Allowing the p2p.Protocol#Run function to construct this peer allows passing it to arbitrary
+service instances sitting on peer connections. These service instances can encapsulate vertical slices
+of business logic without duplicating code related to protocol communication.
+
+Features
+
+* registering multiple handler callbacks for incoming messages
+* automate RLP decoding/encoding based on reflection
+* provide the forever loop to read incoming messages
+* standardise error handling related to communication
+* with disconnection and messaging abstracted out allows protocols to be used
+  in network simulations with or without serialisation, transport and p2p server
+* TODO: automatic generation of wire protocol specification for peers
+
+see the possibly obsolete #2254 for the peer management/connectivity related aspect)

p2p/protocols/protocol.go

@@ -0,0 +1,326 @@
+/*
+Package protocols is an extension to p2p. It offers a user friendly simple way to define
+devp2p subprotocols by abstracting away code standardly shared by protocols.
+The subprotocol architecture is inspired by the node package. Similar to a node
+the standard protocol (class) registers service contructors that are instantiated as service
+instances on the protocol isntance that is launched on a p2p peer connection.
+
+By mounting various protocol modules protocols can encapsulate vertical slices of business logic
+without duplicating code related to protocol communication.
+Standard protocol supports:
+
+* mounting services instantiated with the remote peer when a protocol instance is launched on a newly
+  established peer connection
+* registering module-specific handshakes and offers validation and renegotiation  of handshakes
+* registering multiple handlers for incoming messages
+* automate assigments of code indexes to messages
+* automate RLP decoding/encoding based on reflecting
+* provide the forever loop to read incoming messages
+* standardise error handling related to communication
+* enables access to sister services of the same peer connection analogous to node.Service
+* TODO: automatic generation of wire protocol specification for peers
+* peerPool abstracting out peer management by defining a peerPool that is called to register/unregister
+  peers as they connect and drop (ideally the peerPool also implements the peerPool interface that the
+  p2p server needs to suggest peers to connect to in server-as-initiator mode of operation
+  see https://github.com/ethereum/go-ethereum/issues/2254 for the peer management/connectivity related
+  aspect
+
+*/
+
+package protocols
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p"
+)
+
+// error codes used by this  protocol scheme
+const (
+	ErrMsgTooLong = iota
+	ErrDecode
+	ErrWrite
+	ErrInvalidMsgCode
+	ErrInvalidMsgType
+	ErrLocalHandshake
+	ErrRemoteHandshake
+	ErrNoHandler
+	ErrHandler
+)
+
+// error description strings associated with the codes
+var errorToString = map[int]string{
+	ErrMsgTooLong:      "Message too long",
+	ErrDecode:          "Invalid message (RLP error)",
+	ErrWrite:           "Error sending message",
+	ErrInvalidMsgCode:  "Invalid message code",
+	ErrInvalidMsgType:  "Invalid message type",
+	ErrLocalHandshake:  "Local handshake error",
+	ErrRemoteHandshake: "Remote handshake error",
+	ErrNoHandler:       "No handler registered error",
+	ErrHandler:         "Message handler error",
+}
+
+/*
+Error implements the standard go error interface.
+Use:
+
+  errorf(code, format, params ...interface{})
+
+Prints as:
+
+ <description>: <details>
+
+where description is given by code in errorToString
+and details is fmt.Sprintf(format, params...)
+
+exported field Code can be checked
+*/
+type Error struct {
+	Code    int
+	message string
+	format  string
+	params  []interface{}
+}
+
+func (self Error) Error() (message string) {
+	if len(message) == 0 {
+		name, ok := errorToString[self.Code]
+		if !ok {
+			panic("invalid message code")
+		}
+		self.message = name
+		if self.format != "" {
+			self.message += ": " + fmt.Sprintf(self.format, self.params...)
+		}
+	}
+	return self.message
+}
+
+func errorf(code int, format string, params ...interface{}) *Error {
+	self := &Error{
+		Code:   code,
+		format: format,
+		params: params,
+	}
+
+	return self
+}
+
+// implements the code table spec
+// listing the message codes and types etc
+// and further metadata about the protocol
+type CodeMap struct {
+	Name       string                // name of the protocol
+	Version    uint                  // version
+	MaxMsgSize int                   // max length of message payload size
+	codes      []reflect.Type        // index of codes to msg types - to create zero values
+	messages   map[reflect.Type]uint // index of types to codes, for sending by type
+}
+
+func NewCodeMap(name string, version uint, maxMsgSize int, msgs ...interface{}) *CodeMap {
+	self := &CodeMap{
+		Name:       name,
+		Version:    version,
+		MaxMsgSize: maxMsgSize,
+		messages:   make(map[reflect.Type]uint),
+	}
+	self.Register(msgs...)
+	return self
+}
+
+func (self *CodeMap) Length() uint64 {
+	return uint64(len(self.codes))
+}
+
+func (self *CodeMap) Register(msgs ...interface{}) {
+	code := uint(len(self.codes))
+	for _, msg := range msgs {
+		typ := reflect.TypeOf(msg)
+		_, found := self.messages[typ]
+		if found {
+			// ignore duplicates
+			continue
+		}
+		// next code assigned to message type typ
+		self.messages[typ] = code
+		self.codes = append(self.codes, typ)
+		code++
+	}
+}
+
+// A Peer represents a remote peer or protocol instance that is running on a peer connection with
+// a remote peer
+type Peer struct {
+	ct         *CodeMap                                   // CodeMap for the protocol
+	m          Messenger                                  // defines senf and receive
+	*p2p.Peer                                             // the p2p.Peer object representing the remote
+	rw         p2p.MsgReadWriter                          // p2p.MsgReadWriter to send messages to and read messages from
+	handlers   map[reflect.Type][]func(interface{}) error //  message type -> message handler callback(s) map
+	disconnect func()                                     // Disconnect function set differently for testing
+}
+
+type Messenger interface {
+	SendMsg(p2p.MsgWriter, uint64, interface{}) error
+	ReadMsg(p2p.MsgReader) (p2p.Msg, error)
+}
+
+// NewPeer returns a new peer
+// this constructor is called by the p2p.Protocol#Run function
+// the first two arguments are comming the arguments passed to p2p.Protocol.Run function
+// the third argument is the CodeMap describing the protocol messages and options
+func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, ct *CodeMap, m Messenger, disconn func()) *Peer {
+	return &Peer{
+		ct:         ct,
+		m:          m,
+		Peer:       p,
+		rw:         rw,
+		handlers:   make(map[reflect.Type][]func(interface{}) error),
+		disconnect: disconn,
+	}
+}
+
+// Register is called on the peer typically within the constructor of service instances running on peer connections
+// These constructors are called by the p2p.Protocol#Run function
+// It ties handler callbackss for specific message types
+// A message type can have several handlers registered by the same or different protocol services
+// Register is meant to be called once, deregistering is not currently supported therefore
+// handlers are assumed to be static across handshake renegotiations
+// i.e., a service instance either handles a message or not (irrespective of the handshake)
+// it panics if the message type is not defined in the CodeMap
+func (self *Peer) Register(msg interface{}, handler func(interface{}) error) uint {
+	typ := reflect.TypeOf(msg)
+	code, found := self.ct.messages[typ]
+	if !found {
+		panic(fmt.Sprintf("message type '%v' unknown ", typ))
+	}
+	glog.V(logger.Debug).Infof("registered handle for %v %v", msg, typ)
+	self.handlers[typ] = append(self.handlers[typ], handler)
+	return code
+}
+
+// Run starts the forever loop that handles incoming messages
+// called within the p2p.Protocol#Run function
+func (self *Peer) Run() error {
+	var err error
+	for {
+		_, err = self.handleIncoming()
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// Drop disconnects a peer.
+// falls back to self.disconnect which is set as p2p.Peer#Disconnect except
+// for test peers where it calls p2p.MsgPipe#Close so that the readloop can terminate
+// TODO: may need to implement protocol drop only? don't want to kick off the peer
+// if they are useful for other protocols
+// overwrite Disconnect for testing, so that protocol readloop quits
+func (self *Peer) Drop() {
+	self.disconnect()
+}
+
+// Send takes a message, encodes it in RLP, finds the right message code and sends the
+// message off to the peer
+// this low level call will be wrapped by libraries providing routed or broadcast sends
+// but often just used to forward and push messages to directly connected peers
+func (self *Peer) Send(msg interface{}) error {
+	typ := reflect.TypeOf(msg)
+	code, found := self.ct.messages[typ]
+	if !found {
+		return errorf(ErrInvalidMsgType, "%v", typ)
+	}
+	glog.V(logger.Debug).Infof("=> %v %v (%d)", msg, typ, code)
+	err := self.m.SendMsg(self.rw, uint64(code), msg)
+	if err != nil {
+		self.Drop()
+		return errorf(ErrWrite, "(msg code: %v): %v", code, err)
+	}
+	return nil
+}
+
+// handleIncoming(code)
+// is called each cycle of the main forever loop that handles and dispatches incoming messages
+// if this returns an error the loop returns and the peer is disconnected with the error
+// checks message size, out-of-range message codes, handles decoding with reflection,
+// call handlers as callback onside
+func (self *Peer) handleIncoming() (interface{}, error) {
+	msg, err := self.m.ReadMsg(self.rw)
+	if err != nil {
+		return nil, err
+	}
+	glog.V(logger.Debug).Infof("<= %v", msg)
+	// make sure that the payload has been fully consumed
+	defer msg.Discard()
+
+	if msg.Size > uint32(self.ct.MaxMsgSize) {
+		return nil, errorf(ErrMsgTooLong, "%v > %v", msg.Size, self.ct.MaxMsgSize)
+	}
+
+	// check if the message code is correct
+	maxMsgCode := uint(len(self.ct.messages))
+	if msg.Code >= uint64(maxMsgCode) {
+		return nil, errorf(ErrInvalidMsgCode, "%v (>=%v)", msg.Code, maxMsgCode)
+	}
+
+	// it is safe to be unsafe here
+	typ := self.ct.codes[msg.Code]
+	val := reflect.New(typ)
+	req := val.Elem()
+	req.Set(reflect.Zero(typ))
+	if err := msg.Decode(val.Interface()); err != nil {
+		return nil, errorf(ErrDecode, "<= %v: %v", msg, err)
+	}
+	glog.V(logger.Debug).Infof("<= %v %v (%d)", req, typ, msg.Code)
+
+	// call the registered handler callbacks
+	// a registered callback take the decoded message as argument as an interface
+	// which the handler is supposed to cast to the appropriate type
+	// it is entirely safe not to check the cast in the handler since the handler is
+	// chosen based on the proper type in the first place
+	handlers := self.handlers[typ]
+	if len(handlers) == 0 {
+		glog.V(6).Infof("no handler (msg code %v)", msg.Code)
+		// return nil, errorf(ErrNoHandler, "(msg code %v)", msg.Code)
+	} else {
+		for i, f := range handlers {
+			glog.V(6).Infof("handler %v for %v", i, typ)
+			err = f(req.Interface())
+			if err != nil {
+				return nil, errorf(ErrHandler, "(msg code %v): %v", msg.Code, err)
+			}
+		}
+	}
+	return req.Interface(), nil
+}
+
+// Handshake initiates a handshake on the peer connection
+// * the argument is the local  handshake 	to be sent to the remote peer
+// * expects a remote handshake back of the same type
+// returns the remote hs and an error
+func (self *Peer) Handshake(hs interface{}) (interface{}, error) {
+	typ := reflect.TypeOf(hs)
+	_, found := self.ct.messages[typ]
+	if !found {
+		return nil, errorf(ErrLocalHandshake, "unknown handshake message type: %v", typ)
+	}
+	errc := make(chan error)
+	go func() {
+		err := self.Send(hs)
+		if err != nil {
+			err = errorf(ErrLocalHandshake, "cannot send: %v", err)
+		}
+		errc <- err
+	}()
+	// receiving and validating remote handshake, expect code
+	rhs, err := self.handleIncoming()
+	if err != nil {
+		return nil, errorf(ErrRemoteHandshake, "'%v': %v", self.ct.Name, err)
+	}
+	err = <-errc
+	return rhs, err
+}

p2p/protocols/protocol_test.go

@@ -0,0 +1,367 @@
+package protocols
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
+)
+
+func init() {
+	glog.SetV(6)
+	glog.SetToStderr(true)
+}
+
+// handshake message type
+type hs0 struct {
+	C uint
+}
+
+// message to kill/drop the peer with nodeId
+type kill struct {
+	C *adapters.NodeId
+}
+
+// message to drop connection
+type drop struct {
+}
+
+/// protoHandshake represents module-independent aspects of the protocol and is
+// the first message peers send and receive as part the initial exchange
+type protoHandshake struct {
+	Version   uint   // local and remote peer should have identical version
+	NetworkId string // local and remote peer should have identical network id
+}
+
+// checkProtoHandshake verifies local and remote protoHandshakes match
+func checkProtoHandshake(local, remote *protoHandshake) error {
+
+	if remote.NetworkId != local.NetworkId {
+		return fmt.Errorf("%s (!= %s)", remote.NetworkId, local.NetworkId)
+	}
+
+	if remote.Version != local.Version {
+		return fmt.Errorf("%d (!= %d)", remote.Version, local.Version)
+	}
+	return nil
+}
+
+const networkId = "420"
+
+// newProtocol sets up a protocol
+// the run function here demonstrates a typical protocol using peerPool, handshake
+// and messages registered to handlers
+func newProtocol(pp *p2ptest.TestPeerPool, wg *sync.WaitGroup) func(adapters.NodeAdapter) adapters.ProtoCall {
+	ct := NewCodeMap("test", 42, 1024, &protoHandshake{}, &hs0{}, &kill{}, &drop{})
+
+	return func(na adapters.NodeAdapter) adapters.ProtoCall {
+		return func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
+			if wg != nil {
+				wg.Add(1)
+			}
+			id := &adapters.NodeId{p.ID()}
+			peer := NewPeer(p, rw, ct, na.Messenger(), func() { na.Disconnect(id.Bytes()) })
+
+			// demonstrates use of peerPool, killing another peer connection as a response to a message
+			peer.Register(&kill{}, func(msg interface{}) error {
+				id := msg.(*kill).C
+				pp.Get(id).Drop()
+				return nil
+			})
+
+			// for testing we can trigger self induced disconnect upon receiving drop message
+			peer.Register(&drop{}, func(msg interface{}) error {
+				return fmt.Errorf("received disconnect request")
+			})
+
+			// initiate one-off protohandshake and check validity
+			phs := &protoHandshake{ct.Version, networkId}
+			hs, err := peer.Handshake(phs)
+			if err != nil {
+				return err
+			}
+			rhs := hs.(*protoHandshake)
+			err = checkProtoHandshake(phs, rhs)
+			if err != nil {
+				return err
+			}
+
+			lhs := &hs0{42}
+			// module handshake demonstrating a simple repeatable exchange of same-type message
+			hs, err = peer.Handshake(lhs)
+			if err != nil {
+				return err
+			}
+
+			if rmhs := hs.(*hs0); rmhs.C > lhs.C {
+				return fmt.Errorf("handshake mismatch remote %v > local %v", rmhs.C, lhs.C)
+			}
+
+			peer.Register(lhs, func(msg interface{}) error {
+				rhs := msg.(*hs0)
+				if rhs.C > lhs.C {
+					return fmt.Errorf("handshake mismatch remote %v > local %v", rhs.C, lhs.C)
+				}
+				lhs.C += rhs.C
+				return peer.Send(lhs)
+			})
+
+			// add/remove peer from pool
+			pp.Add(peer)
+			defer pp.Remove(peer)
+			// this launches a forever read loop
+			err = peer.Run()
+			if wg != nil {
+				wg.Done()
+			}
+			return err
+		}
+	}
+}
+
+func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool, wg *sync.WaitGroup) *p2ptest.ExchangeSession {
+	id := p2ptest.RandomNodeId()
+	return p2ptest.NewProtocolTester(t, id, 2, newProtocol(pp, wg))
+}
+
+func protoHandshakeExchange(id *adapters.NodeId, proto *protoHandshake) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		p2ptest.Exchange{
+			Expects: []p2ptest.Expect{
+				p2ptest.Expect{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: id,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 0,
+					Msg:  proto,
+					Peer: id,
+				},
+			},
+		},
+	}
+}
+
+func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) {
+	pp := p2ptest.NewTestPeerPool()
+	s := protocolTester(t, pp, nil)
+	// TODO: make this more than one handshake
+	id := s.Ids[0]
+	s.TestExchanges(protoHandshakeExchange(id, proto)...)
+	var disconnects []*p2ptest.Disconnect
+	for i, err := range errs {
+		disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.Ids[i], Error: err})
+	}
+	s.TestDisconnected(disconnects...)
+}
+
+func TestProtoHandshakeVersionMismatch(t *testing.T) {
+	runProtoHandshake(t, &protoHandshake{41, "420"}, fmt.Errorf("41 (!= 42)"))
+}
+
+func TestProtoHandshakeNetworkIdMismatch(t *testing.T) {
+	runProtoHandshake(t, &protoHandshake{42, "421"}, fmt.Errorf("421 (!= 420)"))
+}
+
+func TestProtoHandshakeSuccess(t *testing.T) {
+	runProtoHandshake(t, &protoHandshake{42, "420"})
+}
+
+func moduleHandshakeExchange(id *adapters.NodeId, resp uint) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		p2ptest.Exchange{
+			Expects: []p2ptest.Expect{
+				p2ptest.Expect{
+					Code: 1,
+					Msg:  &hs0{42},
+					Peer: id,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 1,
+					Msg:  &hs0{resp},
+					Peer: id,
+				},
+			},
+		},
+	}
+}
+
+func runModuleHandshake(t *testing.T, resp uint, errs ...error) {
+	pp := p2ptest.NewTestPeerPool()
+	s := protocolTester(t, pp, nil)
+	id := s.Ids[0]
+	s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...)
+	s.TestExchanges(moduleHandshakeExchange(id, resp)...)
+	var disconnects []*p2ptest.Disconnect
+	for i, err := range errs {
+		disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.Ids[i], Error: err})
+	}
+	s.TestDisconnected(disconnects...)
+}
+
+func TestModuleHandshakeError(t *testing.T) {
+	runModuleHandshake(t, 43, fmt.Errorf("handshake mismatch remote 43 > local 42"))
+}
+
+func TestModuleHandshakeSuccess(t *testing.T) {
+	runModuleHandshake(t, 42)
+}
+
+// testing complex interactions over multiple peers, relaying, dropping
+func testMultiPeerSetup(a, b *adapters.NodeId) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		p2ptest.Exchange{
+			Expects: []p2ptest.Expect{
+				p2ptest.Expect{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: a,
+				},
+				p2ptest.Expect{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: b,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: a,
+				},
+				p2ptest.Trigger{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: b,
+				},
+			},
+			Expects: []p2ptest.Expect{
+				p2ptest.Expect{
+					Code: 1,
+					Msg:  &hs0{42},
+					Peer: a,
+				},
+				p2ptest.Expect{
+					Code: 1,
+					Msg:  &hs0{42},
+					Peer: b,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 1,
+					Msg:  &hs0{41},
+					Peer: a,
+				},
+				p2ptest.Trigger{
+					Code: 1,
+					Msg:  &hs0{41},
+					Peer: b,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 1,
+					Msg:  &hs0{1},
+					Peer: a,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Expects: []p2ptest.Expect{
+				p2ptest.Expect{
+					Code: 1,
+					Msg:  &hs0{43},
+					Peer: a,
+				},
+			},
+		},
+	}
+}
+
+func runMultiplePeers(t *testing.T, peer int, errs ...error) {
+	wg := &sync.WaitGroup{}
+	pp := p2ptest.NewTestPeerPool()
+	s := protocolTester(t, pp, wg)
+
+	s.TestExchanges(testMultiPeerSetup(s.Ids[0], s.Ids[1])...)
+	// after some exchanges of messages, we can test state changes
+	// here this is simply demonstrated by the peerPool
+	// after the handshake negotiations peers must be addded to the pool
+	if !pp.Has(s.Ids[0]) {
+		t.Fatalf("missing peer test-0: %v (%v)", pp, s.Ids)
+	}
+	if !pp.Has(s.Ids[1]) {
+		t.Fatalf("missing peer test-1: %v (%v)", pp, s.Ids)
+	}
+
+	// sending kill request for peer with index <peer>
+	s.TestExchanges(p2ptest.Exchange{
+		Triggers: []p2ptest.Trigger{
+			p2ptest.Trigger{
+				Code: 2,
+				Msg:  &kill{s.Ids[peer]},
+				Peer: s.Ids[0],
+			},
+		},
+	})
+
+	// dropping the remaining peer
+	s.TestExchanges(p2ptest.Exchange{
+		Triggers: []p2ptest.Trigger{
+			p2ptest.Trigger{
+				Code: 3,
+				Msg:  &drop{},
+				Peer: s.Ids[(peer+1)%2],
+			},
+		},
+	})
+	wg.Wait()
+	// check the actual discconnect errors on the individual peers
+	var disconnects []*p2ptest.Disconnect
+	for i, err := range errs {
+		disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.Ids[i], Error: err})
+	}
+	s.TestDisconnected(disconnects...)
+	// test if disconnected peers have been removed from peerPool
+	if pp.Has(s.Ids[peer]) {
+		t.Fatalf("peer test-%v not dropped: %v (%v)", peer, pp, s.Ids)
+	}
+
+}
+
+func TestMultiplePeersDropSelf(t *testing.T) {
+	runMultiplePeers(t, 0,
+		fmt.Errorf("p2p: read or write on closed message pipe"),
+		fmt.Errorf("Message handler error: (msg code 3): received disconnect request"),
+	)
+}
+
+func TestMultiplePeersDropOther(t *testing.T) {
+	runMultiplePeers(t, 1,
+		fmt.Errorf("Message handler error: (msg code 3): received disconnect request"),
+		fmt.Errorf("p2p: read or write on closed message pipe"),
+	)
+}

p2p/server.go

@@ -145,6 +145,8 @@ type Server struct {
 	// the whole protocol stack.
 	newTransport    func(net.Conn) transport
 	newPeerHook     func(*Peer)
+	PeerConnHook    func(*Peer)
+	PeerDisconnHook func(*Peer)
 
 	lock    sync.Mutex // protects running
 	running bool
@@ -755,10 +757,16 @@ func (srv *Server) runPeer(p *Peer) {
 	if srv.newPeerHook != nil {
 		srv.newPeerHook(p)
 	}
+	if srv.PeerConnHook != nil {
+		srv.PeerConnHook(p)
+	}
 	remoteRequested, err := p.run()
 	// Note: run waits for existing peers to be sent on srv.delpeer
 	// before returning, so this send should not select on srv.quit.
 	srv.delpeer <- peerDrop{p, err, remoteRequested}
+	if srv.PeerDisconnHook != nil {
+		srv.PeerDisconnHook(p)
+	}
 }
 
 // NodeInfo represents a short summary of the information known about the host.

p2p/simulations/cytoscape.go

@@ -0,0 +1,81 @@
+package simulations
+
+import (
+	// "fmt"
+
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// TODO: to implement cytoscape global behav
+type CyConfig struct {
+}
+
+type CyData struct {
+	Id     string `json:"id"`
+	Source string `json:"source,omitempty"`
+	Target string `json:"target,omitempty"`
+	Up     bool   `json:"up"`
+}
+
+type CyElement struct {
+	Data    *CyData `json:"data"`
+	Classes string  `json:"classes,omitempty"`
+	Group   string  `json:"group"`
+	// selected: false, // whether the element is selected (default false)
+	// selectable: true, // whether the selection state is mutable (default true)
+	// locked: false, // when locked a node's position is immutable (default false)
+	// grabbable: true, // whether the node can be grabbed and moved by the user
+}
+
+type CyUpdate struct {
+	Add    []*CyElement `json:"add"`
+	Remove []string     `json:"remove"`
+}
+
+func UpdateCy(conf *CyConfig, j *Journal) (*CyUpdate, error) {
+	added := []*CyElement{}
+	removed := []string{}
+	var el *CyElement
+	update := func(e *event.Event) bool {
+		entry := e.Data
+		var action string
+		if ev, ok := entry.(*NodeEvent); ok {
+			el = &CyElement{Group: "nodes", Data: &CyData{Id: ev.node.Id.Label()}}
+			action = ev.Action
+		} else if ev, ok := entry.(*ConnEvent); ok {
+			// mutually exclusive directed edge (caller -> callee)
+			conn := ev.conn
+			id := ConnLabel(conn.One, conn.Other)
+			var source, target string
+			if conn.Reverse {
+				source = conn.Other.Label()
+				target = conn.One.Label()
+			} else {
+				source = conn.One.Label()
+				target = conn.Other.Label()
+			}
+			el = &CyElement{Group: "edges", Data: &CyData{Id: id, Source: source, Target: target}}
+			action = ev.Action
+		} else {
+			panic("unknown event type")
+		}
+
+		switch action {
+		case "up":
+			el.Data.Up = true
+			added = append(added, el)
+		case "down":
+			el.Data.Up = false
+			removed = append(removed, el.Data.Id)
+		default:
+			panic("unknown action")
+		}
+		return true
+	}
+	j.Read(update)
+
+	return &CyUpdate{
+		Add:    added,
+		Remove: removed,
+	}, nil
+}

p2p/simulations/examples/connectivity.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"runtime"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+)
+
+// var server
+func main() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	glog.SetV(6)
+	glog.SetToStderr(true)
+
+	c, quitc := simulations.NewSessionController()
+
+	simulations.StartRestApiServer("8888", c)
+	// wait until server shuts down
+	<-quitc
+
+}

p2p/simulations/journal.go

@@ -0,0 +1,488 @@
+package simulations
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+)
+
+// Journal is an instance of a guaranteed no-loss subscription to network related events
+// (using event.TypeMux). Network components POST events to the TypeMux, which then is
+// read by the journal. Each journal belongs to a subscription.
+type Journal struct {
+	Id      string
+	lock    sync.Mutex
+	counter int
+	cursor  int
+	quitc   chan bool
+	Events  []*event.Event
+}
+
+// NewJournal constructor
+// Journal can get input events from subscriptions, add event logs
+// or scheduled replay of events from another journal
+//
+// see the Read and TimedRead iterators for use
+// the Journal is safe for concurrent reads and writes
+func NewJournal() *Journal {
+	return &Journal{quitc: make(chan bool)}
+}
+
+// Subscribe takes an event.TypeMux and subscibes to types
+// and launches a gorourine that appends any new event to the event log
+// used for journalling history of a network
+// the goroutine terminates when the journal is closed
+func (self *Journal) Subscribe(eventer *event.TypeMux, types ...interface{}) {
+	glog.V(6).Infof("subscribe")
+	sub := eventer.Subscribe(types...)
+	go func() {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case ev := <-sub.Chan():
+				self.append(ev)
+			case <-self.quitc:
+				return
+			}
+		}
+	}()
+}
+
+// AddJournal appends the event log of another journal to the receiver's one
+func (self *Journal) AddJournal(j *Journal) {
+	self.append(j.Events...)
+}
+
+// NewJournalFromJSON decodes a JSON serialised events log
+// into a journal struct
+// used to replay recorded history
+func NewJournalFromJSON(b []byte) (*Journal, error) {
+	self := NewJournal()
+	err := json.Unmarshal(b, self)
+	if err != nil {
+		return nil, err
+	}
+	return self, nil
+}
+
+// Replay replays the events of another journal preserving (relative) timing of events
+// params:
+// * acc: using acceleration factor acc
+// * journal: journal to use
+// * eventer: where to post the replayed events
+func Replay(acc float64, j *Journal, eventer *event.TypeMux) {
+	f := func(d interface{}) bool {
+		// reposts the data with the eventer (the data receives a new timestamp)
+		eventer.Post(d)
+		return true
+	}
+	j.TimedRead(acc, f)
+}
+
+// Snapshot creates a snapshot out of the journal
+// this is simply done by reading the event log backwards and mark the last action
+// on a node/connection ignoring all earlier mentions
+// TODO: implmented
+func Snapshot(conf *SnapshotConfig, j *Journal) (*Journal, error) {
+	return nil, fmt.Errorf("snapshot not implemented")
+}
+
+func (self *Journal) Close() {
+	close(self.quitc)
+}
+
+func (self *Journal) append(evs ...*event.Event) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.Events = append(self.Events, evs...)
+	self.counter++
+}
+
+func (self *Journal) NewEntries() int {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.counter - self.cursor
+}
+
+func (self *Journal) WaitEntries(n int) {
+	for self.NewEntries() < n {
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+func (self *Journal) Read(f func(*event.Event) bool) (read int) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	ok := true
+	for self.cursor < len(self.Events) && ok {
+		read++
+		ok = f(self.Events[self.cursor])
+		self.cursor++
+		select {
+		case <-self.quitc:
+			break
+		default:
+		}
+	}
+	self.reset(self.cursor)
+	return read
+}
+
+// TimedRead reads the events but blocks for intervals that correspond to
+// the original time intervals,
+// NOTE: the events' timestamps are supposed to be strictly ordered otherwise
+// the call panics.
+// acc is an acceleration factor
+func (self *Journal) TimedRead(acc float64, f func(interface{}) bool) (read int) {
+	var lastEvent time.Time
+	timer := time.NewTimer(0)
+	var data interface{}
+	h := func(ev *event.Event) bool {
+		// wait for the interval time passes event time
+		if ev.Time.Before(lastEvent) {
+			panic("events not ordered")
+		}
+		interval := ev.Time.Sub(lastEvent)
+		glog.V(6).Infof("reset timer to interval %v", interval)
+		timer.Reset(time.Duration(acc) * interval)
+		lastEvent = ev.Time
+		data = ev.Data
+		return false
+	}
+	var n int
+	for {
+		// Read blocks for the iteration. need to read one event at a time so that
+		// waiting for the timer to go off does not block concurrent access to the journal
+		n = self.Read(h)
+		if read > 0 && n > 0 {
+			select {
+			case <-self.quitc:
+				break
+			case <-timer.C:
+			}
+		}
+		read += n
+		if n == 0 || !f(data) {
+			glog.V(6).Infof("timed read ends (read %v entries)", read)
+			break
+		}
+	}
+	return read
+}
+
+func (self *Journal) Reset(n int) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.reset(n)
+}
+
+func (self *Journal) reset(n int) {
+	length := len(self.Events)
+	if length == 0 {
+		return
+	}
+	if n >= length-1 {
+		n = length - 1
+	}
+	glog.V(6).Infof("cursor reset from %v to %v/%v (%v)", self.cursor, n, len(self.Events), self.counter)
+	self.Events = self.Events[self.cursor:]
+	self.cursor = 0
+}
+
+func (self *Journal) Counter() int {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.counter
+}
+
+// type History()
+
+func (self *Journal) Cursor() int {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.cursor
+}
+
+type SnapshotConfig struct {
+	Id string
+}
+
+type JournalPlayConfig struct {
+	Id      string
+	SpeedUp float64
+	Journal *Journal
+	Events  []string
+}
+
+func NewJournalPlayersController(eventer *event.TypeMux) Controller {
+	self := NewResourceContoller(
+		&ResourceHandlers{
+			// POST /o/players/
+			Create: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					conf := msg.(*JournalPlayConfig)
+					go Replay(conf.SpeedUp, conf.Journal, eventer)
+					c := NewJournalPlayerController(conf)
+					parent.SetResource(conf.Id, c)
+					return empty, nil
+				},
+				Type: reflect.TypeOf(&JournalPlayConfig{}),
+			},
+		})
+	return self
+}
+
+func NewJournalPlayerController(conf *JournalPlayConfig) Controller {
+	self := NewResourceContoller(
+		&ResourceHandlers{
+			// GET /0/players/<playerId>
+			Retrieve: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					return nil, fmt.Errorf("info about journal player not implemented")
+				},
+			},
+			// DELETE /0/players/<playerId>
+			Destroy: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					conf.Journal.Close() // terminate Replay-> TimedRead routine
+					parent.DeleteResource(conf.Id)
+					return empty, nil
+				},
+			},
+		})
+	return self
+}
+
+type MockerConfig struct {
+	// TODO: frequency/volume etc.
+	Id             string
+	NodeCount      int
+	UpdateInterval time.Duration
+}
+
+func NewMockersController(eventer *event.TypeMux) Controller {
+	self := NewResourceContoller(
+		&ResourceHandlers{
+			// Create: n.StartNode, NodeConfig
+			Create: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					conf := msg.(*MockerConfig)
+					if conf.NodeCount == 0 {
+						conf.NodeCount = 100
+					}
+					ids := RandomNodeIds(conf.NodeCount)
+					if conf.UpdateInterval == 0 {
+						conf.UpdateInterval = 1 * time.Second
+					}
+					ticker := time.NewTicker(conf.UpdateInterval)
+					go MockEvents(eventer, ids, ticker.C)
+					c := NewMockerController(conf, ticker)
+					if len(conf.Id) == 0 {
+						conf.Id = fmt.Sprintf("%d", parent.id)
+					}
+					glog.V(6).Infof("new mocker controller on %v", conf.Id)
+					if parent != nil {
+						parent.SetResource(conf.Id, c)
+					}
+					parent.id++
+					return empty, nil
+				},
+				Type: reflect.TypeOf(&MockerConfig{}),
+			},
+		})
+	return self
+}
+
+func NewMockerController(conf *MockerConfig, ticker *time.Ticker) Controller {
+	self := NewResourceContoller(
+		&ResourceHandlers{
+			// GET /0/mockevents/<mockerId>
+			Retrieve: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					return nil, fmt.Errorf("info about mocker not implemented")
+				},
+			},
+			// DELETE /0/mockevents/<mockerId>
+			Destroy: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					ticker.Stop() //terminate MockEvents routine
+					parent.DeleteResource(conf.Id)
+					return empty, nil
+				},
+			},
+		})
+	return self
+}
+
+// deltas: changes in the number of cumulative actions: non-negative integers.
+// base unit is the fixed minimal interval  between two measurements (time quantum)
+// acceleration : to slow down you just set the base unit higher.
+// to speed up: skip x number of base units
+// frequency: given as the (constant or average) number of base units between measurements
+// if resolution is expressed as the inverse of frequency  = preserved information
+// setting the acceleration
+// beginning of the record (lifespan) of the network is index 0
+// acceleration means that snapshots are rarer so the same span can be generated by the journal
+// then update logs can be compressed (toonly one state transition per affected node)
+// epoch, epochcount
+
+func ConnLabel(source, target *adapters.NodeId) string {
+	var first, second *adapters.NodeId
+	if bytes.Compare(source.Bytes(), target.Bytes()) > 0 {
+		first = target
+		second = source
+	} else {
+		first = source
+		second = target
+	}
+	return fmt.Sprintf("%v-%v", first, second)
+}
+
+// MockEvents generates random connectivity events and posts them
+// to the eventer
+// The journal using the eventer can then be read to visualise or
+// drive connections
+func MockEvents(eventer *event.TypeMux, ids []*adapters.NodeId, ticker <-chan time.Time) {
+
+	var onNodes []*Node
+	offNodes := ids
+	onConnsMap := make(map[string]int)
+	var onConns []*Conn
+	connsMap := make(map[string]int)
+	var conns []*Conn
+	// ids := RandomNodeIds(100)
+	switchonRate := 5
+	dropoutRate := 100
+	newConnCount := 1 // new connection per node per tick
+	connFailRate := 100
+	disconnRate := 100 // fraction of all connections
+	nodesTarget := len(ids) / 2
+	degreeTarget := 8
+	convergenceRate := 5
+	rounds := 0
+	for _ = range ticker {
+		glog.V(6).Infof("rates: %v/%v, %v (%v/%v)", switchonRate, dropoutRate, newConnCount, connFailRate, disconnRate)
+		// here switchon rate will depend
+		nodesUp := len(offNodes) / switchonRate
+		missing := nodesTarget - len(onNodes)
+		if missing > 0 {
+			if nodesUp < missing {
+				nodesUp += (missing-nodesUp)/convergenceRate + 1
+			}
+		}
+
+		nodesDown := len(onNodes) / dropoutRate
+
+		connsUp := len(onNodes) * newConnCount
+		connsUp = connsUp - connsUp/connFailRate
+		missing = nodesTarget*degreeTarget/2 - len(onConns)
+		if missing < connsUp {
+			connsUp = missing
+			if connsUp < 0 {
+				connsUp = 0
+			}
+		}
+		connsDown := len(onConns) / disconnRate
+		glog.V(6).Infof("Nodes Up: %v, Down: %v [ON: %v/%v]\nConns Up: %v, Down: %v [ON: %v/%v(%v)]", nodesUp, nodesDown, len(onNodes), len(onNodes)+len(offNodes), connsUp, connsDown, len(onConns), len(conns)-len(onConns), len(conns))
+
+		for i := 0; len(onNodes) > 0 && i < nodesDown; i++ {
+			c := rand.Intn(len(onNodes))
+			sn := onNodes[c]
+			err := eventer.Post(&NodeEvent{
+				Type:   "node",
+				Action: "down",
+				node:   sn,
+			})
+			if err != nil {
+				panic(err.Error())
+			}
+			onNodes = append(onNodes[0:c], onNodes[c+1:]...)
+			offNodes = append(offNodes, sn.Id)
+		}
+		for i := 0; len(offNodes) > 0 && i < nodesUp; i++ {
+			c := rand.Intn(len(offNodes))
+			sn := &Node{Id: offNodes[c]}
+			err := eventer.Post(&NodeEvent{
+				Type:   "node",
+				Action: "up",
+				node:   sn,
+			})
+			if err != nil {
+				panic(err.Error())
+			}
+			onNodes = append(onNodes, sn)
+			offNodes = append(offNodes[0:c], offNodes[c+1:]...)
+		}
+		var found bool
+		var sc *Conn
+		for i := 0; len(onNodes) > 1 && i < connsUp; i++ {
+			sc = nil
+			n := rand.Intn(len(onNodes) - 1)
+			m := n + 1 + rand.Intn(len(onNodes)-n-1)
+			for i := m; i < len(onNodes); i++ {
+				lab := ConnLabel(onNodes[n].Id, onNodes[i].Id)
+				var j int
+				j, found = onConnsMap[lab]
+				if found {
+					continue
+				}
+				j, found = connsMap[lab]
+				if found {
+					sc = conns[j]
+					break
+				}
+				caller := onNodes[n].Id
+				callee := onNodes[i].Id
+
+				sc := &Conn{
+					One:   caller,
+					Other: callee,
+				}
+				connsMap[lab] = len(conns)
+				conns = append(conns, sc)
+				break
+			}
+
+			if sc == nil {
+				i--
+				continue
+			}
+			lab := ConnLabel(sc.One, sc.Other)
+			onConnsMap[lab] = len(onConns)
+			onConns = append(onConns, sc)
+			err := eventer.Post(&ConnEvent{
+				Type:   "conn",
+				Action: "up",
+				conn:   sc,
+			})
+			if err != nil {
+				panic(err.Error())
+			}
+		}
+
+		for i := 0; len(onConns) > 0 && i < connsDown; i++ {
+			c := rand.Intn(len(onConns))
+			conn := onConns[c]
+			onConns = append(onConns[0:c], onConns[c+1:]...)
+			lab := ConnLabel(conn.One, conn.Other)
+			delete(onConnsMap, lab)
+			err := eventer.Post(&ConnEvent{
+				Type:   "conn",
+				Action: "down",
+				conn:   conn,
+			})
+			if err != nil {
+				panic(err.Error())
+			}
+		}
+		rounds++
+	}
+}

p2p/simulations/journal_test.go

@@ -0,0 +1,141 @@
+package simulations
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+)
+
+func testEvents(intervals ...int) (events []*event.Event) {
+	t := time.Now()
+	for _, interval := range intervals {
+		t = t.Add(time.Duration(interval) * time.Millisecond)
+		events = append(events, &event.Event{
+			Time: t,
+			Data: interface{}(&NodeEvent{
+				Type:   "node",
+				Action: "down",
+			}),
+		})
+	}
+	return events
+}
+
+func TestTimedRead(t *testing.T) {
+	j := NewJournal()
+	intervals := []int{100, 200, 300, 300, 100, 200}
+	j.Events = testEvents(intervals...)
+	var newTimes []time.Time
+	var i int
+	acc := 0.5
+	length := 4
+	f := func(data interface{}) bool {
+		_ = data.(*NodeEvent)
+		newTimes = append(newTimes, time.Now())
+		i++
+		return i <= length
+	}
+	start := time.Now()
+	read := j.TimedRead(acc, f)
+	if read != 5 {
+		t.Fatalf("incorrect number of events read: expected 5, got %v", read)
+	}
+	for i, ti := range newTimes {
+		expInt := time.Duration(acc*float64(intervals[i])) * time.Millisecond
+		gotInt := ti.Sub(start)
+		if gotInt-expInt > 1*time.Millisecond {
+			t.Fatalf("journal timed read incorrect interval: expected %v ,got %v", expInt, gotInt)
+		}
+		start = ti
+	}
+}
+
+func testIDs() (ids []*adapters.NodeId) {
+
+	keys := []string{
+		"aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80aa7cca80",
+		"f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3f5ae22c3",
+	}
+	for _, key := range keys {
+		id := adapters.NewNodeIdFromHex(key)
+		ids = append(ids, id)
+	}
+	return ids
+}
+
+func testJournal(ids []*adapters.NodeId) *Journal {
+	eventer := &event.TypeMux{}
+	journal := NewJournal()
+	journal.Subscribe(eventer, ConnectivityEvents...)
+	mockNewNodes(eventer, ids)
+	journal.WaitEntries(len(ids))
+	return journal
+}
+
+func TestSubscribe(t *testing.T) {
+	ids := testIDs()
+	journal := testJournal(ids)
+	for i, ev := range journal.Events {
+		id := ev.Data.(*NodeEvent).node.Id
+		if id != ids[i] {
+			t.Fatalf("incorrect id: expected %v, got %v", id, ids[i])
+		}
+	}
+}
+
+func loadTestJournal(t *testing.T) ([]byte, *Journal) {
+	b, err := ioutil.ReadFile("./testjournal.json")
+	if err != nil {
+		t.Fatalf("unexpected error reading test journal json: %v", err)
+	}
+	journal, err := NewJournalFromJSON(b)
+	if err != nil {
+		t.Fatalf("unexpected error decoding journal json: %v", err)
+	}
+	return b, journal
+}
+
+func TestLoadSave(t *testing.T) {
+	b, j := loadTestJournal(t)
+
+	jo, err := json.MarshalIndent(j, "", " ")
+	if err != nil {
+		t.Fatalf("unexpected error encoding journal for %v: %v", j, err)
+	}
+	expJSON := string(b)
+	gotJSON := string(jo)
+	if expJSON != gotJSON {
+		t.Fatalf("incorrect json for journal: expected %v, got %v", expJSON, gotJSON)
+	}
+}
+
+func TestReplay(t *testing.T) {
+	_, jo := loadTestJournal(t)
+	eventer := &event.TypeMux{}
+
+	journal := NewJournal()
+	journal.Subscribe(eventer, ConnectivityEvents...)
+
+	Replay(0, jo, eventer)
+	for i, ev := range jo.Events {
+		exp := ev.Data.(*NodeEvent).String()
+		got := journal.Events[i].Data.(*NodeEvent).String()
+		if exp != got {
+			t.Fatalf("incorrent replayed journal entry at pos %v: expected %v, got %v", i, exp, got)
+		}
+	}
+	// ids := RandomNodeIds(7)
+	// ticker := time.NewTicker(1000 * time.Microsecond)
+	// go MockEvents(eventer, ids, ticker.C)
+	// journal.WaitEntries(20)
+	// // eventer.Stop()	// eventer = &event.TypeMux{}
+	// journal = NewJournal()
+	// journal.Subscribe(eventer, &Entry{})
+
+	// func TestReplay(t *testing.T) {
+	// }
+}

p2p/simulations/network.go

@@ -0,0 +1,507 @@
+// Package simulations simulates p2p networks.
+//
+// Network
+//  - has nodes
+//  - has connections
+//  - has triggers (input eventer, triggers things like start and stop of nodes, connecting them)
+//  - has output eventer, where stuff that happens during simulation is sent
+//  - the adapter of new nodes is assigned by the Node Adapter Function.
+//
+// Sources of Trigger events
+// - UI (click of button)
+// - Journal (replay captured events)
+// - Mocker (generate random events)
+//
+// Adapters
+// - each node has an adapter
+// - contains methods to connect to another node using the same adapter type
+// - models communication too (sending and receiving messages)
+//
+// REST API
+// - Session Controller: handles Networks
+// - Network Controller
+//   - handles one Network
+//   - has sub controller for triggering events
+//   - get output events
+//
+package simulations
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+type NetworkConfig struct {
+	// Type   NetworkType
+	// Config json.RawMessage // type-specific configs
+	// type
+	// Events []string
+	Id string
+}
+
+// event types related to connectivity, i.e., nodes coming on dropping off
+// and connections established and dropped
+var ConnectivityEvents = []interface{}{&NodeEvent{}, &ConnEvent{}}
+
+// NewNetworkController creates a ResourceController responding to GET and DELETE methods
+// it embeds a mockers controller, a journal player, node and connection contollers.
+//
+// Events from the eventer go into the provided journal. The content of the journal can be
+// accessed through the HTTP API.
+func NewNetworkController(conf *NetworkConfig, eventer *event.TypeMux, journal *Journal) Controller {
+	self := NewResourceContoller(
+		&ResourceHandlers{
+			// GET /<networkId>/
+			Retrieve: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					glog.V(6).Infof("msg: %v", msg)
+					cyConfig, ok := msg.(*CyConfig)
+					if ok {
+						return UpdateCy(cyConfig, journal)
+					}
+					snapshotConfig, ok := msg.(*SnapshotConfig)
+					if ok {
+						return Snapshot(snapshotConfig, journal)
+					}
+					return nil, fmt.Errorf("invalId json body: must be CyConfig or SnapshotConfig")
+				},
+				Type: reflect.TypeOf(&CyConfig{}),
+			},
+			// DELETE /<networkId>/
+			Destroy: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					parent.DeleteResource(conf.Id)
+					return nil, nil
+				},
+			},
+		},
+	)
+	// subscribe to all event entries (generated)
+	journal.Subscribe(eventer, ConnectivityEvents...)
+	// self.SetResource("nodes", NewNodesController(eventer))
+	// self.SetResource("connections", NewConnectionsController(eventer))
+	self.SetResource("mockevents", NewMockersController(eventer))
+	self.SetResource("journals", NewJournalPlayersController(eventer))
+	return Controller(self)
+}
+
+// Network models a p2p network
+// the actual logic of bringing nodes and connections up and down and
+// messaging is implemented in the particular NodeAdapter interface
+type Network struct {
+	// input trigger events and other events
+	triggers *event.TypeMux // event triggers
+	events   *event.TypeMux // events
+	lock     sync.RWMutex
+	nodeMap  map[discover.NodeID]int
+	connMap  map[string]int
+	Nodes    []*Node `json:"nodes"`
+	Conns    []*Conn `json:"conns"`
+	//
+	// adapters.Messenger
+	// node adapter function that creates the node model for
+	// the particular type of network from a config
+	naf func(*NodeConfig) adapters.NodeAdapter
+}
+
+func NewNetwork(triggers, events *event.TypeMux) *Network {
+	return &Network{
+		triggers: triggers,
+		events:   events,
+		nodeMap:  make(map[discover.NodeID]int),
+		connMap:  make(map[string]int),
+	}
+}
+
+func (self *Network) SetNaf(naf func(*NodeConfig) adapters.NodeAdapter) {
+	self.naf = naf
+}
+
+// Events returns the output eventer of the Network.
+func (self *Network) Events() *event.TypeMux {
+	return self.events
+}
+
+type Node struct {
+	Id     *adapters.NodeId `json:"id"`
+	Up     bool
+	config *NodeConfig
+	na     adapters.NodeAdapter
+}
+
+func (self *Node) Adapter() adapters.NodeAdapter {
+	return self.na
+}
+
+func (self *Node) String() string {
+	return fmt.Sprintf("Node %v", self.Id.Label())
+}
+
+type NodeEvent struct {
+	Action string
+	Type   string
+	node   *Node
+}
+
+type ConnEvent struct {
+	Action string
+	Type   string
+	conn   *Conn
+}
+
+func (self *ConnEvent) String() string {
+	return fmt.Sprintf("<Action: %v, Type: %v, Data: %v>\n", self.Action, self.Type, self.conn)
+}
+
+func (self *NodeEvent) String() string {
+	return fmt.Sprintf("<Action: %v, Type: %v, Data: %v>\n", self.Action, self.Type, self.node)
+}
+
+func (self *Node) event(up bool) *NodeEvent {
+	var action string
+	if up {
+		action = "up"
+	} else {
+		action = "down"
+	}
+	return &NodeEvent{
+		Action: action,
+		Type:   "node",
+		node:   self,
+	}
+}
+
+// active connections are represented by the Node entry object so that
+// you journal updates could filter if passive knowledge about peers is
+// irrelevant
+type Conn struct {
+	One        *adapters.NodeId `json:"one"`
+	Other      *adapters.NodeId `json:"other"`
+	one, other *Node
+	// connection down by default
+	Up bool `json:"up"`
+	// reverse is false by default (One dialled/dropped the Other)
+	Reverse bool `json:"reverse"`
+	// Info
+	// average throughput, recent average throughput etc
+}
+
+func (self *Conn) String() string {
+	return fmt.Sprintf("Conn %v->%v", self.One.Label(), self.Other.Label())
+}
+
+func (self *Conn) event(up, rev bool) *ConnEvent {
+	var action string
+	if up {
+		action = "up"
+	} else {
+		action = "down"
+	}
+	return &ConnEvent{
+		Action: action,
+		Type:   "conn",
+		conn:   self,
+	}
+}
+
+type NodeConfig struct {
+	Id *adapters.NodeId `json:"Id"`
+}
+
+// TODO: ignored for now
+type QueryConfig struct {
+	Format string // "cy.update", "journal",
+}
+
+type Know struct {
+	Subject *adapters.NodeId `json:"subject"`
+	Object  *adapters.NodeId `json:"object"`
+	// Into
+	// number of attempted connections
+	// time of attempted connections
+	// number of active connections during the session
+	// number of active connections since records began
+	// swap balance
+}
+
+// NewNode adds a new node to the network
+// errors if a node by the same id already exist
+func (self *Network) NewNode(conf *NodeConfig) error {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	id := conf.Id
+
+	_, found := self.nodeMap[id.NodeID]
+	if found {
+		return fmt.Errorf("node %v already added", id)
+	}
+	self.nodeMap[id.NodeID] = len(self.Nodes)
+	na := self.naf(conf)
+	node := &Node{
+		Id:     conf.Id,
+		config: conf,
+		na:     na,
+	}
+	self.Nodes = append(self.Nodes, node)
+	glog.V(6).Infof("node %v created", id)
+	return nil
+}
+
+// newConn adds a new connection to the network
+// it errors if the respective nodes do not exist
+func (self *Network) newConn(oneId, otherId *adapters.NodeId) (*Conn, error) {
+	one := self.getNode(oneId)
+	if one == nil {
+		return nil, fmt.Errorf("one %v does not exist", one)
+	}
+	other := self.getNode(otherId)
+	if other == nil {
+		return nil, fmt.Errorf("other %v does not exist", other)
+	}
+	return &Conn{
+		One:   oneId,
+		Other: otherId,
+		one:   one,
+		other: other,
+	}, nil
+}
+
+func (self *Conn) nodesUp() error {
+	if !self.one.Up {
+		return fmt.Errorf("one %v is not up", self.One)
+	}
+	if !self.other.Up {
+		return fmt.Errorf("other %v is not up", self.Other)
+	}
+	return nil
+}
+
+// sa := node.Adapter()
+// err := sa.Stop()
+// if err != nil {
+// 	return err
+// }
+
+// Start(id) starts up the node (relevant only for instance with own p2p or remote)
+func (self *Network) Start(id *adapters.NodeId) error {
+	node := self.GetNode(id)
+	if node == nil {
+		return fmt.Errorf("node %v does not exist", id)
+	}
+	if node.Up {
+		return fmt.Errorf("node %v already up", id)
+	}
+	glog.V(6).Infof("starting node %v: %v adapter %v", id, node.Up, node.Adapter())
+	sa, ok := node.Adapter().(adapters.StartAdapter)
+	if ok {
+		err := sa.Start()
+		if err != nil {
+			return err
+		}
+	}
+	node.Up = true
+	glog.V(6).Infof("started node %v: %v", id, node.Up)
+
+	self.events.Post(&NodeEvent{
+		Action: "up",
+		Type:   "node",
+		node:   node,
+	})
+	return nil
+}
+
+// Stop(id) shuts down the node (relevant only for instance with own p2p or remote)
+func (self *Network) Stop(id *adapters.NodeId) error {
+	node := self.GetNode(id)
+	if node == nil {
+		return fmt.Errorf("node %v does not exist", id)
+	}
+	if !node.Up {
+		return fmt.Errorf("node %v already down", id)
+	}
+	sa, ok := node.Adapter().(adapters.StartAdapter)
+	if ok {
+		err := sa.Stop()
+		if err != nil {
+			return err
+		}
+	}
+	node.Up = false
+	self.events.Post(&NodeEvent{
+		Action: "down",
+		Type:   "node",
+		node:   node,
+	})
+	return nil
+}
+
+// Connect(i, j) attempts to connect nodes i and j (args given as nodeId)
+// calling the node's nodadapters Connect method
+// connection is established (as if) the first node dials out to the other
+func (self *Network) Connect(oneId, otherId *adapters.NodeId) error {
+	conn, err := self.GetOrCreateConn(oneId, otherId)
+	if err != nil {
+		return err
+	}
+	if conn.Up {
+		return fmt.Errorf("%v and %v already connected", oneId, otherId)
+	}
+	err = conn.nodesUp()
+	if err != nil {
+		return err
+	}
+	var rev bool
+	if conn.One.NodeID != oneId.NodeID {
+		rev = true
+	}
+	// if Connect is called because of external trigger, it needs to call
+	// the actual adaptor's connect method
+	// any other way of connection (like peerpool) will need to call back
+	// to this method with connect = false to avoid infinite recursion
+	// this is not relevant for nodes starting up (which can only be externally triggered)
+	if rev {
+		err = conn.other.na.Connect(oneId.Bytes())
+	} else {
+		err = conn.one.na.Connect(otherId.Bytes())
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+	// return self.DidConnect(oneId, otherId)
+}
+
+// Disconnect(i, j) attempts to disconnect nodes i and j (args given as nodeId)
+// calling the node's nodadapters Disconnect method
+// sets the Conn model to Down
+// the disconnect will be initiated (the connection is dropped by) the first node
+// it errors if either of the nodes is down (or does not exist)
+func (self *Network) Disconnect(oneId, otherId *adapters.NodeId, disconnect bool) error {
+	conn := self.GetConn(oneId, otherId)
+	if conn == nil {
+		return fmt.Errorf("connection between %v and %v does not exist", oneId, otherId)
+	}
+	if !conn.Up {
+		return fmt.Errorf("%v and %v already disconnected", oneId, otherId)
+	}
+	var rev bool
+	if conn.One.NodeID != oneId.NodeID {
+		rev = true
+	}
+	// if Disconnect is externally triggered one needs to call the actual
+	// adapter's disconnect method
+	if disconnect {
+		var err error
+		if rev {
+			err = conn.other.na.Disconnect(oneId.Bytes())
+		} else {
+			err = conn.one.na.Disconnect(otherId.Bytes())
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+	// return self.DidDisconnect(oneId, otherId)
+}
+
+func (self *Network) DidConnect(one, other *adapters.NodeId) error {
+	conn := self.GetConn(one, other)
+	if conn == nil {
+		return fmt.Errorf("connection between %v and %v does not exist", one, other)
+	}
+	if conn.Up {
+		return fmt.Errorf("%v and %v already connected", one, other)
+	}
+	conn.Reverse = conn.One.NodeID != one.NodeID
+	conn.Up = true
+	// connection event posted
+	self.events.Post(conn.event(true, conn.Reverse))
+	return nil
+}
+
+func (self *Network) DidDisconnect(one, other *adapters.NodeId) error {
+	conn := self.GetConn(one, other)
+	if conn == nil {
+		return fmt.Errorf("connection between %v and %v does not exist", one, other)
+	}
+	if !conn.Up {
+		return fmt.Errorf("%v and %v already disconnected", one, other)
+	}
+	conn.Reverse = conn.One.NodeID != one.NodeID
+	conn.Up = false
+	self.events.Post(conn.event(false, conn.Reverse))
+	return nil
+}
+
+// GetNodeAdapter(id) returns the NodeAdapter for node with id
+// returns nil if node does not exist
+func (self *Network) GetNodeAdapter(id *adapters.NodeId) adapters.NodeAdapter {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	node := self.getNode(id)
+	if node == nil {
+		return nil
+	}
+	return node.na
+}
+
+// GetNode retrieves the node model for the id given as arg
+// returns nil if the node does not exist
+func (self *Network) GetNode(id *adapters.NodeId) *Node {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.getNode(id)
+}
+
+func (self *Network) getNode(id *adapters.NodeId) *Node {
+	i, found := self.nodeMap[id.NodeID]
+	if !found {
+		return nil
+	}
+	return self.Nodes[i]
+}
+
+// GetConn(i, j) retrieves the connectiton model for the connection between
+// the order of nodes does not matter, i.e., GetConn(i,j) == GetConn(j, i)
+// returns nil if the node does not exist
+func (self *Network) GetConn(oneId, otherId *adapters.NodeId) *Conn {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.getConn(oneId, otherId)
+}
+
+// GetConn(i, j) retrieves the connectiton model for the connection between
+// i and j, or creates a new one if it does not exist
+// the order of nodes does not matter, i.e., GetConn(i,j) == GetConn(j, i)
+func (self *Network) GetOrCreateConn(oneId, otherId *adapters.NodeId) (*Conn, error) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	conn := self.getConn(oneId, otherId)
+	if conn != nil {
+		return conn, nil
+	}
+	conn, err := self.newConn(oneId, otherId)
+	if err != nil {
+		return nil, err
+	}
+	label := ConnLabel(oneId, otherId)
+	self.connMap[label] = len(self.Conns)
+	self.Conns = append(self.Conns, conn)
+	return conn, nil
+}
+
+func (self *Network) getConn(oneId, otherId *adapters.NodeId) *Conn {
+	label := ConnLabel(oneId, otherId)
+	i, found := self.connMap[label]
+	if !found {
+		return nil
+	}
+	return self.Conns[i]
+}

p2p/simulations/rest_api_server.go

@@ -0,0 +1,66 @@
+package simulations
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+type Controller interface {
+	Resource(id string) (Controller, error)
+	Handle(method string) (returnHandler, error)
+	SetResource(id string, c Controller)
+}
+
+// starts up http server
+func StartRestApiServer(port string, c Controller) {
+	serveMux := http.NewServeMux()
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		handle(w, r, c)
+	})
+	fd, err := net.Listen("tcp", ":"+port)
+	if err != nil {
+		glog.Errorf("Can't listen on :%s: %v", port, err)
+		return
+	}
+	go http.Serve(fd, serveMux)
+	glog.V(logger.Info).Infof("Swarm Network Controller HTTP server started on localhost:%s", port)
+}
+
+func handle(w http.ResponseWriter, r *http.Request, c Controller) {
+	requestURL := r.URL
+	glog.V(logger.Debug).Infof("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept"))
+	uri := requestURL.Path
+	w.Header().Set("Content-Type", "text/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	defer r.Body.Close()
+	parts := strings.Split(uri, "/")
+	var err error
+	for _, id := range parts {
+		if len(id) == 0 {
+			continue
+		}
+		c, err = c.Resource(id)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("resource %v not found", id), http.StatusNotFound)
+			return
+		}
+	}
+	handler, err := c.Handle(r.Method)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("method %v not allowed (%v)", r.Method, err), http.StatusMethodNotAllowed)
+		return
+	}
+	// on return we close the request Body so we assume it is read synchronously
+	response, err := handler(r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("handler error: %v", err), http.StatusBadRequest)
+		return
+	}
+	http.ServeContent(w, r, "", time.Now(), response)
+}

p2p/simulations/rest_api_server_test.go

@@ -0,0 +1,137 @@
+package simulations
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+)
+
+const testPort = "8889"
+
+type testController struct {
+}
+
+func (self *testController) SetResource(id string, c Controller) {
+}
+
+func (self *testController) Resource(id string) (Controller, error) {
+	if id == "missing" {
+		return nil, fmt.Errorf("missing")
+	}
+	return Controller(self), nil
+}
+
+func (self *testController) Handle(method string) (returnHandler, error) {
+	switch method {
+	case "POST":
+	case "DELETE":
+	default:
+		return nil, fmt.Errorf("allowed methods: POST DELETE")
+	}
+	return handlerf(method), nil
+}
+
+func handlerf(method string) returnHandler {
+	return func(r io.Reader) (io.ReadSeeker, error) {
+		body, err := ioutil.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+		if string(body) == "invalid" {
+			return nil, fmt.Errorf("invalid body")
+		}
+		return io.ReadSeeker(bytes.NewReader([]byte("response"))), nil
+	}
+}
+
+func init() {
+	StartRestApiServer(testPort, &testController{})
+}
+
+type testRequest struct {
+	method   string
+	path     string
+	body     string
+	response string
+	status   int
+}
+
+type ReadCloser struct {
+	io.Reader
+}
+
+func (ReadCloser) Close() {}
+
+func testResponses(t *testing.T, reqs ...*testRequest) {
+	for _, req := range reqs {
+		path := url(testPort, req.path)
+		var r *http.Response
+		var err error
+		switch req.method {
+		case "POST":
+			r, err = http.Post(path, "text/json", ReadCloser{bytes.NewReader([]byte(req.body))})
+		default:
+			r, err = http.Get(path)
+		}
+		if err != nil {
+			t.Fatalf("unexpected error on request: %v", err)
+		}
+		if r.StatusCode != req.status {
+			t.Fatalf("unexpected status on request: got %v, expected %v", r.StatusCode, req.status)
+		}
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("unexpected error on reading body: %v", err)
+		}
+		if string(body) != req.response {
+			t.Fatalf("unexpected response body. got '%s', expected '%v'", body, req.response)
+		}
+	}
+}
+
+func TestServerMethodNotAllowed(t *testing.T) {
+	testResponses(t,
+		&testRequest{
+			"GET",
+			"anypath",
+			"anybody",
+			"method GET not allowed (allowed methods: POST DELETE)\n",
+			http.StatusMethodNotAllowed,
+		})
+}
+
+func TestServerInvalid(t *testing.T) {
+	testResponses(t,
+		&testRequest{
+			"POST",
+			"anypath",
+			"invalid",
+			"handler error: invalid body\n",
+			http.StatusBadRequest,
+		})
+}
+
+func TestServerResourceNotFound(t *testing.T) {
+	testResponses(t,
+		&testRequest{
+			"POST",
+			"missing",
+			"anybody",
+			"resource missing not found\n",
+			http.StatusNotFound,
+		})
+}
+
+func TestServerSuccess(t *testing.T) {
+	testResponses(t,
+		&testRequest{
+			"POST",
+			"anypath",
+			"anybody",
+			"response",
+			http.StatusOK,
+		})
+}

p2p/simulations/session_controller.go

@@ -0,0 +1,177 @@
+package simulations
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"reflect"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+)
+
+type returnHandler func(body io.Reader) (resp io.ReadSeeker, err error)
+
+type ResourceHandler struct {
+	Handle func(interface{}, *ResourceController) (interface{}, error)
+	Type   reflect.Type
+}
+
+type ResourceHandlers struct {
+	Create, Retrieve, Update, Destroy *ResourceHandler
+}
+
+type ResourceController struct {
+	lock        sync.Mutex
+	controllers map[string]Controller
+	id          int
+	methods     []string
+	*ResourceHandlers
+}
+
+var methodsAvailable = []string{"POST", "GET", "PUT", "DELETE"}
+
+func (self *ResourceHandlers) handler(method string) *ResourceHandler {
+	var h *ResourceHandler
+	switch method {
+	case "POST":
+		h = self.Create
+	case "GET":
+		h = self.Retrieve
+	case "PUT":
+		h = self.Update
+	case "DELETE":
+		h = self.Destroy
+	}
+	return h
+}
+
+func NewResourceContoller(c *ResourceHandlers) *ResourceController {
+	var methods []string
+	for _, method := range methodsAvailable {
+		if c.handler(method) != nil {
+			methods = append(methods, method)
+		}
+	}
+	return &ResourceController{
+		ResourceHandlers: c,
+		controllers:      make(map[string]Controller),
+		methods:          methods,
+	}
+}
+
+var empty = struct{}{}
+
+func NewSessionController() (*ResourceController, chan bool) {
+	quitc := make(chan bool)
+	return NewResourceContoller(
+		&ResourceHandlers{
+
+			Create: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					conf := msg.(*NetworkConfig)
+					m := NewNetworkController(conf, &event.TypeMux{}, NewJournal())
+					if len(conf.Id) == 0 {
+						conf.Id = fmt.Sprintf("%d", parent.id)
+					}
+					glog.V(6).Infof("new network controller on %v", conf.Id)
+					if parent != nil {
+						parent.SetResource(conf.Id, m)
+					}
+					parent.id++
+					return empty, nil
+				},
+				Type: reflect.TypeOf(&NetworkConfig{}),
+			},
+
+			Destroy: &ResourceHandler{
+				Handle: func(msg interface{}, parent *ResourceController) (interface{}, error) {
+					glog.V(6).Infof("destroy handler called")
+					// this can quit the entire app (shut down the backend server)
+					quitc <- true
+					return empty, nil
+				},
+			},
+		},
+	), quitc
+}
+
+func (self *ResourceController) Handle(method string) (returnHandler, error) {
+	h := self.handler(method)
+	if h == nil {
+		return nil, fmt.Errorf("allowed methods: %v", self.methods)
+	}
+	rh := func(r io.Reader) (io.ReadSeeker, error) {
+		input, err := ioutil.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+		var arg interface{}
+		if len(input) == 0 {
+			input = []byte("{}")
+		}
+		if h.Type != nil {
+			val := reflect.New(h.Type)
+			req := val.Elem()
+			req.Set(reflect.Zero(h.Type))
+			err = json.Unmarshal(input, val.Interface())
+			if err != nil {
+				return nil, err
+			}
+			arg = req.Interface()
+		}
+		res, err := h.Handle(arg, self)
+		if err != nil {
+			return nil, err
+		}
+		resp, err := json.MarshalIndent(res, "", "  ")
+		return bytes.NewReader(resp), nil
+	}
+	return rh, nil
+}
+
+func (self *ResourceController) Resource(id string) (Controller, error) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	c, ok := self.controllers[id]
+	if !ok {
+		return nil, fmt.Errorf("not found")
+	}
+	return c, nil
+}
+
+func (self *ResourceController) SetResource(id string, c Controller) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	if c == nil {
+		delete(self.controllers, id)
+	} else {
+		self.controllers[id] = c
+	}
+}
+
+func (self *ResourceController) DeleteResource(id string) {
+	delete(self.controllers, id)
+}
+
+func RandomNodeId() *adapters.NodeId {
+	key, err := crypto.GenerateKey()
+	if err != nil {
+		panic("unable to generate key")
+	}
+	pubkey := crypto.FromECDSAPub(&key.PublicKey)
+	return adapters.NewNodeId(pubkey[1:])
+}
+
+func RandomNodeIds(n int) []*adapters.NodeId {
+	var ids []*adapters.NodeId
+	for i := 0; i < n; i++ {
+		ids = append(ids, RandomNodeId())
+	}
+	return ids
+}

p2p/simulations/session_controller_test.go

@@ -0,0 +1,125 @@
+package simulations
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+)
+
+const (
+	domain = "http://localhost"
+	port   = "8888"
+)
+
+var quitc chan bool
+var controller *ResourceController
+
+func init() {
+	glog.SetV(6)
+	glog.SetToStderr(true)
+	controller, quitc = NewSessionController()
+	StartRestApiServer(port, controller)
+}
+
+func url(port, path string) string {
+	return fmt.Sprintf("%v:%v/%v", domain, port, path)
+}
+
+func TestDelete(t *testing.T) {
+	req, err := http.NewRequest("DELETE", url(port, ""), nil)
+	if err != nil {
+		t.Fatalf("unexpected error")
+	}
+	var resp *http.Response
+	go func() {
+		r, err := (&http.Client{}).Do(req)
+		if err != nil {
+			t.Fatalf("unexpected error")
+		}
+		resp = r
+	}()
+	timeout := time.NewTimer(1000 * time.Millisecond)
+	select {
+	case <-quitc:
+	case <-timeout.C:
+		t.Fatalf("timed out: controller did not quit, response: %v", resp)
+	}
+}
+
+func TestCreate(t *testing.T) {
+	// should test that session controller POST creates network controller
+	// with proper endpoints
+}
+
+func testResponse(t *testing.T, method, addr string, r io.ReadSeeker) []byte {
+
+	req, err := http.NewRequest(method, addr, r)
+	if err != nil {
+		t.Fatalf("unexpected error creating request: %v", err)
+	}
+	resp, err := (&http.Client{}).Do(req)
+	if err != nil {
+		t.Fatalf("unexpected error on http.Client request: %v", err)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("error reading response body: %v", err)
+	}
+	return body
+
+}
+
+func TestUpdate(t *testing.T) {
+
+	ids := testIDs()
+	journal := testJournal(ids)
+
+	conf := &NetworkConfig{
+		Id: "0",
+	}
+	mc := NewNetworkController(conf, &event.TypeMux{}, journal)
+	controller.SetResource(conf.Id, mc)
+	exp := `{
+  "add": [
+    {
+      "data": {
+        "id": "aa7c",
+        "up": true
+      },
+      "group": "nodes"
+    },
+    {
+      "data": {
+        "id": "f5ae",
+        "up": true
+      },
+      "group": "nodes"
+    }
+  ],
+  "remove": []
+}`
+	resp := testResponse(t, "GET", url(port, "0"), bytes.NewReader([]byte("{}")))
+	if string(resp) != exp {
+		t.Fatalf("incorrect response body. got\n'%v', expected\n'%v'", string(resp), exp)
+	}
+}
+
+func mockNewNodes(eventer *event.TypeMux, ids []*adapters.NodeId) {
+	glog.V(6).Infof("mock starting")
+	for _, id := range ids {
+		glog.V(6).Infof("mock adding node %v", id)
+		eventer.Post(&NodeEvent{
+			Action: "up",
+			Type:   "node",
+			node:   &Node{Id: id, config: &NodeConfig{Id: id}},
+		})
+	}
+}

p2p/simulations/testjournal.json

@@ -0,0 +1,382 @@
+{
+ "Id": "test",
+ "Events": [
+  {
+   "Time": "2016-11-14T17:05:21.017272978+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017277407+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.01739284+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "cf18aa5e915be9168fef111b36e05bec4e0515391f2d33d89550624252b2c5444631c89f36c63488cc7ba5541893cfc16762bdb47b351105f84aa835794dd92e"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017394012+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017412873+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017413866+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "cd92c56bb1f94e54edea6cb86a772fa2802df2fd9ebeaac111a2ca8ef886946b7dfcdc641cdbdaeec4648c2e81efaf51a3c730e1fc4cccdb68d4779bdb25b22e"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017446013+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "id": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017447485+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017482324+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017483222+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017503037+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017503789+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017524508+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "cf18aa5e915be9168fef111b36e05bec4e0515391f2d33d89550624252b2c5444631c89f36c63488cc7ba5541893cfc16762bdb47b351105f84aa835794dd92e"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017525749+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017554401+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017555747+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.01757843+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017581922+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017605589+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017607054+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.01762861+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "cf18aa5e915be9168fef111b36e05bec4e0515391f2d33d89550624252b2c5444631c89f36c63488cc7ba5541893cfc16762bdb47b351105f84aa835794dd92e",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017629798+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "cd92c56bb1f94e54edea6cb86a772fa2802df2fd9ebeaac111a2ca8ef886946b7dfcdc641cdbdaeec4648c2e81efaf51a3c730e1fc4cccdb68d4779bdb25b22e",
+     "caller": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017654435+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017655608+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "cd92c56bb1f94e54edea6cb86a772fa2802df2fd9ebeaac111a2ca8ef886946b7dfcdc641cdbdaeec4648c2e81efaf51a3c730e1fc4cccdb68d4779bdb25b22e",
+     "caller": "5788512da4bccc5a970e46c7cacd2462d85fbe6c9ef4922e876a5de0ad9da0c829787cd773d4ad50cf80521a0a774eb60b026a2fbfaea6dd56b96df62153eeb2"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017677473+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017678738+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "cf18aa5e915be9168fef111b36e05bec4e0515391f2d33d89550624252b2c5444631c89f36c63488cc7ba5541893cfc16762bdb47b351105f84aa835794dd92e"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.017709771+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.018258493+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "cd92c56bb1f94e54edea6cb86a772fa2802df2fd9ebeaac111a2ca8ef886946b7dfcdc641cdbdaeec4648c2e81efaf51a3c730e1fc4cccdb68d4779bdb25b22e"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.021255904+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.021259149+02:00",
+   "Data": {
+    "action": "down",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.023276075+02:00",
+   "Data": {
+    "action": "down",
+    "object": {
+     "callee": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.024257855+02:00",
+   "Data": {
+    "action": "down",
+    "object": {
+     "id": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd"
+    },
+    "type": "node"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.024264477+02:00",
+   "Data": {
+    "action": "up",
+    "object": {
+     "callee": "f27f617ed3e0c9fc5171c0dd83232aea074979ad502144932f5df70d3fed93c57df1e10cf5a9ac6407faead2ecbdf1e60327c3b022f3f79fd423ffbb0b006d25",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.024308791+02:00",
+   "Data": {
+    "action": "down",
+    "object": {
+     "callee": "efcfefc81f10907f701b45a7c3f39cb07a1588a45381878ac6a97f0a91c965bf85aa31465a258a040893298ab0cf6fa7f3fad7e1d9f8f194c9e00615faffc0cd",
+     "caller": "dc86df873a9633a915c10634b1558074aded4e58454d51e05f77cbaec5afdbd093758f95466eef0c0fe2a089a172ec16edac1394cabce59c69dc09fb03df6573"
+    },
+    "type": "conn"
+   }
+  },
+  {
+   "Time": "2016-11-14T17:05:21.025253254+02:00",
+   "Data": {
+    "action": "down",
+    "object": {
+     "callee": "cf18aa5e915be9168fef111b36e05bec4e0515391f2d33d89550624252b2c5444631c89f36c63488cc7ba5541893cfc16762bdb47b351105f84aa835794dd92e",
+     "caller": "6a4c012f01eb61456609f06f607d5eaa2d56e7598dc168a06f14ca5a941f1850a94a63732bb7cd4508e4cf8f96c61cfe778d44b769c40aaadb575b1144c391f0"
+    },
+    "type": "conn"
+   }
+  }
+ ]
+}

p2p/testing/exchange.go

@@ -0,0 +1,301 @@
+// Package protocols helpers_test make it easier to
+// write protocol tests by providing convenience functions and structures
+// protocols uses these helpers for  its own tests
+// but ideally should sit in p2p/protocols/testing/ subpackage
+package testing
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+)
+
+// ExchangeTestSession assumes a network with a protocol running on multiple peer connection
+// and is used to test scanarios of message exchange among a select array of nodes
+// the scenarios are sets of exchanges, each with a trigger and an expectation
+// This rigid regime is suitable for
+// * unit testing protocol message exchanges (nodes are peers of a local node)
+// * testing routed messaging between remote non-connected nodes within a group
+type ExchangeTestSession struct {
+	lock sync.Mutex
+	Ids  []*adapters.NodeId
+	TestNetAdapter
+	TestMessenger
+	t *testing.T
+}
+
+// implemented by simulations/
+type TestNetAdapter interface {
+	GetPeer(id *adapters.NodeId) *adapters.Peer
+}
+
+type TestMessenger interface {
+	// MsgPipe([]byte, []byte) p2p,MsgPipe
+	ExpectMsg(p2p.MsgReader, uint64, interface{}) error
+	TriggerMsg(p2p.MsgWriter, uint64, interface{}) error
+}
+
+// exchanges are the basic units of protocol tests
+// an exchange is defined on a session
+type Exchange struct {
+	Triggers []Trigger
+	Expects  []Expect
+}
+
+// part of the exchange, incoming message from a set of peers
+type Trigger struct {
+	Msg     interface{}      // type of message to be sent
+	Code    uint64           // code of message is given
+	Peer    *adapters.NodeId // the peer to send the message to
+	Timeout time.Duration    // timeout duration for the sending
+}
+
+type Expect struct {
+	Msg     interface{}      // type of message to expect
+	Code    uint64           // code of message is now given
+	Peer    *adapters.NodeId // the peer that expects the message
+	Timeout time.Duration    // timeout duration for receiving
+}
+
+type Disconnect struct {
+	Peer  *adapters.NodeId // the peer that expects the message
+	Error error
+}
+
+// NewExchangeTestSession takes a network session and Messenger
+// and returns an exchange session test driver that can
+// be used to unit test protocol communications
+// it allows for resource-driven scenario testing
+// disconnect reason errors are written in session.Errs
+// (correcponding to session.Peers)
+func NewExchangeTestSession(t *testing.T, n TestNetAdapter, m TestMessenger, ids []*adapters.NodeId) *ExchangeTestSession {
+	return &ExchangeTestSession{
+		Ids:            ids,
+		TestNetAdapter: n,
+		TestMessenger:  m,
+		t:              t,
+	}
+}
+
+type TestPeerInfo struct {
+	RW     p2p.MsgReadWriter
+	Flushc chan bool
+	Errc   chan error
+}
+
+// trigger sends messages from peers
+func (self *ExchangeTestSession) trigger(trig Trigger) error {
+	peer := self.GetPeer(trig.Peer)
+	if peer == nil {
+		panic(fmt.Sprintf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.Ids)))
+	}
+	rw := peer.RW
+	if rw == nil {
+		return fmt.Errorf("trigger: peer %v unreachable", trig.Peer)
+	}
+	errc := make(chan error)
+
+	go func() {
+		glog.V(6).Infof("trigger....")
+		errc <- self.TriggerMsg(rw, trig.Code, trig.Msg)
+		glog.V(6).Infof("triggered")
+	}()
+
+	t := trig.Timeout
+	if t == time.Duration(0) {
+		t = 1000 * time.Millisecond
+	}
+	alarm := time.NewTimer(t)
+	select {
+	case err := <-errc:
+		return err
+	case <-alarm.C:
+		return fmt.Errorf("timout expecting %v to send to peer %v", trig.Msg, trig.Peer)
+	}
+}
+
+func Key(id []byte) string {
+	return string(id)
+}
+
+// expect checks an expectation
+func (self *ExchangeTestSession) expect(exp Expect) error {
+	if exp.Msg == nil {
+		panic("no message to expect")
+	}
+	peer := self.GetPeer(exp.Peer)
+	if peer == nil {
+		panic(fmt.Sprintf("expect: peer %v does not exist (1- %v)", exp.Peer, len(self.Ids)))
+	}
+	rw := peer.RW
+	if rw == nil {
+		return fmt.Errorf("trigger: peer %v unreachable", exp.Peer)
+	}
+
+	errc := make(chan error)
+	go func() {
+		glog.V(6).Infof("waiting for msg, %v", exp.Msg)
+		errc <- self.ExpectMsg(rw, exp.Code, exp.Msg)
+	}()
+
+	t := exp.Timeout
+	if t == time.Duration(0) {
+		t = 1000 * time.Millisecond
+	}
+	alarm := time.NewTimer(t)
+	select {
+	case err := <-errc:
+		glog.V(6).Infof("expected msg arrives with error %v", err)
+		return err
+	case <-alarm.C:
+		glog.V(6).Infof("caught timeout")
+		return fmt.Errorf("timout expecting %v sent to peer %v", exp.Msg, exp.Peer)
+	}
+	// fatal upon encountering first exchange error
+}
+
+// TestExchange tests a series of exchanges againsts the session
+func (self *ExchangeTestSession) TestExchanges(exchanges ...Exchange) {
+	// launch all triggers of this exchanges
+
+	for i, e := range exchanges {
+		errc := make(chan error)
+		wg := &sync.WaitGroup{}
+		for _, trig := range e.Triggers {
+			wg.Add(1)
+			// separate go routing to allow parallel requests
+			go func(t Trigger) {
+				defer wg.Done()
+				err := self.trigger(t)
+				if err != nil {
+					errc <- err
+				}
+			}(trig)
+		}
+
+		// each expectation is spawned in separate go-routine
+		// expectations of an exchange are conjunctive but uordered, i.e., only all of them arriving constitutes a pass
+		// each expectation is meant to be for a different peer, otherwise they are expected to panic
+		// testing of an exchange blocks until all expectations are decided
+		// an expectation is decided if
+		//  expected message arrives OR
+		// an unexpected message arrives (panic)
+		// times out on their individual tiemeout
+		for _, ex := range e.Expects {
+			wg.Add(1)
+			// expect msg spawned to separate go routine
+			go func(exp Expect) {
+				defer wg.Done()
+				err := self.expect(exp)
+				if err != nil {
+					glog.V(6).Infof("expect msg fails %v", err)
+					errc <- err
+				}
+			}(ex)
+		}
+
+		// wait for all expectations
+		go func() {
+			wg.Wait()
+			close(errc)
+		}()
+
+		// time out globally or finish when all expectations satisfied
+		alarm := time.NewTimer(1000 * time.Millisecond)
+		select {
+
+		case err := <-errc:
+			if err != nil {
+				self.t.Fatalf("exchange failed with: %v", err)
+			} else {
+				glog.V(6).Infof("exchange %v run successfully", i)
+			}
+		case <-alarm.C:
+			self.t.Fatalf("exchange timed out")
+		}
+	}
+}
+
+type flushMsg struct{}
+
+func flushExchange(c int, ids ...*adapters.NodeId) Exchange {
+	var triggers []Trigger
+	for _, id := range ids {
+		triggers = append(triggers,
+			Trigger{
+				Code: uint64(c),
+				Msg:  &flushMsg{},
+				Peer: id,
+			})
+	}
+	return Exchange{
+		Triggers: triggers,
+	}
+}
+
+var FlushMsg = &flushMsg{}
+
+func (self *ExchangeTestSession) TestConnected(flush bool, peers ...*adapters.NodeId) {
+	timeout := time.NewTimer(1000 * time.Millisecond)
+	var flushc chan bool
+	if !flush {
+		flushc = make(chan bool)
+		close(flushc)
+	}
+	wg := &sync.WaitGroup{}
+	wg.Add(len(peers))
+	for _, id := range peers {
+		ticker := time.NewTicker(100 * time.Millisecond)
+		go func(p *adapters.NodeId) {
+			defer wg.Done()
+			for {
+				peer := self.GetPeer(p)
+				if peer != nil {
+					if flush {
+						flushc = peer.Flushc
+					}
+					select {
+					case <-timeout.C:
+						self.t.Fatalf("exchange timed out waiting for peer %v to flush", p)
+					case err := <-peer.Errc:
+						self.t.Fatalf("peer %v disconnected with error %v", p, err)
+					case <-flushc:
+						glog.V(6).Infof("peer %v is connected", p)
+						return
+					}
+				}
+				select {
+				case <-ticker.C:
+					glog.V(6).Infof("waiting for %v to connect", p)
+				case <-timeout.C:
+					self.t.Fatalf("exchange timed out waiting for peer %v to connect", p)
+				}
+			}
+		}(id)
+	}
+	wg.Wait()
+	glog.V(6).Infof("checking complete")
+
+}
+
+func (self *ExchangeTestSession) TestDisconnected(disconnects ...*Disconnect) {
+	for _, disconnect := range disconnects {
+		id := disconnect.Peer
+		err := disconnect.Error
+		errc := self.GetPeer(id).Errc
+		alarm := time.NewTimer(1000 * time.Millisecond)
+		select {
+		case derr := <-errc:
+			if !((err == nil && derr == nil) || err != nil && derr != nil && err.Error() == derr.Error()) {
+				self.t.Fatalf("unexpected error on peer %v: '%v', wanted '%v'", id, derr, err)
+			}
+		case <-alarm.C:
+			self.t.Fatalf("exchange timed out waiting for peer %v to disconnect", id)
+		}
+	}
+}

p2p/testing/peerpool.go

@@ -0,0 +1,48 @@
+package testing
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+type TestPeer interface {
+	ID() discover.NodeID
+	Drop()
+}
+
+// TestPeerPool is an example peerPool to demonstrate registration of peer connections
+type TestPeerPool struct {
+	lock  sync.Mutex
+	peers map[discover.NodeID]TestPeer
+}
+
+func NewTestPeerPool() *TestPeerPool {
+	return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)}
+}
+
+func (self *TestPeerPool) Add(p TestPeer) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.peers[p.ID()] = p
+}
+
+func (self *TestPeerPool) Remove(p TestPeer) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	delete(self.peers, p.ID())
+}
+
+func (self *TestPeerPool) Has(n *adapters.NodeId) bool {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	_, ok := self.peers[n.NodeID]
+	return ok
+}
+
+func (self *TestPeerPool) Get(n *adapters.NodeId) TestPeer {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.peers[n.NodeID]
+}

p2p/testing/sessions.go

@@ -0,0 +1,122 @@
+package testing
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+)
+
+type PeerAdapter interface {
+	adapters.NodeAdapter
+	TestMessenger
+	TestNetAdapter
+}
+
+type ExchangeSession struct {
+	network *simulations.Network
+	na      adapters.NodeAdapter
+	*ExchangeTestSession
+}
+
+// NewProtocolTester returns an exchange test session
+// this is a resource driver for protocol message exchange
+// scenarios expressed as expects and triggers
+// see p2p/protocols/exhange_test.go for an example
+// this is used primarily to unit test protocols or protocol modules
+// correct message exchange, forwarding, and broadcast
+// higher level or network behaviour should be tested with network simulators
+func NewProtocolTester(t *testing.T, id *adapters.NodeId, n int, run func(id adapters.NodeAdapter) adapters.ProtoCall) *ExchangeSession {
+	simPipe := &adapters.SimPipe{}
+	network := simulations.NewNetwork(nil, nil)
+	naf := func(conf *simulations.NodeConfig) adapters.NodeAdapter {
+		na := adapters.NewSimNode(conf.Id, network, simPipe)
+		if conf.Id.NodeID == id.NodeID {
+			glog.V(6).Infof("adapter run function set to protocol for node %v (=%v)", conf.Id, id)
+			na.Run = run(na)
+		}
+		return na
+	}
+	network.SetNaf(naf)
+	// setup a simulated network of n nodes
+	// Startup pivot node
+	err := network.NewNode(&simulations.NodeConfig{Id: id})
+	if err != nil {
+		panic(err.Error())
+	}
+	glog.V(6).Infof("network created")
+	na := network.GetNode(id).Adapter()
+	s := NewExchangeTestSession(t, na.(TestNetAdapter), na.Messenger().(TestMessenger), nil)
+	self := &ExchangeSession{
+		network:             network,
+		na:                  na,
+		ExchangeTestSession: s,
+	}
+	ids := RandomNodeIds(n)
+	self.Connect(ids...)
+	// Start up connections to virual nodes serving as endpoints for sending/receiving messages for peers
+	return self
+}
+
+func (self *ExchangeTestSession) Flush(code int, ids ...*adapters.NodeId) {
+	self.TestConnected(false, ids...)
+	glog.V(6).Infof("flushing peers %v (code %v)", ids, code)
+	self.TestExchanges(flushExchange(code, ids...))
+	self.TestConnected(true, ids...)
+}
+
+func (self *ExchangeSession) Start(id *adapters.NodeId) error {
+	err := self.network.NewNode(&simulations.NodeConfig{Id: id})
+	if err != nil {
+		return err
+	}
+	node := self.network.GetNode(id)
+	if node == nil {
+		glog.V(6).Infof("node for peer %v not found", id)
+		return nil
+	}
+	if node.Adapter() == nil {
+		glog.V(6).Infof("node adapter for peer %v not found", id)
+		return nil
+	}
+	self.Ids = append(self.Ids, id)
+	return nil
+}
+
+func (self *ExchangeSession) Connect(ids ...*adapters.NodeId) {
+	for _, id := range ids {
+		glog.V(6).Infof("start node %v", id)
+		err := self.Start(id)
+		if err != nil {
+			glog.V(6).Infof("error starting peer %v: %v", id, err)
+		}
+		glog.V(6).Infof("connect to %v", id)
+		err = self.na.Connect(id.Bytes())
+		if err != nil {
+			glog.V(6).Infof("error connecting to peer %v: %v", id, err)
+		}
+	}
+
+}
+
+func RandomNodeId() *adapters.NodeId {
+	key, err := crypto.GenerateKey()
+	if err != nil {
+		panic("unable to generate key")
+	}
+	var id discover.NodeID
+	pubkey := crypto.FromECDSAPub(&key.PublicKey)
+	copy(id[:], pubkey[1:])
+	return &adapters.NodeId{id}
+}
+
+func RandomNodeIds(n int) []*adapters.NodeId {
+	var ids []*adapters.NodeId
+	for i := 0; i < n; i++ {
+		ids = append(ids, RandomNodeId())
+	}
+	return ids
+}

swarm/network/address.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package kademlia
+package network
 
 import (
 	"fmt"
@@ -22,6 +22,12 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
+	// "github.com/ethereum/go-ethereum/logger/glog"
+)
+
+var (
+	zeroAddr = &common.Hash{}
+	zeros    = zeroAddr.Hex()[2:]
 )
 
 type Address common.Hash
@@ -63,16 +69,22 @@ binary representation of the x^y.
 
 (0 farthest, 255 closest, 256 self)
 */
-func proximity(one, other Address) (ret int) {
-	for i := 0; i < len(one); i++ {
+func proximity(one, other Address) (ret int, eq bool) {
+	return posProximity(one, other, 0)
+}
+
+// posProximity(a, b, pos) returns proximity order of b wrt a (symmetric) pretending
+// the first pos bits match, checking only bits at index >= pos
+func posProximity(one, other Address, pos int) (ret int, eq bool) {
+	for i := pos / 8; i < len(one); i++ {
 		oxo := one[i] ^ other[i]
-		for j := 0; j < 8; j++ {
+		for j := pos % 8; j < 8; j++ {
 			if (uint8(oxo)>>uint8(7-j))&0x01 != 0 {
-				return i*8 + j
+				return i*8 + j, false
 			}
 		}
 	}
-	return len(one) * 8
+	return len(one) * 8, true
 }
 
 // Address.ProxCmp compares the distances a->target and b->target.
@@ -96,7 +108,7 @@ func (target Address) ProxCmp(a, b Address) int {
 // if prox is negative a random address is generated
 func RandomAddressAt(self Address, prox int) (addr Address) {
 	addr = self
-	var pos int
+	pos := -1
 	if prox >= 0 {
 		pos = prox / 8
 		trans := prox % 8
@@ -118,18 +130,18 @@ func RandomAddressAt(self Address, prox int) (addr Address) {
 
 // KeyRange(a0, a1, proxLimit) returns the address inclusive address
 // range that contain addresses closer to one than other
-func KeyRange(one, other Address, proxLimit int) (start, stop Address) {
-	prox := proximity(one, other)
-	if prox >= proxLimit {
-		prox = proxLimit
-	}
-	start = CommonBitsAddrByte(one, other, byte(0x00), prox)
-	stop = CommonBitsAddrByte(one, other, byte(0xff), prox)
-	return
-}
+// func KeyRange(one, other Address, proxLimit int) (start, stop Address) {
+// 	prox := proximity(one, other)
+// 	if prox >= proxLimit {
+// 		prox = proxLimit
+// 	}
+// 	start = CommonBitsAddrByte(one, other, byte(0x00), prox)
+// 	stop = CommonBitsAddrByte(one, other, byte(0xff), prox)
+// 	return
+// }
 
 func CommonBitsAddrF(self, other Address, f func() byte, p int) (addr Address) {
-	prox := proximity(self, other)
+	prox, _ := proximity(self, other)
 	var pos int
 	if p <= prox {
 		prox = p
@@ -171,3 +183,57 @@ func CommonBitsAddrByte(self, other Address, b byte, prox int) (addr Address) {
 func RandomAddress() Address {
 	return RandomAddressAt(Address{}, -1)
 }
+
+// wraps an Address to implement the PbVal interface
+type PbAddress struct {
+	Address
+}
+
+// Prefix(addr, pos) return the proximity order of addr wrt to
+// the pinned address of the tree
+// assuming it is greater than or  equal to pos
+func (self *PbAddress) Prefix(val PbVal, pos int) (po int, eq bool) {
+	return posProximity(self.Address, val.(*PbAddress).Address, pos)
+}
+
+type BinAddr struct {
+	addr []bool
+}
+
+func NewBinAddr(s string) *BinAddr {
+	return NewBinAddrXOR(s, zeros[:len(s)])
+}
+
+func NewBinAddrXOR(s, t string) *BinAddr {
+	if len(s) != len(t) {
+		panic("lengths do not match")
+	}
+	addr := make([]bool, len(s))
+	for i, _ := range addr {
+		addr[i] = s[i] != t[i]
+	}
+	return &BinAddr{addr}
+}
+
+func (self *BinAddr) String() string {
+	a := self.addr
+	s := []byte(zeros)[:len(a)]
+	for i, one := range a {
+		if one {
+			s[i] = byte('1')
+			// glog.V(6).Infof("%v", s)
+		}
+	}
+	return string(s)
+}
+
+func (self *BinAddr) Prefix(val PbVal, pos int) (po int, eq bool) {
+	a := self.addr
+	b := val.(*BinAddr).addr
+	for po = pos; po < len(b); po++ {
+		if a[po] != b[po] {
+			return po, false
+		}
+	}
+	return po, true
+}

swarm/network/address_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package kademlia
+package network
 
 import (
 	"math/rand"
@@ -89,8 +89,9 @@ func TestRandomAddressAt(t *testing.T) {
 		a = RandomAddress()
 		prox := rand.Intn(255)
 		b := RandomAddressAt(a, prox)
-		if proximity(a, b) != prox {
-			t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, proximity(a, b), prox)
+		p, _ := proximity(a, b)
+		if p != prox {
+			t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, p, prox)
 		}
 	}
 }

swarm/network/depo.go

@@ -1,217 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-// Handler for storage/retrieval related protocol requests
-// implements the StorageHandler interface used by the bzz protocol
-type Depo struct {
-	hashfunc   storage.Hasher
-	localStore storage.ChunkStore
-	netStore   storage.ChunkStore
-}
-
-func NewDepo(hash storage.Hasher, localStore, remoteStore storage.ChunkStore) *Depo {
-	return &Depo{
-		hashfunc:   hash,
-		localStore: localStore,
-		netStore:   remoteStore, // entrypoint internal
-	}
-}
-
-// Handles UnsyncedKeysMsg after msg decoding - unsynced hashes upto sync state
-// * the remote sync state is just stored and handled in protocol
-// * filters through the new syncRequests and send the ones missing
-// * back immediately as a deliveryRequest message
-// * empty message just pings back for more (is this needed?)
-// * strict signed sync states may be needed.
-func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error {
-	unsynced := req.Unsynced
-	var missing []*syncRequest
-	var chunk *storage.Chunk
-	var err error
-	for _, req := range unsynced {
-		// skip keys that are found,
-		chunk, err = self.localStore.Get(storage.Key(req.Key[:]))
-		if err != nil || chunk.SData == nil {
-			missing = append(missing, req)
-		}
-	}
-	log.Debug(fmt.Sprintf("Depo.HandleUnsyncedKeysMsg: received %v unsynced keys: %v missing. new state: %v", len(unsynced), len(missing), req.State))
-	log.Trace(fmt.Sprintf("Depo.HandleUnsyncedKeysMsg: received %v", unsynced))
-	// send delivery request with missing keys
-	err = p.deliveryRequest(missing)
-	if err != nil {
-		return err
-	}
-	// set peers state to persist
-	p.syncState = req.State
-	return nil
-}
-
-// Handles deliveryRequestMsg
-// * serves actual chunks asked by the remote peer
-// by pushing to the delivery queue (sync db) of the correct priority
-// (remote peer is free to reprioritize)
-// * the message implies remote peer wants more, so trigger for
-// * new outgoing unsynced keys message is fired
-func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error {
-	deliver := req.Deliver
-	// queue the actual delivery of a chunk ()
-	log.Trace(fmt.Sprintf("Depo.HandleDeliveryRequestMsg: received %v delivery requests: %v", len(deliver), deliver))
-	for _, sreq := range deliver {
-		// TODO: look up in cache here or in deliveries
-		// priorities are taken from the message so the remote party can
-		// reprioritise to at their leisure
-		// r = self.pullCached(sreq.Key) // pulls and deletes from cache
-		Push(p, sreq.Key, sreq.Priority)
-	}
-
-	// sends it out as unsyncedKeysMsg
-	p.syncer.sendUnsyncedKeys()
-	return nil
-}
-
-// the entrypoint for store requests coming from the bzz wire protocol
-// if key found locally, return. otherwise
-// remote is untrusted, so hash is verified and chunk passed on to NetStore
-func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
-	var islocal bool
-	req.from = p
-	chunk, err := self.localStore.Get(req.Key)
-	switch {
-	case err != nil:
-		log.Trace(fmt.Sprintf("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key))
-		// not found in memory cache, ie., a genuine store request
-		// create chunk
-		chunk = storage.NewChunk(req.Key, nil)
-
-	case chunk.SData == nil:
-		// found chunk in memory store, needs the data, validate now
-		log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v. request entry found", req))
-
-	default:
-		// data is found, store request ignored
-		// this should update access count?
-		log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v found locally. ignore.", req))
-		islocal = true
-		//return
-	}
-
-	hasher := self.hashfunc()
-	hasher.Write(req.SData)
-	if !bytes.Equal(hasher.Sum(nil), req.Key) {
-		// data does not validate, ignore
-		// TODO: peer should be penalised/dropped?
-		log.Warn(fmt.Sprintf("Depo.HandleStoreRequest: chunk invalid. store request ignored: %v", req))
-		return
-	}
-
-	if islocal {
-		return
-	}
-	// update chunk with size and data
-	chunk.SData = req.SData // protocol validates that SData is minimum 9 bytes long (int64 size  + at least one byte of data)
-	chunk.Size = int64(binary.LittleEndian.Uint64(req.SData[0:8]))
-	log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p))
-	chunk.Source = p
-	self.netStore.Put(chunk)
-}
-
-// entrypoint for retrieve requests coming from the bzz wire protocol
-// checks swap balance - return if peer has no credit
-func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) {
-	req.from = p
-	// swap - record credit for 1 request
-	// note that only charge actual reqsearches
-	var err error
-	if p.swap != nil {
-		err = p.swap.Add(1)
-	}
-	if err != nil {
-		log.Warn(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - cannot process request: %v", req.Key.Log(), err))
-		return
-	}
-
-	// call storage.NetStore#Get which
-	// blocks until local retrieval finished
-	// launches cloud retrieval
-	chunk, _ := self.netStore.Get(req.Key)
-	req = self.strategyUpdateRequest(chunk.Req, req)
-	// check if we can immediately deliver
-	if chunk.SData != nil {
-		log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, delivering...", req.Key.Log()))
-
-		if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size {
-			sreq := &storeRequestMsgData{
-				Id:             req.Id,
-				Key:            chunk.Key,
-				SData:          chunk.SData,
-				requestTimeout: req.timeout, //
-			}
-			p.syncer.addRequest(sreq, DeliverReq)
-		} else {
-			log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, not wanted", req.Key.Log()))
-		}
-	} else {
-		log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content not found locally. asked swarm for help. will get back", req.Key.Log()))
-	}
-}
-
-// add peer request the chunk and decides the timeout for the response if still searching
-func (self *Depo) strategyUpdateRequest(rs *storage.RequestStatus, origReq *retrieveRequestMsgData) (req *retrieveRequestMsgData) {
-	log.Trace(fmt.Sprintf("Depo.strategyUpdateRequest: key %v", origReq.Key.Log()))
-	// we do not create an alternative one
-	req = origReq
-	if rs != nil {
-		self.addRequester(rs, req)
-		req.setTimeout(self.searchTimeout(rs, req))
-	}
-	return
-}
-
-// decides the timeout promise sent with the immediate peers response to a retrieve request
-// if timeout is explicitly set and expired
-func (self *Depo) searchTimeout(rs *storage.RequestStatus, req *retrieveRequestMsgData) (timeout *time.Time) {
-	reqt := req.getTimeout()
-	t := time.Now().Add(searchTimeout)
-	if reqt != nil && reqt.Before(t) {
-		return reqt
-	} else {
-		return &t
-	}
-}
-
-/*
-adds a new peer to an existing open request
-only add if less than requesterCount peers forwarded the same request id so far
-note this is done irrespective of status (searching or found)
-*/
-func (self *Depo) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) {
-	log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id))
-	list := rs.Requesters[req.Id]
-	rs.Requesters[req.Id] = append(list, req)
-}

swarm/network/forwarding.go

@@ -1,150 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"fmt"
-	"math/rand"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-const requesterCount = 3
-
-/*
-forwarder implements the CloudStore interface (use by storage.NetStore)
-and serves as the cloud store backend orchestrating storage/retrieval/delivery
-via the native bzz protocol
-which uses an MSB logarithmic distance-based semi-permanent Kademlia table for
-* recursive forwarding style routing for retrieval
-* smart syncronisation
-*/
-
-type forwarder struct {
-	hive *Hive
-}
-
-func NewForwarder(hive *Hive) *forwarder {
-	return &forwarder{hive: hive}
-}
-
-// generate a unique id uint64
-func generateId() uint64 {
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	return uint64(r.Int63())
-}
-
-var searchTimeout = 3 * time.Second
-
-// forwarding logic
-// logic propagating retrieve requests to peers given by the kademlia hive
-func (self *forwarder) Retrieve(chunk *storage.Chunk) {
-	peers := self.hive.getPeers(chunk.Key, 0)
-	log.Trace(fmt.Sprintf("forwarder.Retrieve: %v - received %d peers from KΛÐΞMLIΛ...", chunk.Key.Log(), len(peers)))
-OUT:
-	for _, p := range peers {
-		log.Trace(fmt.Sprintf("forwarder.Retrieve: sending retrieveRequest %v to peer [%v]", chunk.Key.Log(), p))
-		for _, recipients := range chunk.Req.Requesters {
-			for _, recipient := range recipients {
-				req := recipient.(*retrieveRequestMsgData)
-				if req.from.Addr() == p.Addr() {
-					continue OUT
-				}
-			}
-		}
-		req := &retrieveRequestMsgData{
-			Key: chunk.Key,
-			Id:  generateId(),
-		}
-		var err error
-		if p.swap != nil {
-			err = p.swap.Add(-1)
-		}
-		if err == nil {
-			p.retrieve(req)
-			break OUT
-		}
-		log.Warn(fmt.Sprintf("forwarder.Retrieve: unable to send retrieveRequest to peer [%v]: %v", chunk.Key.Log(), err))
-	}
-}
-
-// requests to specific peers given by the kademlia hive
-// except for peers that the store request came from (if any)
-// delivery queueing taken care of by syncer
-func (self *forwarder) Store(chunk *storage.Chunk) {
-	var n int
-	msg := &storeRequestMsgData{
-		Key:   chunk.Key,
-		SData: chunk.SData,
-	}
-	var source *peer
-	if chunk.Source != nil {
-		source = chunk.Source.(*peer)
-	}
-	for _, p := range self.hive.getPeers(chunk.Key, 0) {
-		log.Trace(fmt.Sprintf("forwarder.Store: %v %v", p, chunk))
-
-		if p.syncer != nil && (source == nil || p.Addr() != source.Addr()) {
-			n++
-			Deliver(p, msg, PropagateReq)
-		}
-	}
-	log.Trace(fmt.Sprintf("forwarder.Store: sent to %v peers (chunk = %v)", n, chunk))
-}
-
-// once a chunk is found deliver it to its requesters unless timed out
-func (self *forwarder) Deliver(chunk *storage.Chunk) {
-	// iterate over request entries
-	for id, requesters := range chunk.Req.Requesters {
-		counter := requesterCount
-		msg := &storeRequestMsgData{
-			Key:   chunk.Key,
-			SData: chunk.SData,
-		}
-		var n int
-		var req *retrieveRequestMsgData
-		// iterate over requesters with the same id
-		for id, r := range requesters {
-			req = r.(*retrieveRequestMsgData)
-			if req.timeout == nil || req.timeout.After(time.Now()) {
-				log.Trace(fmt.Sprintf("forwarder.Deliver: %v -> %v", req.Id, req.from))
-				msg.Id = uint64(id)
-				Deliver(req.from, msg, DeliverReq)
-				n++
-				counter--
-				if counter <= 0 {
-					break
-				}
-			}
-		}
-		log.Trace(fmt.Sprintf("forwarder.Deliver: submit chunk %v (request id %v) for delivery to %v peers", chunk.Key.Log(), id, n))
-	}
-}
-
-// initiate delivery of a chunk to a particular peer via syncer#addRequest
-// depending on syncer mode and priority settings and sync request type
-// this either goes via confirmation roundtrip or queued or pushed directly
-func Deliver(p *peer, req interface{}, ty int) {
-	p.syncer.addRequest(req, ty)
-}
-
-// push chunk over to peer
-func Push(p *peer, key storage.Key, priority uint) {
-	p.syncer.doDelivery(key, priority, p.syncer.quit)
-}

swarm/network/hive.go

@@ -18,190 +18,219 @@ package network
 
 import (
 	"fmt"
-	"math/rand"
-	"path/filepath"
+	// "math/rand"
+	// "sort"
+	"bytes"
+	"sync"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/netutil"
-	"github.com/ethereum/go-ethereum/swarm/network/kademlia"
-	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-// Hive is the logistic manager of the swarm
-// it uses a generic kademlia nodetable to find best peer list
-// for any target
-// this is used by the netstore to search for content in the swarm
-// the bzz protocol peersMsgData exchange is relayed to Kademlia
-// for db storage and filtering
-// connections and disconnections are reported and relayed
-// to keep the nodetable uptodate
+/*
+Hive is the logistic manager of the swarm
+it uses an Overlay Topology driver (e.g., generic kademlia nodetable)
+to find best peer list for any target
+this is used by the netstore to search for content in the swarm
 
+It handles the bzz protocol getPeersMsg peersMsg exchange
+and relay the peer request process to the Overlay module
+
+peer connections and disconnections are reported and registered
+to keep the nodetable uptodate
+*/
+type Overlay interface {
+	Register(NodeAddr) error
+	On(Node) (Node, error)
+	Off(Node)
+
+	EachNode([]byte, int, func(Node) bool)
+	EachNodeAddr([]byte, int, func(NodeAddr) bool)
+
+	SuggestNodeAddr() NodeAddr
+	SuggestOrder() int
+
+	Info() string
+}
+
+// Hive implements the PeerPool interface
 type Hive struct {
-	listenAddr   func() string
-	callInterval uint64
-	id           discover.NodeID
-	addr         kademlia.Address
-	kad          *kademlia.Kademlia
-	path         string
+	*HiveParams // settings
+	Overlay     // the overlay topology driver
+	peers       map[discover.NodeID]Node
+
+	lock   sync.Mutex
 	quit   chan bool
 	toggle chan bool
 	more   chan bool
-
-	// for testing only
-	swapEnabled bool
-	syncEnabled bool
-	blockRead   bool
-	blockWrite  bool
 }
 
 const (
+	peersBroadcastSetSize = 1
+	maxPeersPerRequest    = 5
 	callInterval          = 3000000000
-	// bucketSize   = 3
-	// maxProx      = 8
-	// proxBinSize  = 4
 )
 
 type HiveParams struct {
+	PeersBroadcastSetSize uint
+	MaxPeersPerRequest    uint
 	CallInterval          uint64
-	KadDbPath    string
-	*kademlia.KadParams
 }
 
-func NewHiveParams(path string) *HiveParams {
-	kad := kademlia.NewKadParams()
-	// kad.BucketSize = bucketSize
-	// kad.MaxProx = maxProx
-	// kad.ProxBinSize = proxBinSize
-
+func NewHiveParams() *HiveParams {
 	return &HiveParams{
+		PeersBroadcastSetSize: peersBroadcastSetSize,
+		MaxPeersPerRequest:    maxPeersPerRequest,
 		CallInterval:          callInterval,
-		KadDbPath:    filepath.Join(path, "bzz-peers.json"),
-		KadParams:    kad,
 	}
 }
 
-func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive {
-	kad := kademlia.New(kademlia.Address(addr), params.KadParams)
+// Hive constructor embeds both arguments
+// HiveParams config parameters
+// Overlay Topology Driver Interface
+func NewHive(params *HiveParams, overlay Overlay) *Hive {
 	return &Hive{
-		callInterval: params.CallInterval,
-		kad:          kad,
-		addr:         kad.Addr(),
-		path:         params.KadDbPath,
-		swapEnabled:  swapEnabled,
-		syncEnabled:  syncEnabled,
+		HiveParams: params,
+		Overlay:    overlay,
+		peers:      make(map[discover.NodeID]Node),
 	}
 }
 
-func (self *Hive) SyncEnabled(on bool) {
-	self.syncEnabled = on
+// messages that hive regusters handles for
+var HiveMsgs = []interface{}{
+	&getPeersMsg{},
+	&peersMsg{},
 }
 
-func (self *Hive) SwapEnabled(on bool) {
-	self.swapEnabled = on
+/*
+peersMsg is the message to pass peer information
+It is always a response to a peersRequestMsg
+
+The encoding of a peer is identical to that in the devp2p base protocol peers
+messages: [IP, Port, NodeID]
+note that a node's DPA address is not the NodeID but the hash of the NodeID.
+
+To mitigate against spurious peers messages, requests should be remembered
+and correctness of responses should be checked
+
+If the proxBin of peers in the response is incorrect the sender should be
+disconnected
+*/
+type peersMsg struct {
+	Peers []*peerAddr
 }
 
-func (self *Hive) BlockNetworkRead(on bool) {
-	self.blockRead = on
+func (self peersMsg) String() string {
+	return fmt.Sprintf("%T: %v", self, self.Peers)
 }
 
-func (self *Hive) BlockNetworkWrite(on bool) {
-	self.blockWrite = on
+// getPeersMsg is sent to (random) peers to request (Max) peers of a specific order
+type getPeersMsg struct {
+	Order uint
+	Max   uint
 }
 
-// public accessor to the hive base address
-func (self *Hive) Addr() kademlia.Address {
-	return self.addr
+func (self getPeersMsg) String() string {
+	return fmt.Sprintf("%T: accept max %v peers of PO%03d", self, self.Max, self.Order)
 }
 
 // Start receives network info only at startup
 // listedAddr is a function to retrieve listening address to advertise to peers
 // connectPeer is a function to connect to a peer based on its NodeID or enode URL
+// af() returns an arbitrary ticker channel
 // there are called on the p2p.Server which runs on the node
-func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) {
+func (self *Hive) Start(connectPeer func(string) error, af func() <-chan time.Time) (err error) {
+
 	self.toggle = make(chan bool)
 	self.more = make(chan bool)
 	self.quit = make(chan bool)
-	self.id = id
-	self.listenAddr = listenAddr
-	err = self.kad.Load(self.path, nil)
-	if err != nil {
-		log.Warn(fmt.Sprintf("Warning: error reading kaddb '%s' (skipping): %v", self.path, err))
-		err = nil
-	}
+	order := -1
+	glog.V(logger.Detail).Infof("hive started")
 	// this loop is doing bootstrapping and maintains a healthy table
-	go self.keepAlive()
+	go self.keepAlive(af)
 	go func() {
-		// whenever toggled ask kademlia about most preferred peer
-		for alive := range self.more {
-			if !alive {
+		// each iteration, ask kademlia about most preferred peer
+		for more := range self.more {
+			if !more {
 				// receiving false closes the loop while allowing parallel routines
 				// to attempt to write to more (remove Peer when shutting down)
 				return
 			}
-			node, need, proxLimit := self.kad.Suggest()
+			glog.V(logger.Detail).Infof("hive delegate to overlay driver: suggest addr to connect to")
+			addr := self.SuggestNodeAddr()
+
+			if addr != nil {
+				glog.V(logger.Detail).Infof("========> connect to bee %v", addr)
+				err := connectPeer(NodeId(addr).NodeID.String())
+				if err != nil {
+					glog.V(logger.Detail).Infof("===X====> connect to bee %v failed: %v", addr, err)
 
-			if node != nil && len(node.Url) > 0 {
-				log.Trace(fmt.Sprintf("call known bee %v", node.Url))
-				// enode or any lower level connection address is unnecessary in future
-				// discovery table is used to look it up.
-				connectPeer(node.Url)
 				}
-			if need {
-				// a random peer is taken from the table
-				peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1)
-				if len(peers) > 0 {
-					// a random address at prox bin 0 is sent for lookup
-					randAddr := kademlia.RandomAddressAt(self.addr, proxLimit)
-					req := &retrieveRequestMsgData{
-						Key: storage.Key(randAddr[:]),
 			}
-					log.Trace(fmt.Sprintf("call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0]))
-					peers[0].(*peer).retrieve(req)
-				} else {
-					log.Warn(fmt.Sprintf("no peer"))
+			glog.V(logger.Detail).Infof("hive delegate to overlay driver: suggest order for getPeersMsg")
+			order = self.SuggestOrder()
+			req := &getPeersMsg{
+				Order: uint(order),
+				Max:   self.MaxPeersPerRequest,
 			}
-				log.Trace(fmt.Sprintf("buzz kept alive"))
-			} else {
-				log.Info(fmt.Sprintf("no need for more bees"))
+			var i uint
+			var err error
+			glog.V(logger.Debug).Infof("requesting bees of PO%03d from %v (each max %v)", order, self.PeersBroadcastSetSize, self.MaxPeersPerRequest)
+			self.EachNode(nil, order, func(n Node) bool {
+				glog.V(logger.Debug).Infof("%T sent to %v", req, n.ID())
+				err = n.Send(req)
+				if err == nil {
+					i++
+					if i >= self.PeersBroadcastSetSize {
+						return false
+					}
+				}
+				return true
+			})
+			glog.V(logger.Debug).Infof("sent %T to %d/%d peers", req, i, self.PeersBroadcastSetSize)
+			// only switch off if full
+			var need bool
+			if order < 256 || addr != nil {
+				need = true
 			}
 			select {
 			case self.toggle <- need:
+				glog.V(logger.Debug).Infof("keep hive alive: %v", need)
 			case <-self.quit:
 				return
 			}
-			log.Debug(fmt.Sprintf("queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()))
 		}
+		glog.V(logger.Debug).Infof("%v", self.Info())
 	}()
 	return
 }
 
+func (self *Hive) ticker() <-chan time.Time {
+	return time.NewTicker(time.Duration(self.CallInterval)).C
+}
+
 // keepAlive is a forever loop
 // in its awake state it periodically triggers connection attempts
 // by writing to self.more until Kademlia Table is saturated
 // wake state is toggled by writing to self.toggle
 // it restarts if the table becomes non-full again due to disconnections
-func (self *Hive) keepAlive() {
-	alarm := time.NewTicker(time.Duration(self.callInterval)).C
+func (self *Hive) keepAlive(af func() <-chan time.Time) {
+	glog.V(logger.Debug).Infof("keep alive loop started")
+	alarm := af()
 	for {
 		select {
 		case <-alarm:
-			if self.kad.DBCount() > 0 {
-				select {
-				case self.more <- true:
-					log.Debug(fmt.Sprintf("buzz wakeup"))
-				default:
-				}
-			}
+			glog.V(logger.Debug).Infof("wake up: make hive alive")
+			self.wake()
 		case need := <-self.toggle:
 			if alarm == nil && need {
-				alarm = time.NewTicker(time.Duration(self.callInterval)).C
+				alarm = af()
 			}
+			// if hive saturated, no more peers asked
 			if alarm != nil && !need {
 				alarm = nil
-
 			}
 		case <-self.quit:
 			return
@@ -209,179 +238,132 @@ func (self *Hive) keepAlive() {
 	}
 }
 
-func (self *Hive) Stop() error {
+func (self *Hive) Stop() {
 	// closing toggle channel quits the updateloop
 	close(self.quit)
-	return self.kad.Save(self.path, saveSync)
 }
 
-// called at the end of a successful protocol handshake
-func (self *Hive) addPeer(p *peer) error {
-	defer func() {
+func (self *Hive) wake() {
 	select {
 	case self.more <- true:
+		glog.V(logger.Debug).Infof("hive woken up")
+	case <-self.quit:
 	default:
+		glog.V(logger.Debug).Infof("hive already awake")
 	}
-	}()
-	log.Trace(fmt.Sprintf("hi new bee %v", p))
-	err := self.kad.On(p, loadSync)
+}
+
+// func (self *Hive) anyN(n int, peers []Node) []Node {
+// 	self.lock.Lock()
+// 	defer self.lock.Unlock()
+// 	pick := rand.Perm(len(peers))
+// 	sort.Ints(pick)
+// 	var nodes []Node
+// 	j := 0
+// 	i := 0
+// 	for i, node := range peers {
+// 		if i == pick[j] {
+// 			j++
+// 			nodes = append(nodes, node)
+// 			if j == n {
+// 				break
+// 			}
+// 		}
+// 	}
+// 	return nodes
+// }
+
+// Add is called at the end of a successful protocol handshake to register a peer onlune
+func (self *Hive) Add(p Node) error {
+	defer self.wake()
+	glog.V(logger.Detail).Infof("add new bee %v", p)
+	drop, err := self.On(p)
 	if err != nil {
 		return err
 	}
-	// self lookup (can be encoded as nil/zero key since peers addr known) + no id ()
-	// the most common way of saying hi in bzz is initiation of gossip
-	// let me know about anyone new from my hood , here is the storageradius
-	// to send the 6 byte self lookup
-	// we do not record as request or forward it, just reply with peers
-	p.retrieve(&retrieveRequestMsgData{})
-	log.Trace(fmt.Sprintf("'whatsup wheresdaparty' sent to %v", p))
+	if drop != nil {
+		drop.Drop()
+		return nil
+	}
+
+	self.lock.Lock()
+	self.peers[p.ID()] = p
+	self.lock.Unlock()
+
+	p.Register(&peersMsg{}, self.handlePeersMsg(p))
+	p.Register(&getPeersMsg{}, self.handleGetPeersMsg(p))
 
 	return nil
 }
 
-// called after peer disconnected
-func (self *Hive) removePeer(p *peer) {
-	log.Debug(fmt.Sprintf("bee %v removed", p))
-	self.kad.Off(p, saveSync)
-	select {
-	case self.more <- true:
-	default:
+// Remove called after peer is disconnected
+func (self *Hive) Remove(p Node) {
+	defer self.wake()
+	glog.V(logger.Debug).Infof("remove bee %v", p)
+	self.Off(p)
+	self.lock.Lock()
+	delete(self.peers, p.ID())
+	self.lock.Unlock()
+}
+
+// func (self *Hive) Get(n string) Node {
+// 	return Node(self.peers[n])
+// }
+
+// handlePeersMsg called by the protocol when receiving peerset (for target address)
+// list of nodes ([]NodeAddr in peersMsg is added to the overlay db
+func (self *Hive) handlePeersMsg(p Node) func(interface{}) error {
+	return func(msg interface{}) error {
+		// wake up the hive on news of new arrival
+		defer self.wake()
+		// register all addresses
+		var err error
+		req := msg.(*peersMsg)
+		for _, p := range req.Peers {
+			err = self.Register(p)
+			// TODO: these are known to our peer, so do not resend during the session
 		}
-	if self.kad.Count() == 0 {
-		log.Debug(fmt.Sprintf("empty, all bees gone"))
+		// FIXME: only the last error is returned
+		return err
 	}
 }
 
-// Retrieve a list of live peers that are closer to target than us
-func (self *Hive) getPeers(target storage.Key, max int) (peers []*peer) {
-	var addr kademlia.Address
-	copy(addr[:], target[:])
-	for _, node := range self.kad.FindClosest(addr, max) {
-		peers = append(peers, node.(*peer))
-	}
-	return
-}
-
-// disconnects all the peers
-func (self *Hive) DropAll() {
-	log.Info(fmt.Sprintf("dropping all bees"))
-	for _, node := range self.kad.FindClosest(kademlia.Address{}, 0) {
-		node.Drop()
-	}
-}
-
-// contructor for kademlia.NodeRecord based on peer address alone
-// TODO: should go away and only addr passed to kademlia
-func newNodeRecord(addr *peerAddr) *kademlia.NodeRecord {
-	now := time.Now()
-	return &kademlia.NodeRecord{
-		Addr:  addr.Addr,
-		Url:   addr.String(),
-		Seen:  now,
-		After: now,
-	}
-}
-
-// called by the protocol when receiving peerset (for target address)
+// HandleGetPeersMsg called by the protocol when receiving peerset (for target address)
 // peersMsgData is converted to a slice of NodeRecords for Kademlia
 // this is to store all thats needed
-func (self *Hive) HandlePeersMsg(req *peersMsgData, from *peer) {
-	var nrs []*kademlia.NodeRecord
-	for _, p := range req.Peers {
-		if err := netutil.CheckRelayIP(from.remoteAddr.IP, p.IP); err != nil {
-			log.Trace(fmt.Sprintf("invalid peer IP %v from %v: %v", from.remoteAddr.IP, p.IP, err))
-			continue
+func (self *Hive) handleGetPeersMsg(p Node) func(interface{}) error {
+	return func(msg interface{}) error {
+		req := msg.(*getPeersMsg)
+		var peers []*peerAddr
+		self.EachNode(p.OverlayAddr(), int(req.Order), func(n Node) bool {
+			if bytes.Compare(n.OverlayAddr(), p.OverlayAddr()) != 0 {
+				peers = append(peers, &peerAddr{n.OverlayAddr(), n.UnderlayAddr()})
 			}
-		nrs = append(nrs, newNodeRecord(p))
+			return len(peers) < int(req.Max)
+		})
+
+		resp := &peersMsg{
+			Peers: peers,
 		}
-	self.kad.Add(nrs)
-}
-
-// peer wraps the protocol instance to represent a connected peer
-// it implements kademlia.Node interface
-type peer struct {
-	*bzz // protocol instance running on peer connection
-}
-
-// protocol instance implements kademlia.Node interface (embedded peer)
-func (self *peer) Addr() kademlia.Address {
-	return self.remoteAddr.Addr
-}
-
-func (self *peer) Url() string {
-	return self.remoteAddr.String()
-}
-
-// TODO take into account traffic
-func (self *peer) LastActive() time.Time {
-	return self.lastActive
-}
-
-// reads the serialised form of sync state persisted as the 'Meta' attribute
-// and sets the decoded syncState on the online node
-func loadSync(record *kademlia.NodeRecord, node kademlia.Node) error {
-	p, ok := node.(*peer)
-	if !ok {
-		return fmt.Errorf("invalid type")
+		err := p.Send(resp)
+		if err != nil {
+			return err
 		}
-	if record.Meta == nil {
-		log.Debug(fmt.Sprintf("no sync state for node record %v setting default", record))
-		p.syncState = &syncState{DbSyncState: &storage.DbSyncState{}}
 		return nil
 	}
-	state, err := decodeSync(record.Meta)
-	if err != nil {
-		return fmt.Errorf("error decoding kddb record meta info into a sync state: %v", err)
-	}
-	log.Trace(fmt.Sprintf("sync state for node record %v read from Meta: %s", record, string(*(record.Meta))))
-	p.syncState = state
-	return err
 }
 
-// callback when saving a sync state
-func saveSync(record *kademlia.NodeRecord, node kademlia.Node) {
-	if p, ok := node.(*peer); ok {
-		meta, err := encodeSync(p.syncState)
-		if err != nil {
-			log.Warn(fmt.Sprintf("error saving sync state for %v: %v", node, err))
-			return
-		}
-		log.Trace(fmt.Sprintf("saved sync state for %v: %s", node, string(*meta)))
-		record.Meta = meta
+func (self *Hive) PeerInfo(id discover.NodeID) interface{} {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	p, ok := self.peers[id]
+	if !ok {
+		return nil
 	}
+	return interface{}(&peerAddr{p.OverlayAddr(), p.UnderlayAddr()})
 }
 
-// the immediate response to a retrieve request,
-// sends relevant peer data given by the kademlia hive to the requester
-// TODO: remember peers sent for duration of the session, only new peers sent
-func (self *Hive) peers(req *retrieveRequestMsgData) {
-	if req != nil && req.MaxPeers >= 0 {
-		var addrs []*peerAddr
-		if req.timeout == nil || time.Now().Before(*(req.timeout)) {
-			key := req.Key
-			// self lookup from remote peer
-			if storage.IsZeroKey(key) {
-				addr := req.from.Addr()
-				key = storage.Key(addr[:])
-				req.Key = nil
-			}
-			// get peer addresses from hive
-			for _, peer := range self.getPeers(key, int(req.MaxPeers)) {
-				addrs = append(addrs, peer.remoteAddr)
-			}
-			log.Debug(fmt.Sprintf("Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log()))
-
-			peersData := &peersMsgData{
-				Peers: addrs,
-				Key:   req.Key,
-				Id:    req.Id,
-			}
-			peersData.setTimeout(req.timeout)
-			req.from.peers(peersData)
-		}
-	}
-}
-
-func (self *Hive) String() string {
-	return self.kad.String()
+func HexToBytes(s string) []byte {
+	id := discover.MustHexID(s)
+	return id[:]
 }

swarm/network/hive_test.go

@@ -0,0 +1,128 @@
+package network
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	// "github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/protocols"
+	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
+)
+
+func init() {
+	glog.SetV(6)
+	glog.SetToStderr(true)
+}
+
+type testConnect struct {
+	mu       sync.Mutex
+	conns    []string
+	connectf func(c string) error
+	ticker   chan time.Time
+}
+
+func (self *testConnect) ping() <-chan time.Time {
+	return self.ticker
+}
+
+func (self *testConnect) connect(na string) error {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	self.conns = append(self.conns, na)
+	self.connectf(na)
+	return nil
+}
+
+func newBzzHiveTester(t *testing.T, n int, addr *peerAddr, pp PeerPool, ct *protocols.CodeMap, services func(Node) error) *bzzTester {
+	s := p2ptest.NewProtocolTester(t, NodeId(addr), n, newTestBzzProtocol(addr, pp, ct, services))
+	return &bzzTester{
+		addr:            addr,
+		flushCode:       3,
+		ExchangeSession: s,
+	}
+}
+
+func TestOverlayRegistration(t *testing.T) {
+	// setup
+	addr := RandomAddr()                     // tested peers peer address
+	to := NewTestOverlay(addr.OverlayAddr()) // overlay topology driver
+	pp := NewHive(NewHiveParams(), to)       // hive
+	ct := BzzCodeMap(HiveMsgs...)            // bzz protocol code map
+	s := newBzzHiveTester(t, 1, addr, pp, ct, nil)
+
+	// connect to the other peer
+	id := s.Ids[0]
+	raddr := NodeIdToAddr(id)
+	s.runHandshakes()
+
+	// hive should have called the overlay
+	if to.posMap[string(raddr.OverlayAddr())] == nil {
+		t.Fatalf("Overlay#On not called on new peer")
+	}
+}
+
+func TestRegisterAndConnect(t *testing.T) {
+	addr := RandomAddr()
+	to := NewTestOverlay(addr.OverlayAddr())
+	pp := NewHive(NewHiveParams(), to)
+	ct := BzzCodeMap(HiveMsgs...)
+	s := newBzzHiveTester(t, 0, addr, pp, ct, nil)
+
+	// register the node with the peerPool
+	id := p2ptest.RandomNodeId()
+	s.Start(id)
+	raddr := NodeIdToAddr(id)
+	pp.Register(raddr)
+	glog.V(5).Infof("%v", pp.Info())
+	// start the hive and wait for the connection
+	tc := &testConnect{
+		connectf: func(c string) error {
+			s.Connect(adapters.NewNodeIdFromHex(c))
+			return nil
+		},
+		ticker: make(chan time.Time),
+	}
+	pp.Start(tc.connect, tc.ping)
+	tc.ticker <- time.Now()
+	s.runHandshakes()
+	if to.posMap[string(raddr.OverlayAddr())] == nil {
+		t.Fatalf("Overlay#On not called on new peer")
+	}
+	glog.V(6).Infof("check peer requests for %v", id)
+	// tc.ticker <- time.Now()
+
+	// shakeHands(s, addr, id)
+	// s.Flush(int(ct.Length())-1, 0)
+	// time.Sleep(3)
+	ord := order(raddr.OverlayAddr())
+	o := 0
+	if ord == 0 {
+		o = 1
+	}
+	s.TestExchanges(p2ptest.Exchange{
+		Expects: []p2ptest.Expect{
+			p2ptest.Expect{
+				Code: 1,
+				Msg:  &getPeersMsg{uint(o), 5},
+				Peer: id,
+			},
+		},
+		// Triggers: []p2ptest.Trigger{
+		// 	p2ptest.Trigger{
+		// 		Code: 1,
+		// 		Msg:  &getPeersMsg{0, 1},
+		// 		Peer: 0,
+		// 	},
+		// },
+		// Expects: []p2ptest.Expect{
+		// 	p2ptest.Expect{
+		// 		Code: 1,
+		// 		Msg:  &peersMsg{[]*peerAddr{RandomAddr()}},
+		// 		Peer: 0,
+		// 	},
+		// },
+	})
+}

swarm/network/kademlia/kaddb.go

@@ -1,350 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package kademlia
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"sync"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-)
-
-type NodeData interface {
-	json.Marshaler
-	json.Unmarshaler
-}
-
-// allow inactive peers under
-type NodeRecord struct {
-	Addr  Address          // address of node
-	Url   string           // Url, used to connect to node
-	After time.Time        // next call after time
-	Seen  time.Time        // last connected at time
-	Meta  *json.RawMessage // arbitrary metadata saved for a peer
-
-	node Node
-}
-
-func (self *NodeRecord) setSeen() {
-	t := time.Now()
-	self.Seen = t
-	self.After = t
-}
-
-func (self *NodeRecord) String() string {
-	return fmt.Sprintf("<%v>", self.Addr)
-}
-
-// persisted node record database ()
-type KadDb struct {
-	Address              Address
-	Nodes                [][]*NodeRecord
-	index                map[Address]*NodeRecord
-	cursors              []int
-	lock                 sync.RWMutex
-	purgeInterval        time.Duration
-	initialRetryInterval time.Duration
-	connRetryExp         int
-}
-
-func newKadDb(addr Address, params *KadParams) *KadDb {
-	return &KadDb{
-		Address:              addr,
-		Nodes:                make([][]*NodeRecord, params.MaxProx+1), // overwritten by load
-		cursors:              make([]int, params.MaxProx+1),
-		index:                make(map[Address]*NodeRecord),
-		purgeInterval:        params.PurgeInterval,
-		initialRetryInterval: params.InitialRetryInterval,
-		connRetryExp:         params.ConnRetryExp,
-	}
-}
-
-func (self *KadDb) findOrCreate(index int, a Address, url string) *NodeRecord {
-	defer self.lock.Unlock()
-	self.lock.Lock()
-
-	record, found := self.index[a]
-	if !found {
-		record = &NodeRecord{
-			Addr: a,
-			Url:  url,
-		}
-		log.Info(fmt.Sprintf("add new record %v to kaddb", record))
-		// insert in kaddb
-		self.index[a] = record
-		self.Nodes[index] = append(self.Nodes[index], record)
-	} else {
-		log.Info(fmt.Sprintf("found record %v in kaddb", record))
-	}
-	// update last seen time
-	record.setSeen()
-	// update with url in case IP/port changes
-	record.Url = url
-	return record
-}
-
-// add adds node records to kaddb (persisted node record db)
-func (self *KadDb) add(nrs []*NodeRecord, proximityBin func(Address) int) {
-	defer self.lock.Unlock()
-	self.lock.Lock()
-	var n int
-	var nodes []*NodeRecord
-	for _, node := range nrs {
-		_, found := self.index[node.Addr]
-		if !found && node.Addr != self.Address {
-			node.setSeen()
-			self.index[node.Addr] = node
-			index := proximityBin(node.Addr)
-			dbcursor := self.cursors[index]
-			nodes = self.Nodes[index]
-			// this is inefficient for allocation, need to just append then shift
-			newnodes := make([]*NodeRecord, len(nodes)+1)
-			copy(newnodes[:], nodes[:dbcursor])
-			newnodes[dbcursor] = node
-			copy(newnodes[dbcursor+1:], nodes[dbcursor:])
-			log.Trace(fmt.Sprintf("new nodes: %v, nodes: %v", newnodes, nodes))
-			self.Nodes[index] = newnodes
-			n++
-		}
-	}
-	if n > 0 {
-		log.Debug(fmt.Sprintf("%d/%d node records (new/known)", n, len(nrs)))
-	}
-}
-
-/*
-next return one node record with the highest priority for desired
-connection.
-This is used to pick candidates for live nodes that are most wanted for
-a higly connected low centrality network structure for Swarm which best suits
-for a Kademlia-style routing.
-
-* Starting as naive node with empty db, this implements Kademlia bootstrapping
-* As a mature node, it fills short lines. All on demand.
-
-The candidate is chosen using the following strategy:
-We check for missing online nodes in the buckets for 1 upto Max BucketSize rounds.
-On each round we proceed from the low to high proximity order buckets.
-If the number of active nodes (=connected peers) is < rounds, then start looking
-for a known candidate. To determine if there is a candidate to recommend the
-kaddb node record database row corresponding to the bucket is checked.
-
-If the row cursor is on position i, the ith element in the row is chosen.
-If the record is scheduled not to be retried before NOW, the next element is taken.
-If the record is scheduled to be retried, it is set as checked, scheduled for
-checking and is returned. The time of the next check is in X (duration) such that
-X = ConnRetryExp * delta where delta is the time past since the last check and
-ConnRetryExp is constant obsoletion factor. (Note that when node records are added
-from peer messages, they are marked as checked and placed at the cursor, ie.
-given priority over older entries). Entries which were checked more than
-purgeInterval ago are deleted from the kaddb row. If no candidate is found after
-a full round of checking the next bucket up is considered. If no candidate is
-found when we reach the maximum-proximity bucket, the next round starts.
-
-node record a is more favoured to b a > b iff a is a passive node (record of
-offline past peer)
-|proxBin(a)| < |proxBin(b)|
-|| (proxBin(a) < proxBin(b) && |proxBin(a)| == |proxBin(b)|)
-|| (proxBin(a) == proxBin(b) && lastChecked(a) < lastChecked(b))
-
-
-The second argument returned names the first missing slot found
-*/
-func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRecord, need bool, proxLimit int) {
-	// return nil, proxLimit indicates that all buckets are filled
-	defer self.lock.Unlock()
-	self.lock.Lock()
-
-	var interval time.Duration
-	var found bool
-	var purge []bool
-	var delta time.Duration
-	var cursor int
-	var count int
-	var after time.Time
-
-	// iterate over columns maximum bucketsize times
-	for rounds := 1; rounds <= maxBinSize; rounds++ {
-	ROUND:
-		// iterate over rows from PO 0 upto MaxProx
-		for po, dbrow := range self.Nodes {
-			// if row has rounds connected peers, then take the next
-			if binSize(po) >= rounds {
-				continue ROUND
-			}
-			if !need {
-				// set proxlimit to the PO where the first missing slot is found
-				proxLimit = po
-				need = true
-			}
-			purge = make([]bool, len(dbrow))
-
-			// there is a missing slot - finding a node to connect to
-			// select a node record from the relavant kaddb row (of identical prox order)
-		ROW:
-			for cursor = self.cursors[po]; !found && count < len(dbrow); cursor = (cursor + 1) % len(dbrow) {
-				count++
-				node = dbrow[cursor]
-
-				// skip already connected nodes
-				if node.node != nil {
-					log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d/%d) already connected", node.Addr, po, cursor, len(dbrow)))
-					continue ROW
-				}
-
-				// if node is scheduled to connect
-				if time.Time(node.After).After(time.Now()) {
-					log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) skipped. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After))
-					continue ROW
-				}
-
-				delta = time.Since(time.Time(node.Seen))
-				if delta < self.initialRetryInterval {
-					delta = self.initialRetryInterval
-				}
-				if delta > self.purgeInterval {
-					// remove node
-					purge[cursor] = true
-					log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen))
-					continue ROW
-				}
-
-				log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) ready to be tried. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After))
-
-				// scheduling next check
-				interval = time.Duration(delta * time.Duration(self.connRetryExp))
-				after = time.Now().Add(interval)
-
-				log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval))
-				node.After = after
-				found = true
-			} // ROW
-			self.cursors[po] = cursor
-			self.delete(po, purge)
-			if found {
-				return node, need, proxLimit
-			}
-		} // ROUND
-	} // ROUNDS
-
-	return nil, need, proxLimit
-}
-
-// deletes the noderecords of a kaddb row corresponding to the indexes
-// caller must hold the dblock
-// the call is unsafe, no index checks
-func (self *KadDb) delete(row int, purge []bool) {
-	var nodes []*NodeRecord
-	dbrow := self.Nodes[row]
-	for i, del := range purge {
-		if i == self.cursors[row] {
-			//reset cursor
-			self.cursors[row] = len(nodes)
-		}
-		// delete the entry to be purged
-		if del {
-			delete(self.index, dbrow[i].Addr)
-			continue
-		}
-		// otherwise append to new list
-		nodes = append(nodes, dbrow[i])
-	}
-	self.Nodes[row] = nodes
-}
-
-// save persists kaddb on disk (written to file on path in json format.
-func (self *KadDb) save(path string, cb func(*NodeRecord, Node)) error {
-	defer self.lock.Unlock()
-	self.lock.Lock()
-
-	var n int
-
-	for _, b := range self.Nodes {
-		for _, node := range b {
-			n++
-			node.After = time.Now()
-			node.Seen = time.Now()
-			if cb != nil {
-				cb(node, node.node)
-			}
-		}
-	}
-
-	data, err := json.MarshalIndent(self, "", " ")
-	if err != nil {
-		return err
-	}
-	err = ioutil.WriteFile(path, data, os.ModePerm)
-	if err != nil {
-		log.Warn(fmt.Sprintf("unable to save kaddb with %v nodes to %v: %v", n, path, err))
-	} else {
-		log.Info(fmt.Sprintf("saved kaddb with %v nodes to %v", n, path))
-	}
-	return err
-}
-
-// Load(path) loads the node record database (kaddb) from file on path.
-func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err error) {
-	defer self.lock.Unlock()
-	self.lock.Lock()
-
-	var data []byte
-	data, err = ioutil.ReadFile(path)
-	if err != nil {
-		return
-	}
-
-	err = json.Unmarshal(data, self)
-	if err != nil {
-		return
-	}
-	var n int
-	var purge []bool
-	for po, b := range self.Nodes {
-		purge = make([]bool, len(b))
-	ROW:
-		for i, node := range b {
-			if cb != nil {
-				err = cb(node, node.node)
-				if err != nil {
-					purge[i] = true
-					continue ROW
-				}
-			}
-			n++
-			if (node.After == time.Time{}) {
-				node.After = time.Now()
-			}
-			self.index[node.Addr] = node
-		}
-		self.delete(po, purge)
-	}
-	log.Info(fmt.Sprintf("loaded kaddb with %v nodes from %v", n, path))
-
-	return
-}
-
-// accessor for KAD offline db count
-func (self *KadDb) count() int {
-	defer self.lock.Unlock()
-	self.lock.Lock()
-	return len(self.index)
-}

swarm/network/kademlia/kademlia.go

@@ -1,428 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package kademlia
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-)
-
-const (
-	bucketSize   = 4
-	proxBinSize  = 2
-	maxProx      = 8
-	connRetryExp = 2
-	maxPeers     = 100
-)
-
-var (
-	purgeInterval        = 42 * time.Hour
-	initialRetryInterval = 42 * time.Millisecond
-	maxIdleInterval      = 42 * 1000 * time.Millisecond
-	// maxIdleInterval      = 42 * 10	0 * time.Millisecond
-)
-
-type KadParams struct {
-	// adjustable parameters
-	MaxProx              int
-	ProxBinSize          int
-	BucketSize           int
-	PurgeInterval        time.Duration
-	InitialRetryInterval time.Duration
-	MaxIdleInterval      time.Duration
-	ConnRetryExp         int
-}
-
-func NewKadParams() *KadParams {
-	return &KadParams{
-		MaxProx:              maxProx,
-		ProxBinSize:          proxBinSize,
-		BucketSize:           bucketSize,
-		PurgeInterval:        purgeInterval,
-		InitialRetryInterval: initialRetryInterval,
-		MaxIdleInterval:      maxIdleInterval,
-		ConnRetryExp:         connRetryExp,
-	}
-}
-
-// Kademlia is a table of active nodes
-type Kademlia struct {
-	addr       Address      // immutable baseaddress of the table
-	*KadParams              // Kademlia configuration parameters
-	proxLimit  int          // state, the PO of the first row of the most proximate bin
-	proxSize   int          // state, the number of peers in the most proximate bin
-	count      int          // number of active peers (w live connection)
-	buckets    [][]Node     // the actual bins
-	db         *KadDb       // kaddb, node record database
-	lock       sync.RWMutex // mutex to access buckets
-}
-
-type Node interface {
-	Addr() Address
-	Url() string
-	LastActive() time.Time
-	Drop()
-}
-
-// public constructor
-// add is the base address of the table
-// params is KadParams configuration
-func New(addr Address, params *KadParams) *Kademlia {
-	buckets := make([][]Node, params.MaxProx+1)
-	return &Kademlia{
-		addr:      addr,
-		KadParams: params,
-		buckets:   buckets,
-		db:        newKadDb(addr, params),
-	}
-}
-
-// accessor for KAD base address
-func (self *Kademlia) Addr() Address {
-	return self.addr
-}
-
-// accessor for KAD active node count
-func (self *Kademlia) Count() int {
-	defer self.lock.Unlock()
-	self.lock.Lock()
-	return self.count
-}
-
-// accessor for KAD active node count
-func (self *Kademlia) DBCount() int {
-	return self.db.count()
-}
-
-// On is the entry point called when a new nodes is added
-// unsafe in that node is not checked to be already active node (to be called once)
-func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) {
-	log.Debug(fmt.Sprintf("%v", self))
-	defer self.lock.Unlock()
-	self.lock.Lock()
-
-	index := self.proximityBin(node.Addr())
-	record := self.db.findOrCreate(index, node.Addr(), node.Url())
-
-	if cb != nil {
-		err = cb(record, node)
-		log.Trace(fmt.Sprintf("cb(%v, %v) ->%v", record, node, err))
-		if err != nil {
-			return fmt.Errorf("unable to add node %v, callback error: %v", node.Addr(), err)
-		}
-		log.Debug(fmt.Sprintf("add node record %v with node %v", record, node))
-	}
-
-	// insert in kademlia table of active nodes
-	bucket := self.buckets[index]
-	// if bucket is full insertion replaces the worst node
-	// TODO: give priority to peers with active traffic
-	if len(bucket) < self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation
-		self.buckets[index] = append(bucket, node)
-		log.Debug(fmt.Sprintf("add node %v to table", node))
-		self.setProxLimit(index, true)
-		record.node = node
-		self.count++
-		return nil
-	}
-
-	// always rotate peers
-	idle := self.MaxIdleInterval
-	var pos int
-	var replaced Node
-	for i, p := range bucket {
-		idleInt := time.Since(p.LastActive())
-		if idleInt > idle {
-			idle = idleInt
-			pos = i
-			replaced = p
-		}
-	}
-	if replaced == nil {
-		log.Debug(fmt.Sprintf("all peers wanted, PO%03d bucket full", index))
-		return fmt.Errorf("bucket full")
-	}
-	log.Debug(fmt.Sprintf("node %v replaced by %v (idle for %v  > %v)", replaced, node, idle, self.MaxIdleInterval))
-	replaced.Drop()
-	// actually replace in the row. When off(node) is called, the peer is no longer in the row
-	bucket[pos] = node
-	// there is no change in bucket cardinalities so no prox limit adjustment is needed
-	record.node = node
-	self.count++
-	return nil
-
-}
-
-// Off is the called when a node is taken offline (from the protocol main loop exit)
-func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-
-	index := self.proximityBin(node.Addr())
-	bucket := self.buckets[index]
-	for i := 0; i < len(bucket); i++ {
-		if node.Addr() == bucket[i].Addr() {
-			self.buckets[index] = append(bucket[:i], bucket[(i+1):]...)
-			self.setProxLimit(index, false)
-			break
-		}
-	}
-
-	record := self.db.index[node.Addr()]
-	// callback on remove
-	if cb != nil {
-		cb(record, record.node)
-	}
-	record.node = nil
-	self.count--
-	log.Debug(fmt.Sprintf("remove node %v from table, population now is %v", node, self.count))
-
-	return
-}
-
-// proxLimit is dynamically adjusted so that
-// 1) there is no empty buckets in bin < proxLimit and
-// 2) the sum of all items are the minimum possible but higher than ProxBinSize
-// adjust Prox (proxLimit and proxSize after an insertion/removal of nodes)
-// caller holds the lock
-func (self *Kademlia) setProxLimit(r int, on bool) {
-	// if the change is outside the core (PO lower)
-	// and the change does not leave a bucket empty then
-	// no adjustment needed
-	if r < self.proxLimit && len(self.buckets[r]) > 0 {
-		return
-	}
-	// if on=a node was added, then r must be within prox limit so increment cardinality
-	if on {
-		self.proxSize++
-		curr := len(self.buckets[self.proxLimit])
-		// if now core is big enough without the furthest bucket, then contract
-		// this can result in more than one bucket change
-		for self.proxSize >= self.ProxBinSize+curr && curr > 0 {
-			self.proxSize -= curr
-			self.proxLimit++
-			curr = len(self.buckets[self.proxLimit])
-
-			log.Trace(fmt.Sprintf("proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r))
-		}
-		return
-	}
-	// otherwise
-	if r >= self.proxLimit {
-		self.proxSize--
-	}
-	// expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality
-	for (self.proxSize < self.ProxBinSize || r < self.proxLimit) &&
-		self.proxLimit > 0 {
-		//
-		self.proxLimit--
-		self.proxSize += len(self.buckets[self.proxLimit])
-		log.Trace(fmt.Sprintf("proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r))
-	}
-}
-
-/*
-returns the list of nodes belonging to the same proximity bin
-as the target. The most proximate bin will be the union of the bins between
-proxLimit and MaxProx.
-*/
-func (self *Kademlia) FindClosest(target Address, max int) []Node {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-
-	r := nodesByDistance{
-		target: target,
-	}
-
-	po := self.proximityBin(target)
-	index := po
-	step := 1
-	log.Trace(fmt.Sprintf("serving %v nodes at %v (PO%02d)", max, index, po))
-
-	// if max is set to 0, just want a full bucket, dynamic number
-	min := max
-	// set limit to max
-	limit := max
-	if max == 0 {
-		min = 1
-		limit = maxPeers
-	}
-
-	var n int
-	for index >= 0 {
-		// add entire bucket
-		for _, p := range self.buckets[index] {
-			r.push(p, limit)
-			n++
-		}
-		// terminate if index reached the bottom or enough peers > min
-		log.Trace(fmt.Sprintf("add %v -> %v (PO%02d, PO%03d)", len(self.buckets[index]), n, index, po))
-		if n >= min && (step < 0 || max == 0) {
-			break
-		}
-		// reach top most non-empty PO bucket, turn around
-		if index == self.MaxProx {
-			index = po
-			step = -1
-		}
-		index += step
-	}
-	log.Trace(fmt.Sprintf("serve %d (<=%d) nodes for target lookup %v (PO%03d)", n, max, target, po))
-	return r.nodes
-}
-
-func (self *Kademlia) Suggest() (*NodeRecord, bool, int) {
-	defer self.lock.RUnlock()
-	self.lock.RLock()
-	return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) })
-}
-
-//  adds node records to kaddb (persisted node record db)
-func (self *Kademlia) Add(nrs []*NodeRecord) {
-	self.db.add(nrs, self.proximityBin)
-}
-
-// nodesByDistance is a list of nodes, ordered by distance to target.
-type nodesByDistance struct {
-	nodes  []Node
-	target Address
-}
-
-func sortedByDistanceTo(target Address, slice []Node) bool {
-	var last Address
-	for i, node := range slice {
-		if i > 0 {
-			if target.ProxCmp(node.Addr(), last) < 0 {
-				return false
-			}
-		}
-		last = node.Addr()
-	}
-	return true
-}
-
-// push(node, max) adds the given node to the list, keeping the total size
-// below max elements.
-func (h *nodesByDistance) push(node Node, max int) {
-	// returns the firt index ix such that func(i) returns true
-	ix := sort.Search(len(h.nodes), func(i int) bool {
-		return h.target.ProxCmp(h.nodes[i].Addr(), node.Addr()) >= 0
-	})
-
-	if len(h.nodes) < max {
-		h.nodes = append(h.nodes, node)
-	}
-	if ix < len(h.nodes) {
-		copy(h.nodes[ix+1:], h.nodes[ix:])
-		h.nodes[ix] = node
-	}
-}
-
-/*
-Taking the proximity order relative to a fix point x classifies the points in
-the space (n byte long byte sequences) into bins. Items in each are at
-most half as distant from x as items in the previous bin. Given a sample of
-uniformly distributed items (a hash function over arbitrary sequence) the
-proximity scale maps onto series of subsets with cardinalities on a negative
-exponential scale.
-
-It also has the property that any two item belonging to the same bin are at
-most half as distant from each other as they are from x.
-
-If we think of random sample of items in the bins as connections in a network of interconnected nodes than relative proximity can serve as the basis for local
-decisions for graph traversal where the task is to find a route between two
-points. Since in every hop, the finite distance halves, there is
-a guaranteed constant maximum limit on the number of hops needed to reach one
-node from the other.
-*/
-
-func (self *Kademlia) proximityBin(other Address) (ret int) {
-	ret = proximity(self.addr, other)
-	if ret > self.MaxProx {
-		ret = self.MaxProx
-	}
-	return
-}
-
-// provides keyrange for chunk db iteration
-func (self *Kademlia) KeyRange(other Address) (start, stop Address) {
-	defer self.lock.RUnlock()
-	self.lock.RLock()
-	return KeyRange(self.addr, other, self.proxLimit)
-}
-
-// save persists kaddb on disk (written to file on path in json format.
-func (self *Kademlia) Save(path string, cb func(*NodeRecord, Node)) error {
-	return self.db.save(path, cb)
-}
-
-// Load(path) loads the node record database (kaddb) from file on path.
-func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err error) {
-	return self.db.load(path, cb)
-}
-
-// kademlia table + kaddb table displayed with ascii
-func (self *Kademlia) String() string {
-	defer self.lock.RUnlock()
-	self.lock.RLock()
-	defer self.db.lock.RUnlock()
-	self.db.lock.RLock()
-
-	var rows []string
-	rows = append(rows, "=========================================================================")
-	rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6]))
-	rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize))
-	rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize))
-
-	for i, bucket := range self.buckets {
-
-		if i == self.proxLimit {
-			rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i))
-		}
-		row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))}
-		var k int
-		c := self.db.cursors[i]
-		for ; k < len(bucket); k++ {
-			p := bucket[(c+k)%len(bucket)]
-			row = append(row, p.Addr().String()[:6])
-			if k == 4 {
-				break
-			}
-		}
-		for ; k < 4; k++ {
-			row = append(row, "      ")
-		}
-		row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i]))
-
-		for j, p := range self.db.Nodes[i] {
-			row = append(row, p.Addr.String()[:6])
-			if j == 3 {
-				break
-			}
-		}
-		rows = append(rows, strings.Join(row, " "))
-		if i == self.MaxProx {
-		}
-	}
-	rows = append(rows, "=========================================================================")
-	return strings.Join(rows, "\n")
-}

swarm/network/kademlia/kademlia_test.go

@@ -1,392 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package kademlia
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"reflect"
-	"testing"
-	"testing/quick"
-	"time"
-)
-
-var (
-	quickrand           = rand.New(rand.NewSource(time.Now().Unix()))
-	quickcfgFindClosest = &quick.Config{MaxCount: 50, Rand: quickrand}
-	quickcfgBootStrap   = &quick.Config{MaxCount: 100, Rand: quickrand}
-)
-
-type testNode struct {
-	addr Address
-}
-
-func (n *testNode) String() string {
-	return fmt.Sprintf("%x", n.addr[:])
-}
-
-func (n *testNode) Addr() Address {
-	return n.addr
-}
-
-func (n *testNode) Drop() {
-}
-
-func (n *testNode) Url() string {
-	return ""
-}
-
-func (n *testNode) LastActive() time.Time {
-	return time.Now()
-}
-
-func TestOn(t *testing.T) {
-	addr, ok1 := gen(Address{}, quickrand).(Address)
-	other, ok2 := gen(Address{}, quickrand).(Address)
-	if !ok1 || !ok2 {
-		t.Errorf("oops")
-	}
-	kad := New(addr, NewKadParams())
-	err := kad.On(&testNode{addr: other}, nil)
-	_ = err
-}
-
-func TestBootstrap(t *testing.T) {
-
-	test := func(test *bootstrapTest) bool {
-		// for any node kad.le, Target and N
-		params := NewKadParams()
-		params.MaxProx = test.MaxProx
-		params.BucketSize = test.BucketSize
-		params.ProxBinSize = test.BucketSize
-		kad := New(test.Self, params)
-		var err error
-
-		for p := 0; p < 9; p++ {
-			var nrs []*NodeRecord
-			n := math.Pow(float64(2), float64(7-p))
-			for i := 0; i < int(n); i++ {
-				addr := RandomAddressAt(test.Self, p)
-				nrs = append(nrs, &NodeRecord{
-					Addr: addr,
-				})
-			}
-			kad.Add(nrs)
-		}
-
-		node := &testNode{test.Self}
-
-		n := 0
-		for n < 100 {
-			err = kad.On(node, nil)
-			if err != nil {
-				t.Fatalf("backend not accepting node: %v", err)
-			}
-
-			record, need, _ := kad.Suggest()
-			if !need {
-				break
-			}
-			n++
-			if record == nil {
-				continue
-			}
-			node = &testNode{record.Addr}
-		}
-		exp := test.BucketSize * (test.MaxProx + 1)
-		if kad.Count() != exp {
-			t.Errorf("incorrect number of peers, expected %d, got %d\n%v", exp, kad.Count(), kad)
-			return false
-		}
-		return true
-	}
-	if err := quick.Check(test, quickcfgBootStrap); err != nil {
-		t.Error(err)
-	}
-
-}
-
-func TestFindClosest(t *testing.T) {
-
-	test := func(test *FindClosestTest) bool {
-		// for any node kad.le, Target and N
-		params := NewKadParams()
-		params.MaxProx = 7
-		kad := New(test.Self, params)
-		var err error
-		for _, node := range test.All {
-			err = kad.On(node, nil)
-			if err != nil && err.Error() != "bucket full" {
-				t.Fatalf("backend not accepting node: %v", err)
-			}
-		}
-
-		if len(test.All) == 0 || test.N == 0 {
-			return true
-		}
-		nodes := kad.FindClosest(test.Target, test.N)
-
-		// check that the number of results is min(N, kad.len)
-		wantN := test.N
-		if tlen := kad.Count(); tlen < test.N {
-			wantN = tlen
-		}
-
-		if len(nodes) != wantN {
-			t.Errorf("wrong number of nodes: got %d, want %d", len(nodes), wantN)
-			return false
-		}
-
-		if hasDuplicates(nodes) {
-			t.Errorf("result contains duplicates")
-			return false
-		}
-
-		if !sortedByDistanceTo(test.Target, nodes) {
-			t.Errorf("result is not sorted by distance to target")
-			return false
-		}
-
-		// check that the result nodes have minimum distance to target.
-		farthestResult := nodes[len(nodes)-1].Addr()
-		for i, b := range kad.buckets {
-			for j, n := range b {
-				if contains(nodes, n.Addr()) {
-					continue // don't run the check below for nodes in result
-				}
-				if test.Target.ProxCmp(n.Addr(), farthestResult) < 0 {
-					_ = i * j
-					t.Errorf("kad.le contains node that is closer to target but it's not in result")
-					return false
-				}
-			}
-		}
-		return true
-	}
-	if err := quick.Check(test, quickcfgFindClosest); err != nil {
-		t.Error(err)
-	}
-}
-
-type proxTest struct {
-	add   bool
-	index int
-	addr  Address
-}
-
-var (
-	addresses []Address
-)
-
-func TestProxAdjust(t *testing.T) {
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	self := gen(Address{}, r).(Address)
-	params := NewKadParams()
-	params.MaxProx = 7
-	kad := New(self, params)
-
-	var err error
-	for i := 0; i < 100; i++ {
-		a := gen(Address{}, r).(Address)
-		addresses = append(addresses, a)
-		err = kad.On(&testNode{addr: a}, nil)
-		if err != nil && err.Error() != "bucket full" {
-			t.Fatalf("backend not accepting node: %v", err)
-		}
-		if !kad.proxCheck(t) {
-			return
-		}
-	}
-	test := func(test *proxTest) bool {
-		node := &testNode{test.addr}
-		if test.add {
-			kad.On(node, nil)
-		} else {
-			kad.Off(node, nil)
-		}
-		return kad.proxCheck(t)
-	}
-	if err := quick.Check(test, quickcfgFindClosest); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestSaveLoad(t *testing.T) {
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	addresses := gen([]Address{}, r).([]Address)
-	self := RandomAddress()
-	params := NewKadParams()
-	params.MaxProx = 7
-	kad := New(self, params)
-
-	var err error
-
-	for _, a := range addresses {
-		err = kad.On(&testNode{addr: a}, nil)
-		if err != nil && err.Error() != "bucket full" {
-			t.Fatalf("backend not accepting node: %v", err)
-		}
-	}
-	nodes := kad.FindClosest(self, 100)
-
-	path := filepath.Join(os.TempDir(), "bzz-kad-test-save-load.peers")
-	err = kad.Save(path, nil)
-	if err != nil && err.Error() != "bucket full" {
-		t.Fatalf("unepected error saving kaddb: %v", err)
-	}
-	kad = New(self, params)
-	err = kad.Load(path, nil)
-	if err != nil && err.Error() != "bucket full" {
-		t.Fatalf("unepected error loading kaddb: %v", err)
-	}
-	for _, b := range kad.db.Nodes {
-		for _, node := range b {
-			err = kad.On(&testNode{node.Addr}, nil)
-			if err != nil && err.Error() != "bucket full" {
-				t.Fatalf("backend not accepting node: %v", err)
-			}
-		}
-	}
-	loadednodes := kad.FindClosest(self, 100)
-	for i, node := range loadednodes {
-		if nodes[i].Addr() != node.Addr() {
-			t.Errorf("node mismatch at %d/%d: %v != %v", i, len(nodes), nodes[i].Addr(), node.Addr())
-		}
-	}
-}
-
-func (self *Kademlia) proxCheck(t *testing.T) bool {
-	var sum int
-	for i, b := range self.buckets {
-		l := len(b)
-		// if we are in the high prox multibucket
-		if i >= self.proxLimit {
-			sum += l
-		} else if l == 0 {
-			t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self)
-			return false
-		}
-	}
-	// check if merged high prox bucket does not exceed size
-	if sum > 0 {
-		if sum != self.proxSize {
-			t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize)
-			return false
-		}
-		last := len(self.buckets[self.proxLimit])
-		if last > 0 && sum >= self.ProxBinSize+last {
-			t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)\n%v", self.proxLimit, last, sum-last, self.ProxBinSize, self)
-			return false
-		}
-		if self.proxLimit > 0 && sum < self.ProxBinSize {
-			t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize)
-			return false
-		}
-	}
-	return true
-}
-
-type bootstrapTest struct {
-	MaxProx    int
-	BucketSize int
-	Self       Address
-}
-
-func (*bootstrapTest) Generate(rand *rand.Rand, size int) reflect.Value {
-	t := &bootstrapTest{
-		Self:       gen(Address{}, rand).(Address),
-		MaxProx:    5 + rand.Intn(2),
-		BucketSize: rand.Intn(3) + 1,
-	}
-	return reflect.ValueOf(t)
-}
-
-type FindClosestTest struct {
-	Self   Address
-	Target Address
-	All    []Node
-	N      int
-}
-
-func (c FindClosestTest) String() string {
-	return fmt.Sprintf("A: %064x\nT: %064x\n(%d)\n", c.Self[:], c.Target[:], c.N)
-}
-
-func (*FindClosestTest) Generate(rand *rand.Rand, size int) reflect.Value {
-	t := &FindClosestTest{
-		Self:   gen(Address{}, rand).(Address),
-		Target: gen(Address{}, rand).(Address),
-		N:      rand.Intn(bucketSize),
-	}
-	for _, a := range gen([]Address{}, rand).([]Address) {
-		t.All = append(t.All, &testNode{addr: a})
-	}
-	return reflect.ValueOf(t)
-}
-
-func (*proxTest) Generate(rand *rand.Rand, size int) reflect.Value {
-	var add bool
-	if rand.Intn(1) == 0 {
-		add = true
-	}
-	var t *proxTest
-	if add {
-		t = &proxTest{
-			addr: gen(Address{}, rand).(Address),
-			add:  add,
-		}
-	} else {
-		t = &proxTest{
-			index: rand.Intn(len(addresses)),
-			add:   add,
-		}
-	}
-	return reflect.ValueOf(t)
-}
-
-func hasDuplicates(slice []Node) bool {
-	seen := make(map[Address]bool)
-	for _, node := range slice {
-		if seen[node.Addr()] {
-			return true
-		}
-		seen[node.Addr()] = true
-	}
-	return false
-}
-
-func contains(nodes []Node, addr Address) bool {
-	for _, n := range nodes {
-		if n.Addr() == addr {
-			return true
-		}
-	}
-	return false
-}
-
-// gen wraps quick.Value so it's easier to use.
-// it generates a random value of the given value's type.
-func gen(typ interface{}, rand *rand.Rand) interface{} {
-	v, ok := quick.Value(reflect.TypeOf(typ), rand)
-	if !ok {
-		panic(fmt.Sprintf("couldn't generate random value of type %T", typ))
-	}
-	return v.Interface()
-}

swarm/network/messages.go

@@ -1,317 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/ethereum/go-ethereum/contracts/chequebook"
-	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/swarm/network/kademlia"
-	"github.com/ethereum/go-ethereum/swarm/services/swap"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-/*
-BZZ protocol Message Types and Message Data Types
-*/
-
-// bzz protocol message codes
-const (
-	statusMsg          = iota // 0x01
-	storeRequestMsg           // 0x02
-	retrieveRequestMsg        // 0x03
-	peersMsg                  // 0x04
-	syncRequestMsg            // 0x05
-	deliveryRequestMsg        // 0x06
-	unsyncedKeysMsg           // 0x07
-	paymentMsg                // 0x08
-)
-
-/*
- Handshake
-
-* Version: 8 byte integer version of the protocol
-* ID: arbitrary byte sequence client identifier human readable
-* Addr: the address advertised by the node, format similar to DEVp2p wire protocol
-* Swap: info for the swarm accounting protocol
-* NetworkID: 8 byte integer network identifier
-* Caps: swarm-specific capabilities, format identical to devp2p
-* SyncState: syncronisation state (db iterator key and address space etc) persisted about the peer
-
-*/
-type statusMsgData struct {
-	Version   uint64
-	ID        string
-	Addr      *peerAddr
-	Swap      *swap.SwapProfile
-	NetworkId uint64
-}
-
-func (self *statusMsgData) String() string {
-	return fmt.Sprintf("Status: Version: %v, ID: %v, Addr: %v, Swap: %v, NetworkId: %v", self.Version, self.ID, self.Addr, self.Swap, self.NetworkId)
-}
-
-/*
- store requests are forwarded to the peers in their kademlia proximity bin
- if they are distant
- if they are within our storage radius or have any incentive to store it
- then attach your nodeID to the metadata
- if the storage request is sufficiently close (within our proxLimit, i. e., the
- last row of the routing table)
-*/
-type storeRequestMsgData struct {
-	Key   storage.Key // hash of datasize | data
-	SData []byte      // the actual chunk Data
-	// optional
-	Id             uint64     // request ID. if delivery, the ID is retrieve request ID
-	requestTimeout *time.Time // expiry for forwarding - [not serialised][not currently used]
-	storageTimeout *time.Time // expiry of content - [not serialised][not currently used]
-	from           *peer      // [not serialised] protocol registers the requester
-}
-
-func (self storeRequestMsgData) String() string {
-	var from string
-	if self.from == nil {
-		from = "self"
-	} else {
-		from = self.from.Addr().String()
-	}
-	end := len(self.SData)
-	if len(self.SData) > 10 {
-		end = 10
-	}
-	return fmt.Sprintf("from: %v, Key: %v; ID: %v, requestTimeout: %v, storageTimeout: %v, SData %x", from, self.Key, self.Id, self.requestTimeout, self.storageTimeout, self.SData[:end])
-}
-
-/*
-Retrieve request
-
-Timeout in milliseconds. Note that zero timeout retrieval requests do not request forwarding, but prompt for a peers message response. therefore they serve also
-as messages to retrieve peers.
-
-MaxSize specifies the maximum size that the peer will accept. This is useful in
-particular if we allow storage and delivery of multichunk payload representing
-the entire or partial subtree unfolding from the requested root key.
-So when only interested in limited part of a stream (infinite trees) or only
-testing chunk availability etc etc, we can indicate it by limiting the size here.
-
-Request ID can be newly generated or kept from the request originator.
-If request ID Is missing or zero, the request is handled as a lookup only
-prompting a peers response but not launching a search. Lookup requests are meant
-to be used to bootstrap kademlia tables.
-
-In the special case that the key is the zero value as well, the remote peer's
-address is assumed (the message is to be handled as a self lookup request).
-The response is a PeersMsg with the peers in the kademlia proximity bin
-corresponding to the address.
-*/
-
-type retrieveRequestMsgData struct {
-	Key      storage.Key // target Key address of chunk to be retrieved
-	Id       uint64      // request id, request is a lookup if missing or zero
-	MaxSize  uint64      // maximum size of delivery accepted
-	MaxPeers uint64      // maximum number of peers returned
-	Timeout  uint64      // the longest time we are expecting a response
-	timeout  *time.Time  // [not serialied]
-	from     *peer       //
-}
-
-func (self retrieveRequestMsgData) String() string {
-	var from string
-	if self.from == nil {
-		from = "ourselves"
-	} else {
-		from = self.from.Addr().String()
-	}
-	var target []byte
-	if len(self.Key) > 3 {
-		target = self.Key[:4]
-	}
-	return fmt.Sprintf("from: %v, Key: %x; ID: %v, MaxSize: %v, MaxPeers: %d", from, target, self.Id, self.MaxSize, self.MaxPeers)
-}
-
-// lookups are encoded by missing request ID
-func (self retrieveRequestMsgData) isLookup() bool {
-	return self.Id == 0
-}
-
-// sets timeout fields
-func (self retrieveRequestMsgData) setTimeout(t *time.Time) {
-	self.timeout = t
-	if t != nil {
-		self.Timeout = uint64(t.UnixNano())
-	} else {
-		self.Timeout = 0
-	}
-}
-
-func (self retrieveRequestMsgData) getTimeout() (t *time.Time) {
-	if self.Timeout > 0 && self.timeout == nil {
-		timeout := time.Unix(int64(self.Timeout), 0)
-		t = &timeout
-		self.timeout = t
-	}
-	return
-}
-
-// peerAddr is sent in StatusMsg as part of the handshake
-type peerAddr struct {
-	IP   net.IP
-	Port uint16
-	ID   []byte // the 64 byte NodeID (ECDSA Public Key)
-	Addr kademlia.Address
-}
-
-// peerAddr pretty prints as enode
-func (self peerAddr) String() string {
-	var nodeid discover.NodeID
-	copy(nodeid[:], self.ID)
-	return discover.NewNode(nodeid, self.IP, 0, self.Port).String()
-}
-
-/*
-peers Msg is one response to retrieval; it is always encouraged after a retrieval
-request to respond with a list of peers in the same kademlia proximity bin.
-The encoding of a peer is identical to that in the devp2p base protocol peers
-messages: [IP, Port, NodeID]
-note that a node's DPA address is not the NodeID but the hash of the NodeID.
-
-Timeout serves to indicate whether the responder is forwarding the query within
-the timeout or not.
-
-NodeID serves as the owner of payment contracts and signer of proofs of transfer.
-
-The Key is the target (if response to a retrieval request) or missing (zero value)
-peers address (hash of NodeID) if retrieval request was a self lookup.
-
-Peers message is requested by retrieval requests with a missing or zero value request ID
-*/
-type peersMsgData struct {
-	Peers   []*peerAddr //
-	Timeout uint64      //
-	timeout *time.Time  // indicate whether responder is expected to deliver content
-	Key     storage.Key // present if a response to a retrieval request
-	Id      uint64      // present if a response to a retrieval request
-	from    *peer
-}
-
-// peers msg pretty printer
-func (self peersMsgData) String() string {
-	var from string
-	if self.from == nil {
-		from = "ourselves"
-	} else {
-		from = self.from.Addr().String()
-	}
-	var target []byte
-	if len(self.Key) > 3 {
-		target = self.Key[:4]
-	}
-	return fmt.Sprintf("from: %v, Key: %x; ID: %v, Peers: %v", from, target, self.Id, self.Peers)
-}
-
-func (self peersMsgData) setTimeout(t *time.Time) {
-	self.timeout = t
-	if t != nil {
-		self.Timeout = uint64(t.UnixNano())
-	} else {
-		self.Timeout = 0
-	}
-}
-
-func (self peersMsgData) getTimeout() (t *time.Time) {
-	if self.Timeout > 0 && self.timeout == nil {
-		timeout := time.Unix(int64(self.Timeout), 0)
-		t = &timeout
-		self.timeout = t
-	}
-	return
-}
-
-/*
-syncRequest
-
-is sent after the handshake to initiate syncing
-the syncState of the remote node is persisted in kaddb and set on the
-peer/protocol instance when the node is registered by hive as online{
-*/
-
-type syncRequestMsgData struct {
-	SyncState *syncState `rlp:"nil"`
-}
-
-func (self *syncRequestMsgData) String() string {
-	return fmt.Sprintf("%v", self.SyncState)
-}
-
-/*
-deliveryRequest
-
-is sent once a batch of sync keys is filtered. The ones not found are
-sent as a list of syncReuest (hash, priority) in the Deliver field.
-When the source receives the sync request it continues to iterate
-and fetch at most N items as yet unsynced.
-At the same time responds with deliveries of the items.
-*/
-type deliveryRequestMsgData struct {
-	Deliver []*syncRequest
-}
-
-func (self *deliveryRequestMsgData) String() string {
-	return fmt.Sprintf("sync request for new chunks\ndelivery request for %v chunks", len(self.Deliver))
-}
-
-/*
-unsyncedKeys
-
-is sent first after the handshake if SyncState iterator brings up hundreds, thousands?
-and subsequently sent as a response to deliveryRequestMsgData.
-
-Syncing is the iterative process of exchanging unsyncedKeys and deliveryRequestMsgs
-both ways.
-
-State contains the sync state sent by the source. When the source receives the
-sync state it continues to iterate and fetch at most N items as yet unsynced.
-At the same time responds with deliveries of the items.
-*/
-type unsyncedKeysMsgData struct {
-	Unsynced []*syncRequest
-	State    *syncState
-}
-
-func (self *unsyncedKeysMsgData) String() string {
-	return fmt.Sprintf("sync: keys of %d new chunks (state %v) => synced: %v", len(self.Unsynced), self.State, self.State.Synced)
-}
-
-/*
-payment
-
-is sent when the swap balance is tilted in favour of the remote peer
-and in absolute units exceeds the PayAt parameter in the remote peer's profile
-*/
-
-type paymentMsgData struct {
-	Units   uint               // units actually paid for (checked against amount by swap)
-	Promise *chequebook.Cheque // payment with cheque
-}
-
-func (self *paymentMsgData) String() string {
-	return fmt.Sprintf("payment for %d units: %v", self.Units, self.Promise)
-}

swarm/network/pbtree.go

@@ -0,0 +1,603 @@
+package network
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+/*
+PbTree implements a pinned binary tree.
+It is a generic container type for objects implementing the PbVal interface
+Each item is pinned to the node at pos x such that all items pinned to all
+ancestor nodes share an at least x bits long key prefix
+
+PbTree
+* does not need to copy keys of the item type.
+* retrieval, insertion and deletion by key involves log(n) pointer lookups
+* for any item retrieval respects proximity order on logarithmic distance
+* provide syncronous iterators  respecting proximity ordering  wrt any item
+* provide asyncronous iterator (for parallel execution of operations) over n items
+* allows cheap iteration over ranges
+* TODO: asymmetric parallelisable merge
+
+*/
+// PbTree is the root node type k(same for root, branching node and leat)
+type PbTree struct {
+	lock sync.RWMutex
+	*pbTree
+}
+
+// pbTree is the node type (same for root, branching node and leat)
+type pbTree struct {
+	pin  PbVal
+	bins []*pbTree
+	size int
+	pos  int
+}
+
+// PbVal is the interface the generic container item should implement
+type PbVal interface {
+	Prefix(PbVal, int) (pos int, eq bool)
+	String() string
+}
+
+// PbTree constructor. Requires  value of type PbVal to pin
+// and pos to point to a span in the PbVal key
+// The pinned item counts towards the size
+func NewPbTree(v PbVal, pos int) *PbTree {
+	return &PbTree{
+		pbTree: &pbTree{
+			pin:  v,
+			pos:  pos,
+			size: 1,
+		},
+	}
+}
+
+// Pin() returns the pinned element (key) of the PbTree
+func (t *PbTree) Pin() PbVal {
+	return t.pin
+}
+
+// Size() returns the number of values in the PbTree
+func (t *PbTree) Size() int {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	return t.size
+}
+
+// Add(v) inserts v into the PbTree
+func (t *PbTree) Add(val PbVal) (pos int, found bool) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	return t.add(val)
+}
+
+func (t *pbTree) add(val PbVal) (pos int, found bool) {
+	if t.pin == nil {
+		t.pin = val
+		t.size = 1
+		return 0, false
+	}
+	pos, found = val.Prefix(t.pin, t.pos)
+	if found {
+		t.pin = val
+		return pos, true
+	}
+
+	n, j := t.getPos(pos)
+	if n != nil {
+		p, f := n.add(val)
+		if !f {
+			t.size++
+		}
+		return p, f
+	}
+	// insert empty sub-pbTree and pin it to val
+	ins := &pbTree{
+		pin:  val,
+		pos:  pos,
+		size: 1,
+	}
+	t.size++
+	t.bins = append(t.bins, nil)
+	copy(t.bins[j+1:], t.bins[j:])
+	t.bins[j] = ins
+	return pos, false
+}
+
+// Remove(v) deletes v from the PbTree and returns
+// the proximity order of v
+func (t *PbTree) Remove(val PbVal) (pos int, found bool) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	return t.remove(val)
+}
+
+func (t *pbTree) remove(val PbVal) (pos int, found bool) {
+	pos, found = val.Prefix(t.pin, t.pos)
+	if found {
+		t.size -= 1
+		if t.size == 0 {
+			t.pin = nil
+			return t.pos, true
+		}
+		i := len(t.bins) - 1
+		last := t.bins[i]
+		t.bins = append(t.bins[:i], last.bins...)
+		t.pin = last.pin
+		return t.pos, true
+	}
+	for j, n := range t.bins {
+		if n.pos == pos {
+			p, f := n.remove(val)
+			if f {
+				t.size--
+			}
+			last := len(t.bins) - 1
+			copy(t.bins[j:], t.bins[j+1:])
+			t.bins[last] = nil // or the zero value of T
+			t.bins = t.bins[:last]
+			return p, f
+		}
+		if n.pos > pos {
+			return 0, false
+		}
+	}
+	return 0, false
+}
+
+func (t *PbTree) Merge(t1 *PbTree) int {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	t1.lock.RLock()
+	defer t1.lock.RUnlock()
+	return t.merge(t1.pbTree)
+}
+
+func (t *pbTree) merge(t1 *pbTree) int {
+	if t.pin == nil {
+		if t1.pin == nil {
+			return 0
+		}
+		t.pin = t1.pin
+		t.size = 1
+		return t.merge(t1)
+	}
+	i := 0
+	j := 0
+	added := 0
+
+	var bins []*pbTree
+	var m, t2 *pbTree
+	var is []int
+	pos, _ := t.pin.Prefix(t1.pin, 0)
+	n, l := t.getPos(pos)
+	for {
+		glog.V(4).Infof("%v-%v, i: %v, j: %v, pos: %v, n: %v, l: %v", t, t1, i, j, pos, n, l)
+
+		if i == len(t.bins) && j == len(t1.bins) {
+			if l < len(t.bins) {
+				glog.V(4).Infof("l < len(t.bins): break")
+				break
+			}
+		}
+
+		if i == l && j <= len(t1.bins) {
+			if m == nil {
+				if n == nil {
+					n = &pbTree{
+						pin:  t1.pin,
+						size: 1,
+						pos:  pos,
+					}
+					added++
+				} else {
+					_, found := n.add(t1.pin)
+					if !found {
+						added++
+					}
+					glog.V(4).Infof("%v-%v: i: %v, j: %v. adding t1.pin (found: %v) to n: %v", t.pin, t1.pin, i, j, found, n)
+				}
+				m = n
+				bins = append(bins, n)
+			}
+			if j < len(t1.bins) {
+				if t2 == nil && t1.bins[j].pos == 0 {
+					t2 = t1.bins[j]
+					_, found := n.add(t2.pin)
+					if !found {
+						added++
+					}
+					glog.V(4).Infof("%v-%v: i: %v, j: %v. will merge into 0 the 0 pos branch from 1: %v", t.pin, t1.pin, i, j, t1.bins[j])
+					glog.V(4).Infof("%v-%v: i: %v, j: %v. adding t2.pin: %v (found: %v) to n: %v", t.pin, t1.pin, i, j, t2.pin, found, n)
+				} else {
+					glog.V(4).Infof("%v-%v: i: %v, j: %v. merge into 0 from 1: %v", t.pin, t1.pin, i, j, t1.bins[j])
+					added += n.merge(t1.bins[j])
+				}
+				j++
+				continue
+			}
+			if t2 == nil && l == len(t.bins) {
+				break
+			}
+			if l < len(t.bins) {
+				i++
+			}
+			glog.V(4).Infof("%v-%v: i: %v, j: %v. t1 reset to %v", t.pin, t1.pin, i, j, t2)
+			if t2 != nil {
+				t1 = t2
+				j = 0
+				m = nil
+				t2 = nil
+				pos, _ = t.pin.Prefix(t1.pin, 0)
+				n, l = t.getPos(pos)
+			}
+			continue
+		}
+
+		if j == len(t1.bins) || t1.bins[j].pos > t.bins[i].pos {
+			glog.V(4).Infof("%v-%v: i: %v, j: %v. insert from 0: %v", t.pin, t1.pin, i, j, t.bins[i])
+			bins = append(bins, t.bins[i])
+			i++
+			continue
+		}
+
+		if i < l && t1.bins[j].pos < t.bins[i].pos {
+			glog.V(4).Infof("%v-%v: i: %v, j: %v. insert from 1: %v", t.pin, t1.pin, i, j, t1.bins[j])
+			m := &pbTree{}
+			added += m.merge(t1.bins[j])
+			bins = append(bins, n)
+			j++
+			continue
+		}
+
+		glog.V(4).Infof("%v-%v: i: %v, j: %v. merge: %v", t.pin, t1.pin, i, j, t.bins[i], t1.bins[j])
+		bins = append(bins, t.bins[i])
+		is = append(is, i)
+		i++
+		j++
+	}
+	t.bins = bins
+	wg := sync.WaitGroup{}
+	if len(is) > 0 {
+		wg.Add(len(is))
+		for _, i := range is {
+			go func(k int) {
+				defer wg.Done()
+				is[k] = bins[k].merge(t1.bins[k])
+			}(i)
+		}
+		wg.Wait()
+		for _, a := range is {
+			added += a
+		}
+	}
+	glog.V(4).Infof("%v-%v: added: %v", t.pin, t1.pin, added)
+	t.size += added
+	return added
+}
+
+// func (t *PbTree) Traverse(f func(val PbVal, pos int) (next bool, fork bool)) *PbTree {
+//   t.lock.Lock()
+//   defer t.lock.Unlock()
+//   return t.traverse(f)
+// }
+
+// func (t *pbTree) traverse(n *pbTree, f func(val PbVal, pos int) (next bool, fork bool)) *PbTree {
+
+// next, stop = pinf(t.pin,t.pos)
+//  if !next
+// }
+
+// Each(f) is a synchronous iterator over the bins of a node
+// it does NOT include the pinned item of the root
+// respecting an ordering
+// proximity > pinnedness
+func (t *PbTree) Each(f func(PbVal, int) bool) bool {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	return t.each(f)
+}
+
+func (t *pbTree) each(f func(PbVal, int) bool) bool {
+	var next bool
+	for _, n := range t.bins {
+		next = n.each(f)
+		if !next {
+			return false
+		}
+	}
+	next = f(t.pin, t.pos)
+	if !next {
+		return false
+	}
+
+	return true
+}
+
+// syncronous iterator over neighbours of any target val
+// even if an item at val's exact address is in the pbtree,
+// it is not included in the iteration: $val \not\in Neighbours(val)$
+func (t *PbTree) EachNeighbour(val PbVal, f func(PbVal, int) bool) bool {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	return t.eachNeighbour(val, f)
+}
+
+func (t *pbTree) eachNeighbour(val PbVal, f func(PbVal, int) bool) bool {
+	var next bool
+	l := len(t.bins)
+	var n *pbTree
+	ir := l
+	il := l
+	pos, eq := val.Prefix(t.pin, t.pos)
+	if !eq {
+		n, il = t.getPos(pos)
+		if n != nil {
+			next = n.eachNeighbour(val, f)
+			if !next {
+				return false
+			}
+			ir = il
+		} else {
+			ir = il - 1
+		}
+	}
+
+	next = f(t.pin, pos)
+	if !next {
+		return false
+	}
+
+	for i := l - 1; i > ir; i-- {
+		next = t.bins[i].each(func(v PbVal, _ int) bool {
+			return f(v, pos)
+		})
+		if !next {
+			return false
+		}
+	}
+
+	for i := il - 1; i >= 0; i-- {
+		n := t.bins[i]
+		next = n.each(func(v PbVal, _ int) bool {
+			return f(v, n.pos)
+		})
+		if !next {
+			return false
+		}
+	}
+	return true
+}
+
+func (t *PbTree) EachNeighbourAsync(val PbVal, max int, maxPos int, f func(PbVal, int), wait bool) {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	if max > t.size {
+		max = t.size
+	}
+	var wg *sync.WaitGroup
+	if wait {
+		wg = &sync.WaitGroup{}
+	}
+	_ = t.eachNeighbourAsync(val, max, maxPos, f, wg)
+	if wait {
+		wg.Wait()
+	}
+}
+
+func (t *pbTree) eachNeighbourAsync(val PbVal, max int, maxPos int, f func(PbVal, int), wg *sync.WaitGroup) (extra int) {
+
+	l := len(t.bins)
+	var n *pbTree
+	il := l
+	ir := l
+	// ic := l
+
+	pos, eq := val.Prefix(t.pin, t.pos)
+	glog.V(4).Infof("pin %v: each neighbour iteration async. count: %v/%v, t.pos: %v, pos: %v, maxPos: %v", t.pin, max, t.size, t.pos, pos, maxPos)
+
+	// if pos is too close, set the pivot branch (pom) to maxPos
+	pom := pos
+	if pom > maxPos {
+		pom = maxPos
+	}
+	n, il = t.getPos(pom)
+	ir = il
+	// if pivot branch exists and pos is not too close, iterate on the pivot branch
+	if pom == pos {
+		if n != nil {
+
+			m := n.size
+			if max < m {
+				m = max
+			}
+			max -= m
+
+			glog.V(4).Infof("pin %v recursive branch %v pos: %v/%v (%v), count: %v/%v", t.pin, n.pin, n.pos, maxPos, il, m, max)
+
+			extra = n.eachNeighbourAsync(val, m, maxPos, f, wg)
+
+		} else {
+			if !eq {
+				ir--
+			}
+		}
+	} else {
+		extra++
+		max--
+		if n != nil {
+			il++
+		}
+		// before checking max, add up the extra elements
+		// on the close branches that are skipped (if pos is too close)
+		for i := l - 1; i >= il; i-- {
+			s := t.bins[i]
+			m := s.size
+			if max < m {
+				m = max
+			}
+			max -= m
+			extra += m
+		}
+		glog.V(4).Infof("count extra pos: %v/%v -> %v", pos, maxPos, extra)
+	}
+	glog.V(4).Infof("branch %v: %v/%v, il: %v, ir: %v, l: %v, extra: %v", t.pin, pos, maxPos, il, ir, l, extra)
+
+	var m int
+	// if max <= 0 {
+	// 	return
+	// }
+	// unless pos was too close, call f on the pinned element
+	if pom == pos {
+
+		glog.V(4).Infof("pinned val %v, t.pos: %v, pos: %v (%v), count: %v, max: %v", t.pin, pos, maxPos, "pin", 1, max)
+		glog.V(4).Infof("BEFORE %v %v %v", 1, max, extra)
+		m, max, extra = need(1, max, extra)
+		if m <= 0 {
+			return
+		}
+		glog.V(4).Infof("AFTER %v %v %v", 1, max, extra)
+		glog.V(4).Infof("pinned val %v, t.pos: %v, pos: %v (%v), count: %v, max: %v", t.pin, pos, maxPos, "pin", 1, max)
+
+		if wg != nil {
+			wg.Add(1)
+		}
+		go func() {
+			if wg != nil {
+				defer wg.Done()
+			}
+			f(t.pin, pos)
+		}()
+
+		// otherwise iterats
+		glog.V(4).Infof("closer branches %v: %v/%v, il: %v, ir: %v, l: %v", t.pin, pos, maxPos, il, ir, l)
+		for i := l - 1; i > ir; i-- {
+			n := t.bins[i]
+
+			glog.V(4).Infof("branch %v closer branch %v pos: %v/%v (%v), count: %v, size: %v, max: %v", t.pin, n.pin, pos, maxPos, i, m, n.size, max)
+			glog.V(4).Infof("BEFORE %v %v %v", n.size, max, extra)
+			m, max, extra = need(n.size, max, extra)
+			if m <= 0 {
+				glog.V(4).Infof("branch %v closer branch %v NOT ADDED pos: %v/%v (%v), count: %v, size: %v, max: %v", t.pin, n.pin, pos, maxPos, i, m, n.size, max)
+				return
+			}
+			glog.V(4).Infof("AFTER %v %v %v", m, max, extra)
+
+			glog.V(4).Infof("branch %v closer branch %v pos: %v/%v (%v), count: %v, size: %v, max: %v", t.pin, n.pin, pos, maxPos, i, m, n.size, max)
+
+			if wg != nil {
+				wg.Add(m)
+			}
+			go func(pn *pbTree, pm int) {
+				pn.each(func(v PbVal, _ int) bool {
+					if wg != nil {
+						defer wg.Done()
+					}
+					glog.V(4).Infof("branch %v call f on %v pos: %v/%v (%v), count: %v/%v", pn.pin, v, pos, maxPos, i, pm, max)
+					f(v, pos)
+					pm--
+					return pm > 0
+				})
+			}(n, m)
+
+		}
+	}
+
+	// if max <= 0 {
+	// 	return
+	// }
+	// iterate branches that are farther tham pom with their own po
+	glog.V(4).Infof("further branches %v: %v/%v, il: %v, ir: %v, l: %v, extra: %v", t.pin, pos, maxPos, il, ir, l, extra)
+	for i := il - 1; i >= 0; i-- {
+		n := t.bins[i]
+		// the first time max is less than the size of the entire branch
+		// wait for the pivot thread to release extra elements
+		glog.V(4).Infof("branch %v further branch %v pos: %v/%v (%v), count: %v, size: %v, max: %v", t.pin, n.pin, n.pos, maxPos, i, m, n.size, max)
+		glog.V(4).Infof("BEFORE %v %v %v", n.size, max, extra)
+		m, max, extra = need(n.size, max, extra)
+		if m <= 0 {
+			return
+		}
+		glog.V(4).Infof("AFTER %v %v %v", m, max, extra)
+		glog.V(4).Infof("branch %v further branch %v pos: %v/%v (%v), count: %v, size: %v, max: %v", t.pin, n.pin, n.pos, maxPos, i, m, n.size, max)
+
+		if wg != nil {
+			wg.Add(m)
+		}
+		go func(pn *pbTree, pm int) {
+			pn.each(func(v PbVal, _ int) bool {
+				if wg != nil {
+					defer wg.Done()
+				}
+				f(v, pn.pos)
+				glog.V(4).Infof("branch %v call f on %v pos: %v/%v (%v), count: %v/%v", pn.pin, v, pn.pos, maxPos, i, pm, max)
+				pm--
+				return pm > 0
+			})
+		}(n, m)
+
+	}
+	return max + extra
+
+}
+
+// getPos(n) returns the forking node at PO n and its index if it exists
+// otherwise nil
+// caller is supposed to hold the lock
+func (t *pbTree) getPos(pos int) (n *pbTree, i int) {
+	for i, n = range t.bins {
+		if pos > n.pos {
+			continue
+		}
+		if pos < n.pos {
+			return nil, i
+		}
+		return n, i
+	}
+	return nil, len(t.bins)
+}
+
+// need(m, max, extra) uses max m out of extra, and then max
+// if needed, returns the adjusted counts
+func need(m, max, extra int) (int, int, int) {
+	if m <= extra {
+		return m, max, extra - m
+	}
+	max += extra - m
+	if max <= 0 {
+		return m + max, 0, 0
+	}
+	return m, max, 0
+}
+
+// func need(max int, more chan int) int {
+// 	// if max <= 0 {
+// 	c, ok := <-more
+// 	if ok {
+// 		defer close(more)
+// 		if c > 0 {
+// 			glog.V(4).Infof("need: %v + %v", max, c)
+// 			return max + c
+// 		}
+// 	}
+// 	// }
+// 	return max
+// }
+
+func (t *pbTree) String() string {
+	return t.sstring("")
+}
+
+func (t *pbTree) sstring(indent string) string {
+	var s string
+	indent += "  "
+	s += fmt.Sprintf("%v%v (%v) %v \n", indent, t.pin, t.pos, t.size)
+	for _, n := range t.bins {
+		s += fmt.Sprintf("%v%v\n", indent, n.sstring(indent))
+	}
+	return s
+}

swarm/network/pbtree_test.go

@@ -0,0 +1,511 @@
+package network
+
+import (
+	"errors"
+	"fmt"
+	"math/rand"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+func init() {
+	glog.SetV(4)
+	glog.SetToStderr(true)
+}
+
+type testAddr struct {
+	*BinAddr
+	i int
+}
+
+func NewTestAddr(s string, i int) *testAddr {
+	return &testAddr{NewBinAddr(s), i}
+}
+
+func str(v PbVal) string {
+	if v == nil {
+		return ""
+	}
+	return v.(*testAddr).String()
+}
+
+func indexes(t *PbTree) (i []int, pos []int) {
+	t.Each(func(v PbVal, po int) bool {
+		a := v.(*testAddr)
+		i = append(i, a.i)
+		pos = append(pos, po)
+		return true
+	})
+	return i, pos
+}
+
+func add(t *PbTree, n int, values ...string) {
+	for i, val := range values {
+		t.Add(NewTestAddr(val, i+n))
+	}
+}
+
+func (self *testAddr) Prefix(val PbVal, pos int) (po int, eq bool) {
+	return self.BinAddr.Prefix(val.(*testAddr).BinAddr, pos)
+}
+
+// func RandomBinAddr()
+func TestPbTreeAdd(t *testing.T) {
+	n := NewPbTree(NewTestAddr("001111", 0), 0)
+	// Pin set correctly
+	exp := "001111"
+	got := str(n.Pin())
+	if got != exp {
+		t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got)
+	}
+	// check size
+	goti := n.Size()
+	expi := 1
+	if goti != expi {
+		t.Fatalf("incorrect number of elements in PbTree. Expected %v, got %v", expi, goti)
+	}
+
+	add(n, 1, "011111", "001111", "011111", "000111")
+	// check size
+	goti = n.Size()
+	expi = 3
+	if goti != expi {
+		t.Fatalf("incorrect number of elements in PbTree. Expected %v, got %v", expi, goti)
+	}
+	inds, pos := indexes(n)
+	got = fmt.Sprintf("%v", inds)
+	exp = "[3 4 2]"
+	if got != exp {
+		t.Fatalf("incorrect indexes in iteration over PbTree. Expected %v, got %v", exp, got)
+	}
+	got = fmt.Sprintf("%v", pos)
+	exp = "[1 2 0]"
+	if got != exp {
+		t.Fatalf("incorrect po-s in iteration over PbTree. Expected %v, got %v", exp, got)
+	}
+}
+
+// func RandomBinAddr()
+func TestPbTreeRemove(t *testing.T) {
+	n := NewPbTree(NewTestAddr("001111", 0), 0)
+	n.Remove(NewTestAddr("001111", 0))
+	exp := ""
+	got := str(n.Pin())
+	if got != exp {
+		t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got)
+	}
+	add(n, 1, "000000", "011111", "001111", "000111")
+	n.Remove(NewTestAddr("001111", 0))
+	goti := n.Size()
+	expi := 3
+	if goti != expi {
+		t.Fatalf("incorrect number of elements in PbTree. Expected %v, got %v", expi, goti)
+	}
+	inds, pos := indexes(n)
+	got = fmt.Sprintf("%v", inds)
+	exp = "[2 4 1]"
+	if got != exp {
+		t.Fatalf("incorrect indexes in iteration over PbTree. Expected %v, got %v", exp, got)
+	}
+	got = fmt.Sprintf("%v", pos)
+	exp = "[1 3 0]"
+	if got != exp {
+		t.Fatalf("incorrect po-s in iteration over PbTree. Expected %v, got %v", exp, got)
+	}
+	// remove again
+	n.Remove(NewTestAddr("001111", 0))
+	inds, pos = indexes(n)
+	got = fmt.Sprintf("%v", inds)
+	exp = "[2 4 1]"
+	if got != exp {
+		t.Fatalf("incorrect indexes in iteration over PbTree. Expected %v, got %v", exp, got)
+	}
+
+}
+
+func checkPo(val PbVal) func(PbVal, int) error {
+	return func(v PbVal, po int) error {
+		// check the po
+		exp, _ := val.Prefix(v, 0)
+		if po != exp {
+			return fmt.Errorf("incorrect prox order for item %v in neighbour iteration for %v. Expected %v, got %v", v, val, exp, po)
+		}
+		return nil
+	}
+}
+
+func checkOrder(val PbVal) func(PbVal, int) error {
+	var pos int = keylen
+	return func(v PbVal, po int) error {
+		if pos < po {
+			return fmt.Errorf("incorrect order for item %v in neighbour iteration for %v. PO %v > %v (previous max)", v, val, po, pos)
+		}
+		pos = po
+		return nil
+	}
+}
+
+func checkValues(m map[string]bool, val PbVal) func(PbVal, int) error {
+	return func(v PbVal, po int) error {
+		duplicate, ok := m[v.String()]
+		if !ok {
+			return fmt.Errorf("alien value %v", v)
+		}
+		if duplicate {
+			return fmt.Errorf("duplicate value returned: %v", v)
+		}
+		m[v.String()] = true
+		return nil
+	}
+}
+
+var errNoCount = errors.New("not count")
+
+func testPbTreeEachNeighbour(n *PbTree, val PbVal, expCount int, fs ...func(PbVal, int) error) error {
+	var err error
+	var count int
+	n.EachNeighbour(val, func(v PbVal, po int) bool {
+		for _, f := range fs {
+			err = f(v, po)
+			if err != nil {
+				return err.Error() == errNoCount.Error()
+			}
+		}
+		count++
+		if count == expCount {
+			return false
+		}
+		return true
+	})
+	if err == nil && count < expCount {
+		return fmt.Errorf("not enough neighbours returned, expected %v, got %v", expCount, count)
+	}
+	return err
+}
+
+const (
+	maxEachNeighbourTests = 500
+	maxEachNeighbour      = 4
+	keylen                = 4
+)
+
+func randomTestAddr(n int, i int) *testAddr {
+	v := RandomAddress().Bin()[:n]
+	return NewTestAddr(v, i)
+}
+
+func TestPbTreeMerge(t *testing.T) {
+	for i := 0; i < maxEachNeighbourTests; i++ {
+		max0 := rand.Intn(maxEachNeighbour) + 1
+		max1 := rand.Intn(maxEachNeighbour) + 1
+		n0 := NewPbTree(nil, 0)
+		n1 := NewPbTree(nil, 0)
+		m := make(map[string]bool)
+		for j := 0; j < max0; {
+			v := randomTestAddr(keylen, j)
+			_, found := n0.Add(v)
+			if !found {
+				glog.V(4).Infof("%v: add %v", j, v)
+				m[v.String()] = false
+				j++
+			}
+		}
+		expAdded := 0
+
+		for j := 0; j < max1; {
+			v := randomTestAddr(keylen, j)
+			_, found := n1.Add(v)
+			glog.V(4).Infof("%v: add %v", j, v)
+			if !found {
+				j++
+			}
+			_, found = m[v.String()]
+			if !found {
+				expAdded++
+				glog.V(4).Infof("%v: newly added %v", j-1, v)
+				m[v.String()] = false
+			}
+		}
+		expSize := len(m)
+
+		glog.V(4).Infof("%v-0: pin: %v, size: %v", i, n0.Pin(), max0)
+		glog.V(4).Infof("%v-1: pin: %v, size: %v", i, n1.Pin(), max1)
+		glog.V(4).Infof("%v: %v", i, expSize)
+		added := n0.Merge(n1)
+		size := n0.Size()
+		if expSize != size {
+			t.Fatalf("incorrect number of elements in merged pbTree, expected %v, got %v\n%v", expSize, size, n0)
+		}
+		if expAdded != added {
+			t.Fatalf("incorrect number of added elements in merged pbTree, expected %v, got %v", expAdded, added)
+		}
+		for k, _ := range m {
+			_, found := n0.Add(NewTestAddr(k, 0))
+			if !found {
+				t.Fatalf("merged pbTree missing element %v", k)
+			}
+		}
+	}
+}
+
+func TestPbTreeEachNeighbourSync(t *testing.T) {
+	for i := 0; i < maxEachNeighbourTests; i++ {
+		max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2
+		pin := randomTestAddr(keylen, 0)
+		n := NewPbTree(pin, 0)
+		m := make(map[string]bool)
+		m[pin.String()] = false
+		for j := 1; j <= max; j++ {
+			v := randomTestAddr(keylen, j)
+			n.Add(v)
+			m[v.String()] = false
+		}
+
+		size := n.Size()
+		if size < 2 {
+			continue
+		}
+		count := rand.Intn(size/2) + size/2
+		val := randomTestAddr(keylen, max+1)
+		glog.V(4).Infof("%v: pin: %v, size: %v, val: %v, count: %v", i, n.Pin(), size, val, count)
+		err := testPbTreeEachNeighbour(n, val, count, checkPo(val), checkOrder(val), checkValues(m, val))
+		if err != nil {
+			t.Fatal(err)
+		}
+		minPoFound := keylen
+		maxPoNotFound := 0
+		for k, found := range m {
+			po, _ := val.Prefix(NewTestAddr(k, 0), 0)
+			if found {
+				if po < minPoFound {
+					minPoFound = po
+				}
+			} else {
+				if po > maxPoNotFound {
+					maxPoNotFound = po
+				}
+			}
+		}
+		if minPoFound < maxPoNotFound {
+			t.Fatalf("incorrect neighbours returned: found one with PO %v < there was one not found with PO %v", minPoFound, maxPoNotFound)
+		}
+	}
+}
+
+func TestPbTreeEachNeighbourAsync(t *testing.T) {
+	for i := 0; i < maxEachNeighbourTests; i++ {
+		max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2
+		n := NewPbTree(randomTestAddr(keylen, 0), 0)
+		var size int = 1
+		for j := 1; j <= max; j++ {
+			v := randomTestAddr(keylen, j)
+			_, found := n.Add(v)
+			if !found {
+				size++
+			}
+		}
+		if size != n.Size() {
+			t.Fatal(n)
+		}
+		if size < 2 {
+			continue
+		}
+		count := rand.Intn(size/2) + size/2
+		val := randomTestAddr(keylen, max+1)
+
+		mu := sync.Mutex{}
+		m := make(map[string]bool)
+		maxPos := rand.Intn(keylen)
+		glog.V(5).Infof("%v: pin: %v, size: %v, val: %v, count: %v, maxPos: %v", i, n.Pin(), size, val, count, maxPos)
+		msize := 0
+		remember := func(v PbVal, po int) error {
+			// mu.Lock()
+			// defer mu.Unlock()
+			if po > maxPos {
+				// glog.V(4).Infof("NOT ADD %v", v)
+				return errNoCount
+			}
+			// glog.V(4).Infof("ADD %v, %v", v, msize)
+			m[v.String()] = true
+			msize++
+			return nil
+		}
+		if i == 0 {
+			continue
+		}
+		err := testPbTreeEachNeighbour(n, val, count, remember)
+		if err != nil {
+			glog.V(6).Info(err)
+		}
+		d := 0
+		forget := func(v PbVal, po int) {
+			mu.Lock()
+			defer mu.Unlock()
+			d++
+			// glog.V(4).Infof("DEL %v", v)
+			delete(m, v.String())
+		}
+
+		n.EachNeighbourAsync(val, count, maxPos, forget, true)
+		if d != msize {
+			t.Fatalf("incorrect number of neighbour calls in async iterator. expected %v, got %v", msize, d)
+		}
+		if len(m) != 0 {
+
+			t.Fatalf("incorrect neighbour calls in async iterator. %v items missed:\n%v", len(m), n)
+		}
+	}
+}
+
+func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) {
+	t.ReportAllocs()
+	pin := randomTestAddr(keylen, 0)
+	n := NewPbTree(pin, 0)
+	for j := 1; j <= max; {
+		v := randomTestAddr(keylen, j)
+		_, found := n.Add(v)
+		if !found {
+			j++
+		}
+	}
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		val := randomTestAddr(keylen, max+1)
+		m := 0
+		n.EachNeighbour(val, func(v PbVal, po int) bool {
+			time.Sleep(d)
+			m++
+			if m == count {
+				return false
+			}
+			return true
+		})
+	}
+	t.StopTimer()
+	stats := new(runtime.MemStats)
+	runtime.ReadMemStats(stats)
+	// fmt.Println(stats.Sys)
+}
+
+func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) {
+	t.ReportAllocs()
+	pin := randomTestAddr(keylen, 0)
+	n := NewPbTree(pin, 0)
+	for j := 1; j <= max; {
+		v := randomTestAddr(keylen, j)
+		_, found := n.Add(v)
+		if !found {
+			j++
+		}
+	}
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		val := randomTestAddr(keylen, max+1)
+		n.EachNeighbourAsync(val, count, keylen, func(v PbVal, po int) {
+			time.Sleep(d)
+		}, true)
+	}
+	t.StopTimer()
+	stats := new(runtime.MemStats)
+	runtime.ReadMemStats(stats)
+	// fmt.Println(stats.Sys)
+}
+
+func BenchmarkEachNeighbourSync_3_1_0(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 10, 1*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_1_0(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 10, 1*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_2_0(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 100, 1*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_2_0(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 100, 1*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_3_0(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 1000, 1*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_3_0(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 1000, 1*time.Microsecond)
+}
+
+func BenchmarkEachNeighbourSync_3_1_1(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 10, 2*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_1_1(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 10, 2*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_2_1(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 100, 2*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_2_1(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 100, 2*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_3_1(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 1000, 2*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_3_1(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 1000, 2*time.Microsecond)
+}
+
+func BenchmarkEachNeighbourSync_3_1_2(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 10, 4*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_1_2(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 10, 4*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_2_2(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 100, 4*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_2_2(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 100, 4*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_3_2(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 1000, 4*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_3_2(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 1000, 4*time.Microsecond)
+}
+
+func BenchmarkEachNeighbourSync_3_1_3(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 10, 8*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_1_3(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 10, 8*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_2_3(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 100, 8*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_2_3(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 100, 8*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_3_3(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 1000, 8*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_3_3(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 1000, 8*time.Microsecond)
+}
+
+func BenchmarkEachNeighbourSync_3_1_4(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 10, 16*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_1_4(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 10, 16*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_2_4(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 100, 16*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_2_4(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 100, 16*time.Microsecond)
+}
+func BenchmarkEachNeighbourSync_3_3_4(t *testing.B) {
+	benchmarkEachNeighbourSync(t, 1000, 1000, 16*time.Microsecond)
+}
+func BenchmarkEachNeighboursAsync_3_3_4(t *testing.B) {
+	benchmarkEachNeighbourAsync(t, 1000, 1000, 16*time.Microsecond)
+}

swarm/network/protocol.go

@@ -16,498 +16,242 @@
 
 package network
 
-/*
-bzz implements the swarm wire protocol [bzz] (sister of eth and shh)
-the protocol instance is launched on each peer by the network layer if the
-bzz protocol handler is registered on the p2p server.
-
-The bzz protocol component speaks the bzz protocol
-* handle the protocol handshake
-* register peers in the KΛÐΞMLIΛ table via the hive logistic manager
-* dispatch to hive for handling the DHT logic
-* encode and decode requests for storage and retrieval
-* handle sync protocol messages via the syncer
-* talks the SWAP payment protocol (swap accounting is done within NetStore)
-*/
-
 import (
-	"errors"
 	"fmt"
-	"net"
-	"strconv"
 	"time"
 
-	"github.com/ethereum/go-ethereum/contracts/chequebook"
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	bzzswap "github.com/ethereum/go-ethereum/swarm/services/swap"
-	"github.com/ethereum/go-ethereum/swarm/services/swap/swap"
-	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/p2p/protocols"
 )
 
 const (
+	ProtocolName       = "bzz"
 	Version            = 0
-	ProtocolLength     = uint64(8)
+	NetworkId          = 322 // BZZ in l33t
 	ProtocolMaxMsgSize = 10 * 1024 * 1024
-	NetworkId          = 3
 )
 
-// bzz represents the swarm wire protocol
-// an instance is running on each peer
+// bzz is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer)
 type bzz struct {
-	selfID     discover.NodeID      // peer's node id used in peer advertising in handshake
-	key        storage.Key          // baseaddress as storage.Key
-	storage    StorageHandler       // handler storage/retrieval related requests coming via the bzz wire protocol
-	hive       *Hive                // the logistic manager, peerPool, routing service and peer handler
-	dbAccess   *DbAccess            // access to db storage counter and iterator for syncing
-	requestDb  *storage.LDBDatabase // db to persist backlog of deliveries to aid syncing
-	remoteAddr *peerAddr            // remote peers address
-	peer       *p2p.Peer            // the p2p peer object
-	rw         p2p.MsgReadWriter    // messageReadWriter to send messages to
-	backend    chequebook.Backend
+	*protocols.Peer
+	hive       PeerPool
+	network    adapters.NodeAdapter
+	localAddr  *peerAddr
+	*peerAddr  // remote address
 	lastActive time.Time
-	NetworkId  uint64
-
-	swap        *swap.Swap          // swap instance for the peer connection
-	swapParams  *bzzswap.SwapParams // swap settings both local and remote
-	swapEnabled bool                // flag to enable SWAP (will be set via Caps in handshake)
-	syncEnabled bool                // flag to enable SYNC (will be set via Caps in handshake)
-	syncer      *syncer             // syncer instance for the peer connection
-	syncParams  *SyncParams         // syncer params
-	syncState   *syncState          // outgoing syncronisation state (contains reference to remote peers db counter)
 }
 
-// interface type for handler of storage/retrieval related requests coming
-// via the bzz wire protocol
-// messages: UnsyncedKeys, DeliveryRequest, StoreRequest, RetrieveRequest
-type StorageHandler interface {
-	HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error
-	HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error
-	HandleStoreRequestMsg(req *storeRequestMsgData, p *peer)
-	HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer)
+func (self *bzz) LastActive() time.Time {
+	return self.lastActive
 }
 
-/*
-main entrypoint, wrappers starting a server that will run the bzz protocol
-use this constructor to attach the protocol ("class") to server caps
-This is done by node.Node#Register(func(node.ServiceContext) (Service, error))
-Service implements Protocols() which is an array of protocol constructors
-at node startup the protocols are initialised
-the Dev p2p layer then calls Run(p *p2p.Peer, rw p2p.MsgReadWriter) error
-on each peer connection
-The Run function of the Bzz protocol class creates a bzz instance
-which will represent the peer for the swarm hive and all peer-aware components
-*/
-func Bzz(cloud StorageHandler, backend chequebook.Backend, hive *Hive, dbaccess *DbAccess, sp *bzzswap.SwapParams, sy *SyncParams, networkId uint64) (p2p.Protocol, error) {
-
-	// a single global request db is created for all peer connections
-	// this is to persist delivery backlog and aid syncronisation
-	requestDb, err := storage.NewLDBDatabase(sy.RequestDbPath)
-	if err != nil {
-		return p2p.Protocol{}, fmt.Errorf("error setting up request db: %v", err)
-	}
-	if networkId == 0 {
-		networkId = NetworkId
-	}
-	return p2p.Protocol{
-		Name:    "bzz",
-		Version: Version,
-		Length:  ProtocolLength,
-		Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
-			return run(requestDb, cloud, backend, hive, dbaccess, sp, sy, networkId, p, rw)
-		},
-	}, nil
+// implemented by peerAddr and peerAddr
+type NodeAddr interface {
+	OverlayAddr() []byte
+	UnderlayAddr() []byte
 }
 
-/*
-the main protocol loop that
- * does the handshake by exchanging statusMsg
- * if peer is valid and accepted, registers with the hive
- * then enters into a forever loop handling incoming messages
- * storage and retrieval related queries coming via bzz are dispatched to StorageHandler
- * peer-related messages are dispatched to the hive
- * payment related messages are relayed to SWAP service
- * on disconnect, unregister the peer in the hive (note RemovePeer in the post-disconnect hook)
- * whenever the loop terminates, the peer will disconnect with Subprotocol error
- * whenever handlers return an error the loop terminates
-*/
-func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend chequebook.Backend, hive *Hive, dbaccess *DbAccess, sp *bzzswap.SwapParams, sy *SyncParams, networkId uint64, p *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
+// the Node interface that peerPool needs
+type Node interface {
+	NodeAddr
+	String() string      // pretty printable the Node
+	ID() discover.NodeID // the key that uniquely identifies the Node for the peerPool
 
-	self := &bzz{
-		storage:     depo,
-		backend:     backend,
-		hive:        hive,
-		dbAccess:    dbaccess,
-		requestDb:   requestDb,
-		peer:        p,
-		rw:          rw,
-		swapParams:  sp,
-		syncParams:  sy,
-		swapEnabled: hive.swapEnabled,
-		syncEnabled: true,
-		NetworkId:   networkId,
-	}
+	Send(interface{}) error                             // can send messages
+	Drop()                                              // disconnect this peer
+	Register(interface{}, func(interface{}) error) uint // register message-handler callbacks
+}
 
+// PeerPool is the interface for the connectivity manager
+// directly interacts with the p2p server to suggest connections
+type PeerPool interface {
+	Add(Node) error
+	Remove(Node)
+}
+
+type PeerInfo interface {
+	Info() interface{}
+	PeerInfo(discover.NodeID) interface{}
+}
+
+func BzzCodeMap(msgs ...interface{}) *protocols.CodeMap {
+	ct := protocols.NewCodeMap(ProtocolName, Version, ProtocolMaxMsgSize)
+	ct.Register(&bzzHandshake{})
+	ct.Register(msgs...)
+	return ct
+}
+
+// Bzz is the protocol constructor
+// returns p2p.Protocol that is to be offered by the node.Service
+func Bzz(localAddr []byte, hive PeerPool, na adapters.NodeAdapter, m adapters.Messenger, ct *protocols.CodeMap, services func(Node) error) *p2p.Protocol {
 	// handle handshake
-	err = self.handleStatus()
+
+	run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
+		glog.V(6).Infof("protocol starting on %v connected to %v", localAddr, p.ID())
+
+		id := p.ID()
+		peer := protocols.NewPeer(p, rw, ct, m, func() { na.Disconnect(id[:]) })
+		addr := &peerAddr{localAddr, na.LocalAddr()}
+
+		bee := &bzz{Peer: peer, hive: hive, network: na, localAddr: addr}
+		// protocol handshake and its validation
+		// sets remote peer address
+		err := bee.bzzHandshake()
 		if err != nil {
+			glog.V(6).Infof("handshake error in peer %v: %v", bee.ID(), err)
 			return err
 		}
-	defer func() {
-		// if the handler loop exits, the peer is disconnecting
-		// deregister the peer in the hive
-		self.hive.removePeer(&peer{bzz: self})
-		if self.syncer != nil {
-			self.syncer.stop() // quits request db and delivery loops, save requests
-		}
-		if self.swap != nil {
-			self.swap.Stop() // quits chequebox autocash etc
-		}
-	}()
 
-	// the main forever loop that handles incoming requests
-	for {
-		if self.hive.blockRead {
-			log.Warn(fmt.Sprintf("Cannot read network"))
-			time.Sleep(100 * time.Millisecond)
-			continue
-		}
-		err = self.handle()
-		if err != nil {
-			return
-		}
-	}
-}
-
-// TODO: may need to implement protocol drop only? don't want to kick off the peer
-// if they are useful for other protocols
-func (self *bzz) Drop() {
-	self.peer.Disconnect(p2p.DiscSubprotocolError)
-}
-
-// one cycle of the main forever loop that handles and dispatches incoming messages
-func (self *bzz) handle() error {
-	msg, err := self.rw.ReadMsg()
-	log.Debug(fmt.Sprintf("<- %v", msg))
+		// mount external service models on the peer connection (swap, sync)
+		if services != nil {
+			err = services(bee)
 			if err != nil {
+				glog.V(6).Infof("protocol service error for peer %v: %v", bee.ID(), err)
 				return err
 			}
-	if msg.Size > ProtocolMaxMsgSize {
-		return fmt.Errorf("message too long: %v > %v", msg.Size, ProtocolMaxMsgSize)
 		}
-	// make sure that the payload has been fully consumed
-	defer msg.Discard()
 
-	switch msg.Code {
-
-	case statusMsg:
-		// no extra status message allowed. The one needed already handled by
-		// handleStatus
-		log.Debug(fmt.Sprintf("Status message: %v", msg))
-		return errors.New("extra status message")
-
-	case storeRequestMsg:
-		// store requests are dispatched to netStore
-		var req storeRequestMsgData
-		if err := msg.Decode(&req); err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
-		}
-		if n := len(req.SData); n < 9 {
-			return fmt.Errorf("<- %v: Data too short (%v)", msg, n)
-		}
-		// last Active time is set only when receiving chunks
-		self.lastActive = time.Now()
-		log.Trace(fmt.Sprintf("incoming store request: %s", req.String()))
-		// swap accounting is done within forwarding
-		self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self})
-
-	case retrieveRequestMsg:
-		// retrieve Requests are dispatched to netStore
-		var req retrieveRequestMsgData
-		if err := msg.Decode(&req); err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
-		}
-		req.from = &peer{bzz: self}
-		// if request is lookup and not to be delivered
-		if req.isLookup() {
-			log.Trace(fmt.Sprintf("self lookup for %v: responding with peers only...", req.from))
-		} else if req.Key == nil {
-			return fmt.Errorf("protocol handler: req.Key == nil || req.Timeout == nil")
-		} else {
-			// swap accounting is done within netStore
-			self.storage.HandleRetrieveRequestMsg(&req, &peer{bzz: self})
-		}
-		// direct response with peers, TODO: sort this out
-		self.hive.peers(&req)
-
-	case peersMsg:
-		// response to lookups and immediate response to retrieve requests
-		// dispatches new peer data to the hive that adds them to KADDB
-		var req peersMsgData
-		if err := msg.Decode(&req); err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
-		}
-		req.from = &peer{bzz: self}
-		log.Trace(fmt.Sprintf("<- peer addresses: %v", req))
-		self.hive.HandlePeersMsg(&req, &peer{bzz: self})
-
-	case syncRequestMsg:
-		var req syncRequestMsgData
-		if err := msg.Decode(&req); err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
-		}
-		log.Debug(fmt.Sprintf("<- sync request: %v", req))
-		self.lastActive = time.Now()
-		self.sync(req.SyncState)
-
-	case unsyncedKeysMsg:
-		// coming from parent node offering
-		var req unsyncedKeysMsgData
-		if err := msg.Decode(&req); err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
-		}
-		log.Debug(fmt.Sprintf("<- unsynced keys : %s", req.String()))
-		err := self.storage.HandleUnsyncedKeysMsg(&req, &peer{bzz: self})
-		self.lastActive = time.Now()
+		err = hive.Add(bee)
 		if err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
+			glog.V(6).Infof("failed to add peer '%v' to hive: %v", bee.ID(), err)
+			return err
 		}
 
-	case deliveryRequestMsg:
-		// response to syncKeysMsg hashes filtered not existing in db
-		// also relays the last synced state to the source
-		var req deliveryRequestMsgData
-		if err := msg.Decode(&req); err != nil {
-			return fmt.Errorf("<-msg %v: %v", msg, err)
-		}
-		log.Debug(fmt.Sprintf("<- delivery request: %s", req.String()))
-		err := self.storage.HandleDeliveryRequestMsg(&req, &peer{bzz: self})
-		self.lastActive = time.Now()
-		if err != nil {
-			return fmt.Errorf("<- %v: %v", msg, err)
+		defer hive.Remove(bee)
+		return bee.Run()
 	}
 
-	case paymentMsg:
-		// swap protocol message for payment, Units paid for, Cheque paid with
-		if self.swapEnabled {
-			var req paymentMsgData
-			if err := msg.Decode(&req); err != nil {
-				return fmt.Errorf("<- %v: %v", msg, err)
-			}
-			log.Debug(fmt.Sprintf("<- payment: %s", req.String()))
-			self.swap.Receive(int(req.Units), req.Promise)
+	var info func() interface{}
+
+	var peerInfo func(discover.NodeID) interface{}
+
+	if o, ok := hive.(PeerInfo); ok {
+		info = o.Info
+		peerInfo = o.PeerInfo
 	}
 
-	default:
-		// no other message is allowed
-		return fmt.Errorf("invalid message code: %v", msg.Code)
+	return &p2p.Protocol{
+		Name:     ProtocolName,
+		Version:  Version,
+		Length:   ct.Length(),
+		Run:      run,
+		NodeInfo: info,
+		PeerInfo: peerInfo,
 	}
-	return nil
 }
 
-func (self *bzz) handleStatus() (err error) {
+/*
+ Handshake
 
-	handshake := &statusMsgData{
+* Version: 8 byte integer version of the protocol
+* NetworkID: 8 byte integer network identifier
+* Addr: the address advertised by the node including underlay and overlay connecctions
+*/
+type bzzHandshake struct {
+	Version   uint64
+	NetworkId uint64
+	Addr      *peerAddr
+}
+
+func (self *bzzHandshake) String() string {
+	return fmt.Sprintf("Handshake: Version: %v, NetworkId: %v, Addr: %v", self.Version, self.NetworkId, self.Addr)
+}
+
+type peerAddr struct {
+	OAddr []byte
+	UAddr []byte
+}
+
+func (self *peerAddr) OverlayAddr() []byte {
+	return self.OAddr
+}
+
+func (self *peerAddr) UnderlayAddr() []byte {
+	return self.UAddr
+}
+
+func (self *peerAddr) String() string {
+	return fmt.Sprintf("%x <%x>", self.OAddr, self.UAddr)
+}
+
+// bzzHandshake negotiates the bzz master handshake
+// and validates the response, returns error when
+// mismatch/incompatibility is evident
+func (self *bzz) bzzHandshake() error {
+
+	lhs := &bzzHandshake{
 		Version:   uint64(Version),
-		ID:        "honey",
-		Addr:      self.selfAddr(),
-		NetworkId: uint64(self.NetworkId),
-		Swap: &bzzswap.SwapProfile{
-			Profile:    self.swapParams.Profile,
-			PayProfile: self.swapParams.PayProfile,
-		},
+		NetworkId: uint64(NetworkId),
+		Addr:      self.localAddr,
 	}
 
-	err = p2p.Send(self.rw, statusMsg, handshake)
+	hs, err := self.Handshake(lhs)
+	if err != nil {
+		glog.V(6).Infof("handshake failed: %v", err)
+		return err
+	}
+
+	rhs := hs.(*bzzHandshake)
+	err = checkBzzHandshake(rhs)
+	if err != nil {
+		glog.V(6).Infof("handshake between %v and %v  failed: %v", self.localAddr, self.peerAddr)
+		return err
+	}
+
+	addr := rhs.Addr
+	// Addr returns the remote address of the network connection.
+	// with rlpx use this to set adverrtised IP
+	self.localAddr.UAddr, err = self.network.ParseAddr(self.localAddr.UAddr, self.RemoteAddr().String())
 	if err != nil {
 		return err
 	}
 
-	// read and handle remote status
-	var msg p2p.Msg
-	msg, err = self.rw.ReadMsg()
-	if err != nil {
-		return err
+	glog.V(logger.Debug).Infof("self: advertised net address: %x, local address: %v\npeer: advertised: %v, remote address: %v\n", self.network.LocalAddr(), self.LocalAddr(), NodeId(addr), self.RemoteAddr())
+	self.peerAddr = addr
+	return nil
+
+}
+
+func checkBzzHandshake(rhs *bzzHandshake) error {
+
+	if NetworkId != rhs.NetworkId {
+		return fmt.Errorf("network id mismatch %d (!= %d)", rhs.NetworkId, NetworkId)
 	}
 
-	if msg.Code != statusMsg {
-		return fmt.Errorf("first msg has code %x (!= %x)", msg.Code, statusMsg)
+	if Version != rhs.Version {
+		return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, Version)
 	}
 
-	if msg.Size > ProtocolMaxMsgSize {
-		return fmt.Errorf("message too long: %v > %v", msg.Size, ProtocolMaxMsgSize)
-	}
-
-	var status statusMsgData
-	if err := msg.Decode(&status); err != nil {
-		return fmt.Errorf("<- %v: %v", msg, err)
-	}
-
-	if status.NetworkId != self.NetworkId {
-		return fmt.Errorf("network id mismatch: %d (!= %d)", status.NetworkId, self.NetworkId)
-	}
-
-	if Version != status.Version {
-		return fmt.Errorf("protocol version mismatch: %d (!= %d)", status.Version, Version)
-	}
-
-	self.remoteAddr = self.peerAddr(status.Addr)
-	log.Trace(fmt.Sprintf("self: advertised IP: %v, peer advertised: %v, local address: %v\npeer: advertised IP: %v, remote address: %v\n", self.selfAddr(), self.remoteAddr, self.peer.LocalAddr(), status.Addr.IP, self.peer.RemoteAddr()))
-
-	if self.swapEnabled {
-		// set remote profile for accounting
-		self.swap, err = bzzswap.NewSwap(self.swapParams, status.Swap, self.backend, self)
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Info(fmt.Sprintf("Peer %08x is capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId))
-	err = self.hive.addPeer(&peer{bzz: self})
-	if err != nil {
-		return err
-	}
-
-	// hive sets syncstate so sync should start after node added
-	log.Info(fmt.Sprintf("syncronisation request sent with %v", self.syncState))
-	self.syncRequest()
-
 	return nil
 }
 
-func (self *bzz) sync(state *syncState) error {
-	// syncer setup
-	if self.syncer != nil {
-		return errors.New("sync request can only be sent once")
-	}
-
-	cnt := self.dbAccess.counter()
-	remoteaddr := self.remoteAddr.Addr
-	start, stop := self.hive.kad.KeyRange(remoteaddr)
-
-	// an explicitly received nil syncstate disables syncronisation
-	if state == nil {
-		self.syncEnabled = false
-		log.Warn(fmt.Sprintf("syncronisation disabled for peer %v", self))
-		state = &syncState{DbSyncState: &storage.DbSyncState{}, Synced: true}
-	} else {
-		state.synced = make(chan bool)
-		state.SessionAt = cnt
-		if storage.IsZeroKey(state.Stop) && state.Synced {
-			state.Start = storage.Key(start[:])
-			state.Stop = storage.Key(stop[:])
-		}
-		log.Debug(fmt.Sprintf("syncronisation requested by peer %v at state %v", self, state))
-	}
-	var err error
-	self.syncer, err = newSyncer(
-		self.requestDb,
-		storage.Key(remoteaddr[:]),
-		self.dbAccess,
-		self.unsyncedKeys, self.store,
-		self.syncParams, state, func() bool { return self.syncEnabled },
-	)
+func RandomAddr() *peerAddr {
+	key, err := crypto.GenerateKey()
 	if err != nil {
-		return nil
+		panic("unable to generate key")
 	}
-	log.Trace(fmt.Sprintf("syncer set for peer %v", self))
-	return nil
-}
-
-func (self *bzz) String() string {
-	return self.remoteAddr.String()
-}
-
-// repair reported address if IP missing
-func (self *bzz) peerAddr(base *peerAddr) *peerAddr {
-	if base.IP.IsUnspecified() {
-		host, _, _ := net.SplitHostPort(self.peer.RemoteAddr().String())
-		base.IP = net.ParseIP(host)
+	pubkey := crypto.FromECDSAPub(&key.PublicKey)
+	var id discover.NodeID
+	copy(id[:], pubkey[1:])
+	return &peerAddr{
+		OAddr: crypto.Keccak256(pubkey[1:]),
+		UAddr: id[:],
 	}
-	return base
 }
 
-// returns self advertised node connection info (listening address w enodes)
-// IP will get repaired on the other end if missing
-// or resolved via ID by discovery at dialout
-func (self *bzz) selfAddr() *peerAddr {
-	id := self.hive.id
-	host, port, _ := net.SplitHostPort(self.hive.listenAddr())
-	intport, _ := strconv.Atoi(port)
-	addr := &peerAddr{
-		Addr: self.hive.addr,
-		ID:   id[:],
-		IP:   net.ParseIP(host),
-		Port: uint16(intport),
+func NodeId(addr NodeAddr) *adapters.NodeId {
+	return adapters.NewNodeId(addr.UnderlayAddr())
+}
+
+func NodeIdToAddr(n *adapters.NodeId) *peerAddr {
+	id := n.NodeID
+	return &peerAddr{
+		OAddr: crypto.Keccak256(id[:]),
+		UAddr: id[:],
 	}
-	return addr
-}
-
-// outgoing messages
-// send retrieveRequestMsg
-func (self *bzz) retrieve(req *retrieveRequestMsgData) error {
-	return self.send(retrieveRequestMsg, req)
-}
-
-// send storeRequestMsg
-func (self *bzz) store(req *storeRequestMsgData) error {
-	return self.send(storeRequestMsg, req)
-}
-
-func (self *bzz) syncRequest() error {
-	req := &syncRequestMsgData{}
-	if self.hive.syncEnabled {
-		log.Debug(fmt.Sprintf("syncronisation request to peer %v at state %v", self, self.syncState))
-		req.SyncState = self.syncState
-	}
-	if self.syncState == nil {
-		log.Warn(fmt.Sprintf("syncronisation disabled for peer %v at state %v", self, self.syncState))
-	}
-	return self.send(syncRequestMsg, req)
-}
-
-// queue storeRequestMsg in request db
-func (self *bzz) deliveryRequest(reqs []*syncRequest) error {
-	req := &deliveryRequestMsgData{
-		Deliver: reqs,
-	}
-	return self.send(deliveryRequestMsg, req)
-}
-
-// batch of syncRequests to send off
-func (self *bzz) unsyncedKeys(reqs []*syncRequest, state *syncState) error {
-	req := &unsyncedKeysMsgData{
-		Unsynced: reqs,
-		State:    state,
-	}
-	return self.send(unsyncedKeysMsg, req)
-}
-
-// send paymentMsg
-func (self *bzz) Pay(units int, promise swap.Promise) {
-	req := &paymentMsgData{uint(units), promise.(*chequebook.Cheque)}
-	self.payment(req)
-}
-
-// send paymentMsg
-func (self *bzz) payment(req *paymentMsgData) error {
-	return self.send(paymentMsg, req)
-}
-
-// sends peersMsg
-func (self *bzz) peers(req *peersMsgData) error {
-	return self.send(peersMsg, req)
-}
-
-func (self *bzz) send(msg uint64, data interface{}) error {
-	if self.hive.blockWrite {
-		return fmt.Errorf("network write blocked")
-	}
-	log.Trace(fmt.Sprintf("-> %v: %v (%T) to %v", msg, data, data, self))
-	err := p2p.Send(self.rw, msg, data)
-	if err != nil {
-		self.Drop()
-	}
-	return err
 }

swarm/network/protocol_test.go

@@ -1,17 +1,247 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
 package network
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/protocols"
+	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
+)
+
+func bzzHandshakeExchange(lhs, rhs *bzzHandshake, id *adapters.NodeId) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		p2ptest.Exchange{
+			Expects: []p2ptest.Expect{
+				p2ptest.Expect{
+					Code: 0,
+					Msg:  lhs,
+					Peer: id,
+				},
+			},
+		},
+		p2ptest.Exchange{
+			Triggers: []p2ptest.Trigger{
+				p2ptest.Trigger{
+					Code: 0,
+					Msg:  rhs,
+					Peer: id,
+				},
+			},
+		},
+	}
+}
+
+func newTestBzzProtocol(addr *peerAddr, pp PeerPool, ct *protocols.CodeMap, services func(Node) error) func(adapters.NodeAdapter) adapters.ProtoCall {
+	if ct == nil {
+		ct = BzzCodeMap()
+	}
+	ct.Register(p2ptest.FlushMsg)
+	return func(na adapters.NodeAdapter) adapters.ProtoCall {
+		srv := func(p Node) error {
+			if services != nil {
+				err := services(p)
+				if err != nil {
+					return err
+				}
+			}
+			id := p.ID()
+			p.Register(p2ptest.FlushMsg, func(interface{}) error {
+				flushc := na.(p2ptest.TestNetAdapter).GetPeer(&adapters.NodeId{id}).Flushc
+				flushc <- true
+				return nil
+			})
+			return nil
+		}
+
+		protocol := Bzz(addr.OverlayAddr(), pp, na, na.Messenger(), ct, srv)
+		return protocol.Run
+	}
+}
+
+type bzzTester struct {
+	*p2ptest.ExchangeSession
+	flushCode int
+	addr      *peerAddr
+}
+
+// should test handshakes in one exchange? parallelisation
+func (s *bzzTester) testHandshake(lhs, rhs *bzzHandshake, disconnects ...*p2ptest.Disconnect) {
+	var peers []*adapters.NodeId
+	id := NodeId(rhs.Addr)
+	if len(disconnects) > 0 {
+		for _, d := range disconnects {
+			peers = append(peers, d.Peer)
+		}
+	} else {
+		peers = []*adapters.NodeId{id}
+	}
+	s.TestConnected(false, peers...)
+	s.TestExchanges(bzzHandshakeExchange(lhs, rhs, id)...)
+	s.TestDisconnected(disconnects...)
+}
+
+func (s *bzzTester) flush(ids ...*adapters.NodeId) {
+	s.Flush(s.flushCode, ids...)
+}
+
+func (s *bzzTester) runHandshakes(ids ...*adapters.NodeId) {
+	if len(ids) == 0 {
+		ids = s.Ids
+	}
+	for _, id := range ids {
+		glog.V(6).Infof("\n\n\nrun handshake with %v", id)
+		time.Sleep(1)
+		s.testHandshake(correctBzzHandshake(s.addr), correctBzzHandshake(NodeIdToAddr(id)))
+		time.Sleep(1)
+	}
+	glog.V(6).Infof("flush %v", ids)
+	s.flush(ids...)
+}
+
+func correctBzzHandshake(addr *peerAddr) *bzzHandshake {
+	return &bzzHandshake{0, 322, addr}
+}
+
+func newBzzTester(t *testing.T, addr *peerAddr, pp PeerPool, ct *protocols.CodeMap, services func(Node) error) *bzzTester {
+	s := p2ptest.NewProtocolTester(t, NodeId(addr), 1, newTestBzzProtocol(addr, pp, ct, services))
+	return &bzzTester{
+		addr:            addr,
+		flushCode:       1,
+		ExchangeSession: s,
+	}
+}
+
+func TestBzzHandshakeNetworkIdMismatch(t *testing.T) {
+	pp := NewTestPeerPool()
+	addr := RandomAddr()
+	s := newBzzTester(t, addr, pp, nil, nil)
+	id := s.Ids[0]
+	s.testHandshake(
+		correctBzzHandshake(addr),
+		&bzzHandshake{0, 321, NodeIdToAddr(id)},
+		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("network id mismatch 321 (!= 322)")},
+	)
+}
+
+func TestBzzHandshakeVersionMismatch(t *testing.T) {
+	pp := NewTestPeerPool()
+	addr := RandomAddr()
+	s := newBzzTester(t, addr, pp, nil, nil)
+	id := s.Ids[0]
+	s.testHandshake(
+		correctBzzHandshake(addr),
+		&bzzHandshake{1, 322, NodeIdToAddr(id)},
+		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("version mismatch 1 (!= 0)")},
+	)
+}
+
+func TestBzzHandshakeSuccess(t *testing.T) {
+	pp := NewTestPeerPool()
+	addr := RandomAddr()
+	s := newBzzTester(t, addr, pp, nil, nil)
+	id := s.Ids[0]
+	s.testHandshake(
+		correctBzzHandshake(addr),
+		&bzzHandshake{0, 322, NodeIdToAddr(id)},
+	)
+}
+
+func TestBzzPeerPoolAdd(t *testing.T) {
+	pp := NewTestPeerPool()
+	addr := RandomAddr()
+	s := newBzzTester(t, addr, pp, nil, nil)
+
+	id := s.Ids[0]
+	glog.V(6).Infof("handshake with %v", id)
+	s.runHandshakes()
+	if !pp.Has(id) {
+		t.Fatalf("peer '%v' not added: %v", id, pp)
+	}
+}
+
+func TestBzzPeerPoolRemove(t *testing.T) {
+	addr := RandomAddr()
+	pp := NewTestPeerPool()
+	s := newBzzTester(t, addr, pp, nil, nil)
+	s.runHandshakes()
+
+	id := s.Ids[0]
+	pp.Get(id).Drop()
+	s.TestDisconnected(&p2ptest.Disconnect{id, fmt.Errorf("p2p: read or write on closed message pipe")})
+	if pp.Has(id) {
+		t.Fatalf("peer '%v' not removed: %v", id, pp)
+	}
+}
+
+func TestBzzPeerPoolBothAddRemove(t *testing.T) {
+	addr := RandomAddr()
+	pp := NewTestPeerPool()
+	s := newBzzTester(t, addr, pp, nil, nil)
+	s.runHandshakes()
+
+	id := s.Ids[0]
+	if !pp.Has(id) {
+		t.Fatalf("peer '%v' not added: %v", id, pp)
+	}
+
+	pp.Get(id).Drop()
+	s.TestDisconnected(&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("p2p: read or write on closed message pipe")})
+	if pp.Has(id) {
+		t.Fatalf("peer '%v' not removed: %v", id, pp)
+	}
+}
+
+func TestBzzPeerPoolNotAdd(t *testing.T) {
+	addr := RandomAddr()
+	pp := NewTestPeerPool()
+	s := newBzzTester(t, addr, pp, nil, nil)
+
+	id := s.Ids[0]
+	s.testHandshake(correctBzzHandshake(addr), &bzzHandshake{0, 321, NodeIdToAddr(id)}, &p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("network id mismatch 321 (!= 322)")})
+	if pp.Has(id) {
+		t.Fatalf("peer %v incorrectly added: %v", id, pp)
+	}
+}
+
+// TestPeerPool is an example peerPool to demonstrate registration of peer connections
+type TestPeerPool struct {
+	lock  sync.Mutex
+	peers map[discover.NodeID]Node
+}
+
+func NewTestPeerPool() *TestPeerPool {
+	return &TestPeerPool{peers: make(map[discover.NodeID]Node)}
+}
+
+func (self *TestPeerPool) Add(p Node) error {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	self.peers[p.ID()] = p
+	return nil
+}
+
+func (self *TestPeerPool) Remove(p Node) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	// glog.V(6).Infof("removing peer %v", p.ID())
+	delete(self.peers, p.ID())
+}
+
+func (self *TestPeerPool) Has(n *adapters.NodeId) bool {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	_, ok := self.peers[n.NodeID]
+	return ok
+}
+
+func (self *TestPeerPool) Get(n *adapters.NodeId) Node {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.peers[n.NodeID]
+}

swarm/network/simulations/overlay.go

@@ -0,0 +1,154 @@
+// +build none
+
+// You can run this simulation using
+//
+//    go run ./swarm/network/simulations/overlay.go
+package main
+
+import (
+	"fmt"
+	"reflect"
+	"runtime"
+	"time"
+
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p/adapters"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
+	"github.com/ethereum/go-ethereum/swarm/network"
+)
+
+// Network extends simulations.Network with hives for each node.
+type Network struct {
+	*simulations.Network
+	hives     []*network.Hive
+	messenger *adapters.SimPipe
+}
+
+// SimNode is the adapter used by Swarm simulations.
+type SimNode struct {
+	hive *network.Hive
+	adapters.NodeAdapter
+}
+
+// the hive update ticker for hive
+func af() <-chan time.Time {
+	return time.NewTicker(5 * time.Second).C
+}
+
+// Start() starts up the hive
+// makes SimNode implement *NodeAdapter
+func (self *SimNode) Start() error {
+	connect := func(s string) error {
+		id := network.HexToBytes(s)
+		return self.Connect(id)
+	}
+	return self.hive.Start(connect, af)
+}
+
+// Stop() shuts down the hive
+// makes SimNode implement *NodeAdapter
+func (self *SimNode) Stop() error {
+	self.hive.Stop()
+	return nil
+}
+
+// NewSimNode creates adapters for nodes in the simulation.
+func (self *Network) NewSimNode(conf *simulations.NodeConfig) adapters.NodeAdapter {
+	id := conf.Id
+	na := adapters.NewSimNode(id, self.Network, self.messenger)
+	addr := network.NodeIdToAddr(id)
+	to := network.NewTestOverlay(addr.OverlayAddr())   // overlay topology driver
+	pp := network.NewHive(network.NewHiveParams(), to) // hive
+	self.hives = append(self.hives, pp)                // remember hive
+	// bzz protocol Run function. messaging through SimPipe
+	ct := network.BzzCodeMap(network.HiveMsgs...) // bzz protocol code map
+	na.Run = network.Bzz(addr.OverlayAddr(), pp, na, &adapters.SimPipe{}, ct, nil).Run
+	return &SimNode{
+		hive:        pp,
+		NodeAdapter: na,
+	}
+}
+
+func NewNetwork(network *simulations.Network, messenger *adapters.SimPipe) *Network {
+	n := &Network{
+		// hives:
+		Network:   network,
+		messenger: messenger,
+	}
+	n.SetNaf(n.NewSimNode)
+	return n
+}
+
+// NewSessionController sits as the top-most controller for this simulation
+// creates an inprocess simulation of basic node running their own bzz+hive
+func NewSessionController() (*simulations.ResourceController, chan bool) {
+	quitc := make(chan bool)
+	return simulations.NewResourceContoller(
+		&simulations.ResourceHandlers{
+			// POST /
+			Create: &simulations.ResourceHandler{
+				Handle: func(msg interface{}, parent *simulations.ResourceController) (interface{}, error) {
+					conf := msg.(*simulations.NetworkConfig)
+					messenger := &adapters.SimPipe{}
+					net := simulations.NewNetwork(nil, &event.TypeMux{})
+					ppnet := NewNetwork(net, messenger)
+					c := simulations.NewNetworkController(conf, net.Events(), simulations.NewJournal())
+					if len(conf.Id) == 0 {
+						conf.Id = fmt.Sprintf("%d", 0)
+					}
+					glog.V(6).Infof("new network controller on %v", conf.Id)
+					if parent != nil {
+						parent.SetResource(conf.Id, c)
+					}
+					ids := p2ptest.RandomNodeIds(10)
+					for _, id := range ids {
+						ppnet.NewNode(&simulations.NodeConfig{Id: id})
+						ppnet.Start(id)
+						glog.V(6).Infof("node %v starting up", id)
+					}
+					// the nodes only know about their 2 neighbours (cyclically)
+					for i, _ := range ids {
+						var peerId *adapters.NodeId
+						if i == 0 {
+							peerId = ids[len(ids)-1]
+						} else {
+							peerId = ids[i-1]
+						}
+						err := ppnet.hives[i].Register(network.NodeIdToAddr(peerId))
+						if err != nil {
+							panic(err.Error())
+						}
+					}
+					return struct{}{}, nil
+				},
+				Type: reflect.TypeOf(&simulations.NetworkConfig{}),
+				// Type: reflect.TypeOf(&simulations.NetworkConfig{}),
+			},
+			// DELETE /
+			Destroy: &simulations.ResourceHandler{
+				Handle: func(msg interface{}, parent *simulations.ResourceController) (interface{}, error) {
+					glog.V(6).Infof("destroy handler called")
+					// this can quit the entire app (shut down the backend server)
+					quitc <- true
+					return struct{}{}, nil
+				},
+			},
+		},
+	), quitc
+}
+
+// var server
+func main() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	glog.SetV(6)
+	glog.SetToStderr(true)
+
+	c, quitc := NewSessionController()
+
+	simulations.StartRestApiServer("8888", c)
+	// wait until server shuts down
+	<-quitc
+
+}

swarm/network/syncdb.go

@@ -1,389 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"encoding/binary"
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-)
-
-const counterKeyPrefix = 0x01
-
-/*
-syncDb is a queueing service for outgoing deliveries.
-One instance per priority queue for each peer
-
-a syncDb instance maintains an in-memory buffer (of capacity bufferSize)
-once its in-memory buffer is full it switches to persisting in db
-and dbRead iterator iterates through the items keeping their order
-once the db read catches up (there is no more items in the db) then
-it switches back to in-memory buffer.
-
-when syncdb is stopped all items in the buffer are saved to the db
-*/
-type syncDb struct {
-	start          []byte               // this syncdb starting index in requestdb
-	key            storage.Key          // remote peers address key
-	counterKey     []byte               // db key to persist counter
-	priority       uint                 // priotity High|Medium|Low
-	buffer         chan interface{}     // incoming request channel
-	db             *storage.LDBDatabase // underlying db (TODO should be interface)
-	done           chan bool            // chan to signal goroutines finished quitting
-	quit           chan bool            // chan to signal quitting to goroutines
-	total, dbTotal int                  // counts for one session
-	batch          chan chan int        // channel for batch requests
-	dbBatchSize    uint                 // number of items before batch is saved
-}
-
-// constructor needs a shared request db (leveldb)
-// priority is used in the index key
-// uses a buffer and a leveldb for persistent storage
-// bufferSize, dbBatchSize are config parameters
-func newSyncDb(db *storage.LDBDatabase, key storage.Key, priority uint, bufferSize, dbBatchSize uint, deliver func(interface{}, chan bool) bool) *syncDb {
-	start := make([]byte, 42)
-	start[1] = byte(priorities - priority)
-	copy(start[2:34], key)
-
-	counterKey := make([]byte, 34)
-	counterKey[0] = counterKeyPrefix
-	copy(counterKey[1:], start[1:34])
-
-	syncdb := &syncDb{
-		start:       start,
-		key:         key,
-		counterKey:  counterKey,
-		priority:    priority,
-		buffer:      make(chan interface{}, bufferSize),
-		db:          db,
-		done:        make(chan bool),
-		quit:        make(chan bool),
-		batch:       make(chan chan int),
-		dbBatchSize: dbBatchSize,
-	}
-	log.Trace(fmt.Sprintf("syncDb[peer: %v, priority: %v] - initialised", key.Log(), priority))
-
-	// starts the main forever loop reading from buffer
-	go syncdb.bufferRead(deliver)
-	return syncdb
-}
-
-/*
-bufferRead is a forever iterator loop that takes care of delivering
-outgoing store requests reads from incoming buffer
-
-its argument is the deliver function taking the item as first argument
-and a quit channel as second.
-Closing of this channel is supposed to abort all waiting for delivery
-(typically network write)
-
-The iteration switches between 2 modes,
-* buffer mode reads the in-memory buffer and delivers the items directly
-* db mode reads from the buffer and writes to the db, parallelly another
-routine is started that reads from the db and delivers items
-
-If there is buffer contention in buffer mode (slow network, high upload volume)
-syncdb switches to db mode and starts dbRead
-Once db backlog is delivered, it reverts back to in-memory buffer
-
-It is automatically started when syncdb is initialised.
-
-It saves the buffer to db upon receiving quit signal. syncDb#stop()
-*/
-func (self *syncDb) bufferRead(deliver func(interface{}, chan bool) bool) {
-	var buffer, db chan interface{} // channels representing the two read modes
-	var more bool
-	var req interface{}
-	var entry *syncDbEntry
-	var inBatch, inDb int
-	batch := new(leveldb.Batch)
-	var dbSize chan int
-	quit := self.quit
-	counterValue := make([]byte, 8)
-
-	// counter is used for keeping the items in order, persisted to db
-	// start counter where db was at, 0 if not found
-	data, err := self.db.Get(self.counterKey)
-	var counter uint64
-	if err == nil {
-		counter = binary.BigEndian.Uint64(data)
-		log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter read from db at %v", self.key.Log(), self.priority, counter))
-	} else {
-		log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter starts at %v", self.key.Log(), self.priority, counter))
-	}
-
-LOOP:
-	for {
-		// waiting for item next in the buffer, or quit signal or batch request
-		select {
-		// buffer only closes when writing to db
-		case req = <-buffer:
-			// deliver request : this is blocking on network write so
-			// it is passed the quit channel as argument, so that it returns
-			// if syncdb is stopped. In this case we need to save the item to the db
-			more = deliver(req, self.quit)
-			if !more {
-				log.Debug(fmt.Sprintf("syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total))
-				// received quit signal, save request currently waiting delivery
-				// by switching to db mode and closing the buffer
-				buffer = nil
-				db = self.buffer
-				close(db)
-				quit = nil // needs to block the quit case in select
-				break      // break from select, this item will be written to the db
-			}
-			self.total++
-			log.Trace(fmt.Sprintf("syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total))
-			// by the time deliver returns, there were new writes to the buffer
-			// if buffer contention is detected, switch to db mode which drains
-			// the buffer so no process will block on pushing store requests
-			if len(buffer) == cap(buffer) {
-				log.Debug(fmt.Sprintf("syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total))
-				buffer = nil
-				db = self.buffer
-			}
-			continue LOOP
-
-			// incoming entry to put into db
-		case req, more = <-db:
-			if !more {
-				// only if quit is called, saved all the buffer
-				binary.BigEndian.PutUint64(counterValue, counter)
-				batch.Put(self.counterKey, counterValue) // persist counter in batch
-				self.writeSyncBatch(batch)               // save batch
-				log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority))
-				break LOOP
-			}
-			self.dbTotal++
-			self.total++
-			// otherwise break after select
-		case dbSize = <-self.batch:
-			// explicit request for batch
-			if inBatch == 0 && quit != nil {
-				// there was no writes since the last batch so db depleted
-				// switch to buffer mode
-				log.Debug(fmt.Sprintf("syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority))
-				db = nil
-				buffer = self.buffer
-				dbSize <- 0 // indicates to 'caller' that batch has been written
-				inDb = 0
-				continue LOOP
-			}
-			binary.BigEndian.PutUint64(counterValue, counter)
-			batch.Put(self.counterKey, counterValue)
-			log.Debug(fmt.Sprintf("syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue))
-			batch = self.writeSyncBatch(batch)
-			dbSize <- inBatch // indicates to 'caller' that batch has been written
-			inBatch = 0
-			continue LOOP
-
-			// closing syncDb#quit channel is used to signal to all goroutines to quit
-		case <-quit:
-			// need to save backlog, so switch to db mode
-			db = self.buffer
-			buffer = nil
-			quit = nil
-			log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority))
-			close(db)
-			continue LOOP
-		}
-
-		// only get here if we put req into db
-		entry, err = self.newSyncDbEntry(req, counter)
-		if err != nil {
-			log.Warn(fmt.Sprintf("syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err))
-			continue LOOP
-		}
-		batch.Put(entry.key, entry.val)
-		log.Trace(fmt.Sprintf("syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter))
-		// if just switched to db mode and not quitting, then launch dbRead
-		// in a parallel go routine to send deliveries from db
-		if inDb == 0 && quit != nil {
-			log.Trace(fmt.Sprintf("syncDb[%v/%v] start dbRead", self.key.Log(), self.priority))
-			go self.dbRead(true, counter, deliver)
-		}
-		inDb++
-		inBatch++
-		counter++
-		// need to save the batch if it gets too large (== dbBatchSize)
-		if inBatch%int(self.dbBatchSize) == 0 {
-			batch = self.writeSyncBatch(batch)
-		}
-	}
-	log.Info(fmt.Sprintf("syncDb[%v:%v]: saved %v keys (saved counter at %v)", self.key.Log(), self.priority, inBatch, counter))
-	close(self.done)
-}
-
-// writes the batch to the db and returns a new batch object
-func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch {
-	err := self.db.Write(batch)
-	if err != nil {
-		log.Warn(fmt.Sprintf("syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err))
-		return batch
-	}
-	return new(leveldb.Batch)
-}
-
-// abstract type for db entries (TODO could be a feature of Receipts)
-type syncDbEntry struct {
-	key, val []byte
-}
-
-func (self syncDbEntry) String() string {
-	return fmt.Sprintf("key: %x, value: %x", self.key, self.val)
-}
-
-/*
-	dbRead is iterating over store requests to be sent over to the peer
-	this is mainly to prevent crashes due to network output buffer contention (???)
-	as well as to make syncronisation resilient to disconnects
-	the messages are supposed to be sent in the p2p priority queue.
-
-	the request DB is shared between peers, but domains for each syncdb
-	are disjoint. dbkeys (42 bytes) are structured:
-	* 0: 0x00 (0x01 reserved for counter key)
-	* 1: priorities - priority (so that high priority can be replayed first)
-	* 2-33: peers address
-	* 34-41: syncdb counter to preserve order (this field is missing for the counter key)
-
-	values (40 bytes) are:
-	* 0-31: key
-	* 32-39: request id
-
-dbRead needs a boolean to indicate if on first round all the historical
-record is synced. Second argument to indicate current db counter
-The third is the function to apply
-*/
-func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}, chan bool) bool) {
-	key := make([]byte, 42)
-	copy(key, self.start)
-	binary.BigEndian.PutUint64(key[34:], counter)
-	var batches, n, cnt, total int
-	var more bool
-	var entry *syncDbEntry
-	var it iterator.Iterator
-	var del *leveldb.Batch
-	batchSizes := make(chan int)
-
-	for {
-		// if useBatches is false, cnt is not set
-		if useBatches {
-			// this could be called before all cnt items sent out
-			// so that loop is not blocking while delivering
-			// only relevant if cnt is large
-			select {
-			case self.batch <- batchSizes:
-			case <-self.quit:
-				return
-			}
-			// wait for the write to finish and get the item count in the next batch
-			cnt = <-batchSizes
-			batches++
-			if cnt == 0 {
-				// empty
-				return
-			}
-		}
-		it = self.db.NewIterator()
-		it.Seek(key)
-		if !it.Valid() {
-			copy(key, self.start)
-			useBatches = true
-			continue
-		}
-		del = new(leveldb.Batch)
-		log.Trace(fmt.Sprintf("syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt))
-
-		for n = 0; !useBatches || n < cnt; it.Next() {
-			copy(key, it.Key())
-			if len(key) == 0 || key[0] != 0 {
-				copy(key, self.start)
-				useBatches = true
-				break
-			}
-			val := make([]byte, 40)
-			copy(val, it.Value())
-			entry = &syncDbEntry{key, val}
-			// log.Trace(fmt.Sprintf("syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total))
-			more = fun(entry, self.quit)
-			if !more {
-				// quit received when waiting to deliver entry, the entry will not be deleted
-				log.Trace(fmt.Sprintf("syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt))
-				break
-			}
-			// since subsequent batches of the same db session are indexed incrementally
-			// deleting earlier batches can be delayed and parallelised
-			// this could be batch delete when db is idle (but added complexity esp when quitting)
-			del.Delete(key)
-			n++
-			total++
-		}
-		log.Debug(fmt.Sprintf("syncDb[%v/%v] - db session closed, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, batches, total, self.dbTotal, self.total))
-		self.db.Write(del) // this could be async called only when db is idle
-		it.Release()
-	}
-}
-
-//
-func (self *syncDb) stop() {
-	close(self.quit)
-	<-self.done
-}
-
-// calculate a dbkey for the request, for the db to work
-// see syncdb for db key structure
-// polimorphic: accepted types, see syncer#addRequest
-func (self *syncDb) newSyncDbEntry(req interface{}, counter uint64) (entry *syncDbEntry, err error) {
-	var key storage.Key
-	var chunk *storage.Chunk
-	var id uint64
-	var ok bool
-	var sreq *storeRequestMsgData
-
-	if key, ok = req.(storage.Key); ok {
-		id = generateId()
-	} else if chunk, ok = req.(*storage.Chunk); ok {
-		key = chunk.Key
-		id = generateId()
-	} else if sreq, ok = req.(*storeRequestMsgData); ok {
-		key = sreq.Key
-		id = sreq.Id
-	} else if entry, ok = req.(*syncDbEntry); !ok {
-		return nil, fmt.Errorf("type not allowed: %v (%T)", req, req)
-	}
-
-	// order by peer > priority > seqid
-	// value is request id if exists
-	if entry == nil {
-		dbkey := make([]byte, 42)
-		dbval := make([]byte, 40)
-
-		// encode key
-		copy(dbkey[:], self.start[:34]) // db  peer
-		binary.BigEndian.PutUint64(dbkey[34:], counter)
-		// encode value
-		copy(dbval, key[:])
-		binary.BigEndian.PutUint64(dbval[32:], id)
-
-		entry = &syncDbEntry{dbkey, dbval}
-	}
-	return
-}

swarm/network/syncdb_test.go

@@ -1,222 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-func init() {
-	log.Root().SetHandler(log.LvlFilterHandler(log.LvlCrit, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
-}
-
-type testSyncDb struct {
-	*syncDb
-	c         int
-	t         *testing.T
-	fromDb    chan bool
-	delivered [][]byte
-	sent      []int
-	dbdir     string
-	at        int
-}
-
-func newTestSyncDb(priority, bufferSize, batchSize int, dbdir string, t *testing.T) *testSyncDb {
-	if len(dbdir) == 0 {
-		tmp, err := ioutil.TempDir(os.TempDir(), "syncdb-test")
-		if err != nil {
-			t.Fatalf("unable to create temporary direcory %v: %v", tmp, err)
-		}
-		dbdir = tmp
-	}
-	db, err := storage.NewLDBDatabase(filepath.Join(dbdir, "requestdb"))
-	if err != nil {
-		t.Fatalf("unable to create db: %v", err)
-	}
-	self := &testSyncDb{
-		fromDb: make(chan bool),
-		dbdir:  dbdir,
-		t:      t,
-	}
-	h := crypto.Keccak256Hash([]byte{0})
-	key := storage.Key(h[:])
-	self.syncDb = newSyncDb(db, key, uint(priority), uint(bufferSize), uint(batchSize), self.deliver)
-	// kick off db iterator right away, if no items on db this will allow
-	// reading from the buffer
-	return self
-
-}
-
-func (self *testSyncDb) close() {
-	self.db.Close()
-	os.RemoveAll(self.dbdir)
-}
-
-func (self *testSyncDb) push(n int) {
-	for i := 0; i < n; i++ {
-		self.buffer <- storage.Key(crypto.Keccak256([]byte{byte(self.c)}))
-		self.sent = append(self.sent, self.c)
-		self.c++
-	}
-	log.Debug(fmt.Sprintf("pushed %v requests", n))
-}
-
-func (self *testSyncDb) draindb() {
-	it := self.db.NewIterator()
-	defer it.Release()
-	for {
-		it.Seek(self.start)
-		if !it.Valid() {
-			return
-		}
-		k := it.Key()
-		if len(k) == 0 || k[0] == 1 {
-			return
-		}
-		it.Release()
-		it = self.db.NewIterator()
-	}
-}
-
-func (self *testSyncDb) deliver(req interface{}, quit chan bool) bool {
-	_, db := req.(*syncDbEntry)
-	key, _, _, _, err := parseRequest(req)
-	if err != nil {
-		self.t.Fatalf("unexpected error of key %v: %v", key, err)
-	}
-	self.delivered = append(self.delivered, key)
-	select {
-	case self.fromDb <- db:
-		return true
-	case <-quit:
-		return false
-	}
-}
-
-func (self *testSyncDb) expect(n int, db bool) {
-	var ok bool
-	// for n items
-	for i := 0; i < n; i++ {
-		ok = <-self.fromDb
-		if self.at+1 > len(self.delivered) {
-			self.t.Fatalf("expected %v, got %v", self.at+1, len(self.delivered))
-		}
-		if len(self.sent) > self.at && !bytes.Equal(crypto.Keccak256([]byte{byte(self.sent[self.at])}), self.delivered[self.at]) {
-			self.t.Fatalf("expected delivery %v/%v/%v to be hash of  %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db)
-			log.Debug(fmt.Sprintf("%v/%v/%v to be hash of  %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db))
-		}
-		if !ok && db {
-			self.t.Fatalf("expected delivery %v/%v/%v from db", i, n, self.at)
-		}
-		if ok && !db {
-			self.t.Fatalf("expected delivery %v/%v/%v from cache", i, n, self.at)
-		}
-		self.at++
-	}
-}
-
-func TestSyncDb(t *testing.T) {
-	t.Skip("fails randomly on all platforms")
-
-	priority := High
-	bufferSize := 5
-	batchSize := 2 * bufferSize
-	s := newTestSyncDb(priority, bufferSize, batchSize, "", t)
-	defer s.close()
-	defer s.stop()
-	s.dbRead(false, 0, s.deliver)
-	s.draindb()
-
-	s.push(4)
-	s.expect(1, false)
-	// 3 in buffer
-	time.Sleep(100 * time.Millisecond)
-	s.push(3)
-	// push over limit
-	s.expect(1, false)
-	// one popped from the buffer, then contention detected
-	s.expect(4, true)
-	s.push(4)
-	s.expect(5, true)
-	// depleted db, switch back to buffer
-	s.draindb()
-	s.push(5)
-	s.expect(4, false)
-	s.push(3)
-	s.expect(4, false)
-	// buffer depleted
-	time.Sleep(100 * time.Millisecond)
-	s.push(6)
-	s.expect(1, false)
-	// push into buffer full, switch to db
-	s.expect(5, true)
-	s.draindb()
-	s.push(1)
-	s.expect(1, false)
-}
-
-func TestSaveSyncDb(t *testing.T) {
-	amount := 30
-	priority := High
-	bufferSize := amount
-	batchSize := 10
-	s := newTestSyncDb(priority, bufferSize, batchSize, "", t)
-	go s.dbRead(false, 0, s.deliver)
-	s.push(amount)
-	s.stop()
-	s.db.Close()
-
-	s = newTestSyncDb(priority, bufferSize, batchSize, s.dbdir, t)
-	go s.dbRead(false, 0, s.deliver)
-	s.expect(amount, true)
-	for i, key := range s.delivered {
-		expKey := crypto.Keccak256([]byte{byte(i)})
-		if !bytes.Equal(key, expKey) {
-			t.Fatalf("delivery %v expected to be key %x, got %x", i, expKey, key)
-		}
-	}
-	s.push(amount)
-	s.expect(amount, false)
-	for i := amount; i < 2*amount; i++ {
-		key := s.delivered[i]
-		expKey := crypto.Keccak256([]byte{byte(i - amount)})
-		if !bytes.Equal(key, expKey) {
-			t.Fatalf("delivery %v expected to be key %x, got %x", i, expKey, key)
-		}
-	}
-	s.stop()
-	s.db.Close()
-
-	s = newTestSyncDb(priority, bufferSize, batchSize, s.dbdir, t)
-	defer s.close()
-	defer s.stop()
-
-	go s.dbRead(false, 0, s.deliver)
-	s.push(1)
-	s.expect(1, false)
-
-}

swarm/network/syncer.go

@@ -1,777 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package network
-
-import (
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"path/filepath"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-// syncer parameters (global, not peer specific) default values
-const (
-	requestDbBatchSize = 512  // size of batch before written to request db
-	keyBufferSize      = 1024 // size of buffer  for unsynced keys
-	syncBatchSize      = 128  // maximum batchsize for outgoing requests
-	syncBufferSize     = 128  // size of buffer  for delivery requests
-	syncCacheSize      = 1024 // cache capacity to store request queue in memory
-)
-
-// priorities
-const (
-	Low        = iota // 0
-	Medium            // 1
-	High              // 2
-	priorities        // 3 number of priority levels
-)
-
-// request types
-const (
-	DeliverReq   = iota // 0
-	PushReq             // 1
-	PropagateReq        // 2
-	HistoryReq          // 3
-	BacklogReq          // 4
-)
-
-// json serialisable struct to record the syncronisation state between 2 peers
-type syncState struct {
-	*storage.DbSyncState // embeds the following 4 fields:
-	// Start      Key    // lower limit of address space
-	// Stop       Key    // upper limit of address space
-	// First      uint64 // counter taken from last sync state
-	// Last       uint64 // counter of remote peer dbStore at the time of last connection
-	SessionAt  uint64      // set at the time of connection
-	LastSeenAt uint64      // set at the time of connection
-	Latest     storage.Key // cursor of dbstore when last (continuously set by syncer)
-	Synced     bool        // true iff Sync is done up to the last disconnect
-	synced     chan bool   // signal that sync stage finished
-}
-
-// wrapper of db-s to provide mockable custom local chunk store access to syncer
-type DbAccess struct {
-	db  *storage.DbStore
-	loc *storage.LocalStore
-}
-
-func NewDbAccess(loc *storage.LocalStore) *DbAccess {
-	return &DbAccess{loc.DbStore.(*storage.DbStore), loc}
-}
-
-// to obtain the chunks from key or request db entry only
-func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) {
-	return self.loc.Get(key)
-}
-
-// current storage counter of chunk db
-func (self *DbAccess) counter() uint64 {
-	return self.db.Counter()
-}
-
-// implemented by dbStoreSyncIterator
-type keyIterator interface {
-	Next() storage.Key
-}
-
-// generator function for iteration by address range and storage counter
-func (self *DbAccess) iterator(s *syncState) keyIterator {
-	it, err := self.db.NewSyncIterator(*(s.DbSyncState))
-	if err != nil {
-		return nil
-	}
-	return keyIterator(it)
-}
-
-func (self syncState) String() string {
-	if self.Synced {
-		return fmt.Sprintf(
-			"session started at: %v, last seen at: %v, latest key: %v",
-			self.SessionAt, self.LastSeenAt,
-			self.Latest.Log(),
-		)
-	} else {
-		return fmt.Sprintf(
-			"address: %v-%v, index: %v-%v, session started at: %v, last seen at: %v, latest key: %v",
-			self.Start.Log(), self.Stop.Log(),
-			self.First, self.Last,
-			self.SessionAt, self.LastSeenAt,
-			self.Latest.Log(),
-		)
-	}
-}
-
-// syncer parameters (global, not peer specific)
-type SyncParams struct {
-	RequestDbPath      string // path for request db (leveldb)
-	RequestDbBatchSize uint   // nuber of items before batch is saved to requestdb
-	KeyBufferSize      uint   // size of key buffer
-	SyncBatchSize      uint   // maximum batchsize for outgoing requests
-	SyncBufferSize     uint   // size of buffer for
-	SyncCacheSize      uint   // cache capacity to store request queue in memory
-	SyncPriorities     []uint // list of priority levels for req types 0-3
-	SyncModes          []bool // list of sync modes for  for req types 0-3
-}
-
-// constructor with default values
-func NewSyncParams(bzzdir string) *SyncParams {
-	return &SyncParams{
-		RequestDbPath:      filepath.Join(bzzdir, "requests"),
-		RequestDbBatchSize: requestDbBatchSize,
-		KeyBufferSize:      keyBufferSize,
-		SyncBufferSize:     syncBufferSize,
-		SyncBatchSize:      syncBatchSize,
-		SyncCacheSize:      syncCacheSize,
-		SyncPriorities:     []uint{High, Medium, Medium, Low, Low},
-		SyncModes:          []bool{true, true, true, true, false},
-	}
-}
-
-// syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding
-type syncer struct {
-	*SyncParams                     // sync parameters
-	syncF           func() bool     // if syncing is needed
-	key             storage.Key     // remote peers address key
-	state           *syncState      // sync state for our dbStore
-	syncStates      chan *syncState // different stages of sync
-	deliveryRequest chan bool       // one of two triggers needed to send unsyncedKeys
-	newUnsyncedKeys chan bool       // one of two triggers needed to send unsynced keys
-	quit            chan bool       // signal to quit loops
-
-	// DB related fields
-	dbAccess *DbAccess            // access to dbStore
-	db       *storage.LDBDatabase // delivery msg db
-
-	// native fields
-	queues     [priorities]*syncDb                   // in-memory cache / queues for sync reqs
-	keys       [priorities]chan interface{}          // buffer for unsynced keys
-	deliveries [priorities]chan *storeRequestMsgData // delivery
-
-	// bzz protocol instance outgoing message callbacks (mockable for testing)
-	unsyncedKeys func([]*syncRequest, *syncState) error // send unsyncedKeysMsg
-	store        func(*storeRequestMsgData) error       // send storeRequestMsg
-}
-
-// a syncer instance is linked to each peer connection
-// constructor is called from protocol after successful handshake
-// the returned instance is attached to the peer and can be called
-// by the forwarder
-func newSyncer(
-	db *storage.LDBDatabase, remotekey storage.Key,
-	dbAccess *DbAccess,
-	unsyncedKeys func([]*syncRequest, *syncState) error,
-	store func(*storeRequestMsgData) error,
-	params *SyncParams,
-	state *syncState,
-	syncF func() bool,
-) (*syncer, error) {
-
-	syncBufferSize := params.SyncBufferSize
-	keyBufferSize := params.KeyBufferSize
-	dbBatchSize := params.RequestDbBatchSize
-
-	self := &syncer{
-		syncF:           syncF,
-		key:             remotekey,
-		dbAccess:        dbAccess,
-		syncStates:      make(chan *syncState, 20),
-		deliveryRequest: make(chan bool, 1),
-		newUnsyncedKeys: make(chan bool, 1),
-		SyncParams:      params,
-		state:           state,
-		quit:            make(chan bool),
-		unsyncedKeys:    unsyncedKeys,
-		store:           store,
-	}
-
-	// initialising
-	for i := 0; i < priorities; i++ {
-		self.keys[i] = make(chan interface{}, keyBufferSize)
-		self.deliveries[i] = make(chan *storeRequestMsgData)
-		// initialise a syncdb instance for each priority queue
-		self.queues[i] = newSyncDb(db, remotekey, uint(i), syncBufferSize, dbBatchSize, self.deliver(uint(i)))
-	}
-	log.Info(fmt.Sprintf("syncer started: %v", state))
-	// launch chunk delivery service
-	go self.syncDeliveries()
-	// launch sync task manager
-	if self.syncF() {
-		go self.sync()
-	}
-	// process unsynced keys to broadcast
-	go self.syncUnsyncedKeys()
-
-	return self, nil
-}
-
-// metadata serialisation
-func encodeSync(state *syncState) (*json.RawMessage, error) {
-	data, err := json.MarshalIndent(state, "", " ")
-	if err != nil {
-		return nil, err
-	}
-	meta := json.RawMessage(data)
-	return &meta, nil
-}
-
-func decodeSync(meta *json.RawMessage) (*syncState, error) {
-	if meta == nil {
-		return nil, fmt.Errorf("unable to deserialise sync state from <nil>")
-	}
-	data := []byte(*(meta))
-	if len(data) == 0 {
-		return nil, fmt.Errorf("unable to deserialise sync state from <nil>")
-	}
-	state := &syncState{DbSyncState: &storage.DbSyncState{}}
-	err := json.Unmarshal(data, state)
-	return state, err
-}
-
-/*
- sync implements the syncing script
- * first all items left in the request Db are replayed
-   * type = StaleSync
-   * Mode: by default once again via confirmation roundtrip
-   * Priority: the items are replayed as the proirity specified for StaleSync
-   * but within the order respects earlier priority level of request
- * after all items are consumed for a priority level, the the respective
-  queue for delivery requests is open (this way new reqs not written to db)
-  (TODO: this should be checked)
- * the sync state provided by the remote peer is used to sync history
-   * all the backlog from earlier (aborted) syncing is completed starting from latest
-   * if Last  < LastSeenAt then all items in between then process all
-     backlog from upto last disconnect
-   * if Last > 0 &&
-
- sync is called from the syncer constructor and is not supposed to be used externally
-*/
-func (self *syncer) sync() {
-	state := self.state
-	// sync finished
-	defer close(self.syncStates)
-
-	// 0. first replay stale requests from request db
-	if state.SessionAt == 0 {
-		log.Debug(fmt.Sprintf("syncer[%v]: nothing to sync", self.key.Log()))
-		return
-	}
-	log.Debug(fmt.Sprintf("syncer[%v]: start replaying stale requests from request db", self.key.Log()))
-	for p := priorities - 1; p >= 0; p-- {
-		self.queues[p].dbRead(false, 0, self.replay())
-	}
-	log.Debug(fmt.Sprintf("syncer[%v]: done replaying stale requests from request db", self.key.Log()))
-
-	// unless peer is synced sync unfinished history beginning on
-	if !state.Synced {
-		start := state.Start
-
-		if !storage.IsZeroKey(state.Latest) {
-			// 1. there is unfinished earlier sync
-			state.Start = state.Latest
-			log.Debug(fmt.Sprintf("syncer[%v]: start syncronising backlog (unfinished sync: %v)", self.key.Log(), state))
-			// blocks while the entire history upto state is synced
-			self.syncState(state)
-			if state.Last < state.SessionAt {
-				state.First = state.Last + 1
-			}
-		}
-		state.Latest = storage.ZeroKey
-		state.Start = start
-		// 2. sync up to last disconnect1
-		if state.First < state.LastSeenAt {
-			state.Last = state.LastSeenAt
-			log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history upto last disconnect at %v: %v", self.key.Log(), state.LastSeenAt, state))
-			self.syncState(state)
-			state.First = state.LastSeenAt
-		}
-		state.Latest = storage.ZeroKey
-
-	} else {
-		// synchronisation starts at end of last session
-		state.First = state.LastSeenAt
-	}
-
-	// 3. sync up to current session start
-	// if there have been new chunks since last session
-	if state.LastSeenAt < state.SessionAt {
-		state.Last = state.SessionAt
-		log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state))
-		// blocks until state syncing is finished
-		self.syncState(state)
-	}
-	log.Info(fmt.Sprintf("syncer[%v]: syncing all history complete", self.key.Log()))
-
-}
-
-// wait till syncronised block uptil state is synced
-func (self *syncer) syncState(state *syncState) {
-	self.syncStates <- state
-	select {
-	case <-state.synced:
-	case <-self.quit:
-	}
-}
-
-// stop quits both request processor and saves the request cache to disk
-func (self *syncer) stop() {
-	close(self.quit)
-	log.Trace(fmt.Sprintf("syncer[%v]: stop and save sync request db backlog", self.key.Log()))
-	for _, db := range self.queues {
-		db.stop()
-	}
-}
-
-// rlp serialisable sync request
-type syncRequest struct {
-	Key      storage.Key
-	Priority uint
-}
-
-func (self *syncRequest) String() string {
-	return fmt.Sprintf("<Key: %v, Priority: %v>", self.Key.Log(), self.Priority)
-}
-
-func (self *syncer) newSyncRequest(req interface{}, p int) (*syncRequest, error) {
-	key, _, _, _, err := parseRequest(req)
-	// TODO: if req has chunk, it should be put in a cache
-	// create
-	if err != nil {
-		return nil, err
-	}
-	return &syncRequest{key, uint(p)}, nil
-}
-
-// serves historical items from the DB
-// * read is on demand, blocking unless history channel is read
-// * accepts sync requests (syncStates) to create new db iterator
-// * closes the channel one iteration finishes
-func (self *syncer) syncHistory(state *syncState) chan interface{} {
-	var n uint
-	history := make(chan interface{})
-	log.Debug(fmt.Sprintf("syncer[%v]: syncing history between %v - %v for chunk addresses %v - %v", self.key.Log(), state.First, state.Last, state.Start, state.Stop))
-	it := self.dbAccess.iterator(state)
-	if it != nil {
-		go func() {
-			// signal end of the iteration ended
-			defer close(history)
-		IT:
-			for {
-				key := it.Next()
-				if key == nil {
-					break IT
-				}
-				select {
-				// blocking until history channel is read from
-				case history <- storage.Key(key):
-					n++
-					log.Trace(fmt.Sprintf("syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n))
-					state.Latest = key
-				case <-self.quit:
-					return
-				}
-			}
-			log.Debug(fmt.Sprintf("syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n))
-		}()
-	}
-	return history
-}
-
-// triggers key syncronisation
-func (self *syncer) sendUnsyncedKeys() {
-	select {
-	case self.deliveryRequest <- true:
-	default:
-	}
-}
-
-// assembles a new batch of unsynced keys
-// * keys are drawn from the key buffers in order of priority queue
-// * if the queues of priority for History (HistoryReq) or higher are depleted,
-//   historical data is used so historical items are lower priority within
-//   their priority group.
-// * Order of historical data is unspecified
-func (self *syncer) syncUnsyncedKeys() {
-	// send out new
-	var unsynced []*syncRequest
-	var more, justSynced bool
-	var keyCount, historyCnt int
-	var history chan interface{}
-
-	priority := High
-	keys := self.keys[priority]
-	var newUnsyncedKeys, deliveryRequest chan bool
-	keyCounts := make([]int, priorities)
-	histPrior := self.SyncPriorities[HistoryReq]
-	syncStates := self.syncStates
-	state := self.state
-
-LOOP:
-	for {
-
-		var req interface{}
-		// select the highest priority channel to read from
-		// keys channels are buffered so the highest priority ones
-		// are checked first - integrity can only be guaranteed if writing
-		// is locked while selecting
-		if priority != High || len(keys) == 0 {
-			// selection is not needed if the High priority queue has items
-			keys = nil
-		PRIORITIES:
-			for priority = High; priority >= 0; priority-- {
-				// the first priority channel that is non-empty will be assigned to keys
-				if len(self.keys[priority]) > 0 {
-					log.Trace(fmt.Sprintf("syncer[%v]: reading request with	priority %v", self.key.Log(), priority))
-					keys = self.keys[priority]
-					break PRIORITIES
-				}
-				log.Trace(fmt.Sprintf("syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low])))
-				// if the input queue is empty on this level, resort to history if there is any
-				if uint(priority) == histPrior && history != nil {
-					log.Trace(fmt.Sprintf("syncer[%v]: reading history for %v", self.key.Log(), self.key))
-					keys = history
-					break PRIORITIES
-				}
-			}
-		}
-
-		// if peer ready to receive but nothing to send
-		if keys == nil && deliveryRequest == nil {
-			// if no items left and switch to waiting mode
-			log.Trace(fmt.Sprintf("syncer[%v]: buffers consumed. Waiting", self.key.Log()))
-			newUnsyncedKeys = self.newUnsyncedKeys
-		}
-
-		// send msg iff
-		// * peer is ready to receive keys AND (
-		// * all queues and history are depleted OR
-		// * batch full OR
-		// * all history have been consumed, synced)
-		if deliveryRequest == nil &&
-			(justSynced ||
-				len(unsynced) > 0 && keys == nil ||
-				len(unsynced) == int(self.SyncBatchSize)) {
-			justSynced = false
-			// listen to requests
-			deliveryRequest = self.deliveryRequest
-			newUnsyncedKeys = nil // not care about data until next req comes in
-			// set sync to current counter
-			// (all nonhistorical outgoing traffic sheduled and persisted
-			state.LastSeenAt = self.dbAccess.counter()
-			state.Latest = storage.ZeroKey
-			log.Trace(fmt.Sprintf("syncer[%v]: sending %v", self.key.Log(), unsynced))
-			//  send the unsynced keys
-			stateCopy := *state
-			err := self.unsyncedKeys(unsynced, &stateCopy)
-			if err != nil {
-				log.Warn(fmt.Sprintf("syncer[%v]: unable to send unsynced keys: %v", self.key.Log(), err))
-			}
-			self.state = state
-			log.Debug(fmt.Sprintf("syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy))
-			unsynced = nil
-			keys = nil
-		}
-
-		// process item and add it to the batch
-		select {
-		case <-self.quit:
-			break LOOP
-		case req, more = <-keys:
-			if keys == history && !more {
-				log.Trace(fmt.Sprintf("syncer[%v]: syncing history segment complete", self.key.Log()))
-				// history channel is closed, waiting for new state (called from sync())
-				syncStates = self.syncStates
-				state.Synced = true // this signals that the  current segment is complete
-				select {
-				case state.synced <- false:
-				case <-self.quit:
-					break LOOP
-				}
-				justSynced = true
-				history = nil
-			}
-		case <-deliveryRequest:
-			log.Trace(fmt.Sprintf("syncer[%v]: peer ready to receive", self.key.Log()))
-
-			// this 1 cap channel can wake up the loop
-			// signaling that peer is ready to receive unsynced Keys
-			// the channel is set to nil any further writes will be ignored
-			deliveryRequest = nil
-
-		case <-newUnsyncedKeys:
-			log.Trace(fmt.Sprintf("syncer[%v]: new unsynced keys available", self.key.Log()))
-			// this 1 cap channel can wake up the loop
-			// signals that data is available to send if peer is ready to receive
-			newUnsyncedKeys = nil
-			keys = self.keys[High]
-
-		case state, more = <-syncStates:
-			// this resets the state
-			if !more {
-				state = self.state
-				log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing complete upto %v)", self.key.Log(), priority, state))
-				state.Synced = true
-				syncStates = nil
-			} else {
-				log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing history upto %v priority %v)", self.key.Log(), priority, state, histPrior))
-				state.Synced = false
-				history = self.syncHistory(state)
-				// only one history at a time, only allow another one once the
-				// history channel is closed
-				syncStates = nil
-			}
-		}
-		if req == nil {
-			continue LOOP
-		}
-
-		log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) added to unsynced keys: %v", self.key.Log(), priority, req))
-		keyCounts[priority]++
-		keyCount++
-		if keys == history {
-			log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) history item %v (synced = %v)", self.key.Log(), priority, req, state.Synced))
-			historyCnt++
-		}
-		if sreq, err := self.newSyncRequest(req, priority); err == nil {
-			// extract key from req
-			log.Trace(fmt.Sprintf("syncer[%v]: (priority %v): request %v (synced = %v)", self.key.Log(), priority, req, state.Synced))
-			unsynced = append(unsynced, sreq)
-		} else {
-			log.Warn(fmt.Sprintf("syncer[%v]: (priority %v): error creating request for %v: %v)", self.key.Log(), priority, req, err))
-		}
-
-	}
-}
-
-// delivery loop
-// takes into account priority, send store Requests with chunk (delivery)
-// idle blocking if no new deliveries in any of the queues
-func (self *syncer) syncDeliveries() {
-	var req *storeRequestMsgData
-	p := High
-	var deliveries chan *storeRequestMsgData
-	var msg *storeRequestMsgData
-	var err error
-	var c = [priorities]int{}
-	var n = [priorities]int{}
-	var total, success uint
-
-	for {
-		deliveries = self.deliveries[p]
-		select {
-		case req = <-deliveries:
-			n[p]++
-			c[p]++
-		default:
-			if p == Low {
-				// blocking, depletion on all channels, no preference for priority
-				select {
-				case req = <-self.deliveries[High]:
-					n[High]++
-				case req = <-self.deliveries[Medium]:
-					n[Medium]++
-				case req = <-self.deliveries[Low]:
-					n[Low]++
-				case <-self.quit:
-					return
-				}
-				p = High
-			} else {
-				p--
-				continue
-			}
-		}
-		total++
-		msg, err = self.newStoreRequestMsgData(req)
-		if err != nil {
-			log.Warn(fmt.Sprintf("syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err))
-		} else {
-			err = self.store(msg)
-			if err != nil {
-				log.Warn(fmt.Sprintf("syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err))
-			} else {
-				success++
-				log.Trace(fmt.Sprintf("syncer[%v]: %v successfully delivered", self.key.Log(), req))
-			}
-		}
-		if total%self.SyncBatchSize == 0 {
-			log.Debug(fmt.Sprintf("syncer[%v]: deliver Total: %v, Success: %v, High: %v/%v, Medium: %v/%v, Low %v/%v", self.key.Log(), total, success, c[High], n[High], c[Medium], n[Medium], c[Low], n[Low]))
-		}
-	}
-}
-
-/*
- addRequest handles requests for delivery
- it accepts 4 types:
-
- * storeRequestMsgData: coming from netstore propagate response
- * chunk: coming from forwarding (questionable: id?)
- * key: from incoming syncRequest
- * syncDbEntry: key,id encoded in db
-
- If sync mode is on for the type of request, then
- it sends the request to the keys queue of the correct priority
- channel buffered with capacity (SyncBufferSize)
-
- If sync mode is off then, requests are directly sent to deliveries
-*/
-func (self *syncer) addRequest(req interface{}, ty int) {
-	// retrieve priority for request type name int8
-
-	priority := self.SyncPriorities[ty]
-	// sync mode for this type ON
-	if self.syncF() || ty == DeliverReq {
-		if self.SyncModes[ty] {
-			self.addKey(req, priority, self.quit)
-		} else {
-			self.addDelivery(req, priority, self.quit)
-		}
-	}
-}
-
-// addKey queues sync request for sync confirmation with given priority
-// ie the key will go out in an unsyncedKeys message
-func (self *syncer) addKey(req interface{}, priority uint, quit chan bool) bool {
-	select {
-	case self.keys[priority] <- req:
-		// this wakes up the unsynced keys loop if idle
-		select {
-		case self.newUnsyncedKeys <- true:
-		default:
-		}
-		return true
-	case <-quit:
-		return false
-	}
-}
-
-// addDelivery queues delivery request for with given priority
-// ie the chunk will be delivered ASAP mod priority queueing handled by syncdb
-// requests are persisted across sessions for correct sync
-func (self *syncer) addDelivery(req interface{}, priority uint, quit chan bool) bool {
-	select {
-	case self.queues[priority].buffer <- req:
-		return true
-	case <-quit:
-		return false
-	}
-}
-
-// doDelivery delivers the chunk for the request with given priority
-// without queuing
-func (self *syncer) doDelivery(req interface{}, priority uint, quit chan bool) bool {
-	msgdata, err := self.newStoreRequestMsgData(req)
-	if err != nil {
-		log.Warn(fmt.Sprintf("unable to deliver request %v: %v", msgdata, err))
-		return false
-	}
-	select {
-	case self.deliveries[priority] <- msgdata:
-		return true
-	case <-quit:
-		return false
-	}
-}
-
-// returns the delivery function for given priority
-// passed on to syncDb
-func (self *syncer) deliver(priority uint) func(req interface{}, quit chan bool) bool {
-	return func(req interface{}, quit chan bool) bool {
-		return self.doDelivery(req, priority, quit)
-	}
-}
-
-// returns the replay function passed on to syncDb
-// depending on sync mode settings for BacklogReq,
-// re	play of request db backlog sends items via confirmation
-// or directly delivers
-func (self *syncer) replay() func(req interface{}, quit chan bool) bool {
-	sync := self.SyncModes[BacklogReq]
-	priority := self.SyncPriorities[BacklogReq]
-	// sync mode for this type ON
-	if sync {
-		return func(req interface{}, quit chan bool) bool {
-			return self.addKey(req, priority, quit)
-		}
-	} else {
-		return func(req interface{}, quit chan bool) bool {
-			return self.doDelivery(req, priority, quit)
-		}
-
-	}
-}
-
-// given a request, extends it to a full storeRequestMsgData
-// polimorphic: see addRequest for the types accepted
-func (self *syncer) newStoreRequestMsgData(req interface{}) (*storeRequestMsgData, error) {
-
-	key, id, chunk, sreq, err := parseRequest(req)
-	if err != nil {
-		return nil, err
-	}
-
-	if sreq == nil {
-		if chunk == nil {
-			var err error
-			chunk, err = self.dbAccess.get(key)
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		sreq = &storeRequestMsgData{
-			Id:    id,
-			Key:   chunk.Key,
-			SData: chunk.SData,
-		}
-	}
-
-	return sreq, nil
-}
-
-// parse request types and extracts, key, id, chunk, request if available
-// does not do chunk lookup !
-func parseRequest(req interface{}) (storage.Key, uint64, *storage.Chunk, *storeRequestMsgData, error) {
-	var key storage.Key
-	var entry *syncDbEntry
-	var chunk *storage.Chunk
-	var id uint64
-	var ok bool
-	var sreq *storeRequestMsgData
-	var err error
-
-	if key, ok = req.(storage.Key); ok {
-		id = generateId()
-
-	} else if entry, ok = req.(*syncDbEntry); ok {
-		id = binary.BigEndian.Uint64(entry.val[32:])
-		key = storage.Key(entry.val[:32])
-
-	} else if chunk, ok = req.(*storage.Chunk); ok {
-		key = chunk.Key
-		id = generateId()
-
-	} else if sreq, ok = req.(*storeRequestMsgData); ok {
-		key = sreq.Key
-	} else {
-		err = fmt.Errorf("type not allowed: %v (%T)", req, req)
-	}
-
-	return key, id, chunk, sreq, err
-}

swarm/network/test_overlay.go

@@ -0,0 +1,190 @@
+package network
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	// "github.com/ethereum/go-ethereum/p2p/adapters"
+	// "github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+const orders = 8
+
+type testOverlay struct {
+	mu     sync.Mutex
+	addr   []byte
+	pos    [][]*testNodeAddr
+	posMap map[string]*testNodeAddr
+}
+
+type testNodeAddr struct {
+	NodeAddr
+	Node Node
+}
+
+func (self *testOverlay) Register(na NodeAddr) error {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	return self.register(na)
+}
+
+func (self *testOverlay) register(na NodeAddr) error {
+	tna := &testNodeAddr{NodeAddr: na}
+	addr := na.OverlayAddr()
+	self.posMap[string(addr)] = tna
+	o := order(addr)
+	glog.V(6).Infof("PO: %v, orders: %v", o, orders)
+	self.pos[o] = append(self.pos[o], tna)
+	return nil
+}
+
+func order(addr []byte) int {
+	return int(addr[0]) / 32
+}
+
+func (self *testOverlay) On(n Node) (Node, error) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	addr := n.OverlayAddr()
+	na := self.posMap[string(addr)]
+	if na == nil {
+		self.register(n)
+		na = self.posMap[string(addr)]
+	} else if na.Node != nil {
+		return nil, nil
+	}
+	glog.V(6).Infof("Online: %v", fmt.Sprintf("%x", addr[:4]))
+	na.Node = n
+	o := order(addr)
+	ons := self.on(self.pos[o])
+	if len(ons) > 2 {
+		return ons[0], nil
+	}
+	return nil, nil
+}
+
+func (self *testOverlay) Off(n Node) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	addr := n.OverlayAddr()
+	na := self.posMap[string(addr)]
+	if na == nil {
+		return
+	}
+	na.Node = nil
+}
+
+// caller must hold the lock
+func (self *testOverlay) on(po []*testNodeAddr) (nodes []Node) {
+	for _, na := range po {
+		if na.Node != nil {
+			nodes = append(nodes, na.Node)
+		}
+	}
+	return nodes
+}
+
+// caller must hold the lock
+func (self *testOverlay) off(po []*testNodeAddr) (nas []NodeAddr) {
+	for _, na := range po {
+		if na.Node == nil {
+			nas = append(nas, NodeAddr(na))
+		}
+	}
+	return nas
+}
+
+func (self *testOverlay) EachNode(base []byte, o int, f func(Node) bool) {
+	if base == nil {
+		base = self.addr
+	}
+	for i := o; i < len(self.pos); i++ {
+		for _, na := range self.pos[i] {
+			if na.Node != nil {
+				if !f(na.Node) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (self *testOverlay) EachNodeAddr(base []byte, o int, f func(NodeAddr) bool) {
+	if base == nil {
+		base = self.addr
+	}
+	for i := o; i < len(self.pos); i++ {
+		for _, na := range self.pos[i] {
+			if !f(na) {
+				return
+			}
+		}
+	}
+}
+
+func (self *testOverlay) SuggestNodeAddr() NodeAddr {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	for _, po := range self.pos {
+		ons := self.on(po)
+		if len(ons) < 2 {
+			offs := self.off(po)
+			if len(offs) > 0 {
+				glog.V(6).Infof("node %v is off", offs[0])
+				return offs[0]
+			}
+		}
+	}
+	return nil
+}
+
+func (self *testOverlay) SuggestOrder() int {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	for o, po := range self.pos {
+		off := self.off(po)
+		if len(off) < 5 {
+			glog.V(6).Infof("suggest PO%02d / %v", o, len(self.pos)-1)
+			return o
+		}
+	}
+	return 256
+
+}
+
+func (self *testOverlay) Info() string {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+	var t []string
+	var ons, offs int
+	var ns []Node
+	var nas []NodeAddr
+	for o, po := range self.pos {
+		var row []string
+		ns = self.on(po)
+		ons = len(ns)
+		for _, n := range ns {
+			addr := n.OverlayAddr()
+			row = append(row, fmt.Sprintf("%x", addr[:4]))
+		}
+		row = append(row, "|")
+		nas = self.off(po)
+		offs = len(nas)
+		for _, na := range nas {
+			addr := na.OverlayAddr()
+			row = append(row, fmt.Sprintf("%x", addr[:4]))
+		}
+		t = append(t, fmt.Sprintf("%v: (%v/%v) %v", o, ons, offs, strings.Join(row, " ")))
+	}
+	return strings.Join(t, "\n")
+}
+
+func NewTestOverlay(addr []byte) *testOverlay {
+	return &testOverlay{
+		addr:   addr,
+		posMap: make(map[string]*testNodeAddr),
+		pos:    make([][]*testNodeAddr, orders),
+	}
+}