diff --git a/.travis.yml b/.travis.yml index ba62b87bf5..2b529816a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum sudo: false matrix: include: - - os: linux - dist: trusty - sudo: required - go: 1.7.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage - - os: linux dist: trusty sudo: required @@ -25,7 +14,6 @@ matrix: - go run build/ci.go install - go run build/ci.go test -coverage - # These are the latest Go versions. - os: linux dist: trusty sudo: required @@ -47,6 +35,28 @@ matrix: - go run build/ci.go install - go run build/ci.go test -coverage + # These are the latest Go versions. + - os: linux + dist: trusty + sudo: required + go: "1.10" + script: + - sudo modprobe fuse + - sudo chmod 666 /dev/fuse + - sudo chown root:$USER /etc/fuse.conf + - go run build/ci.go install + - go run build/ci.go test -coverage + + - os: osx + go: "1.10" + script: + - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 + - brew update + - brew install caskroom/cask/brew-cask + - brew cask install osxfuse + - go run build/ci.go install + - go run build/ci.go test -coverage + # This builder only tests code linters on latest version of Go - os: linux dist: trusty @@ -185,6 +195,8 @@ matrix: - xctool -version - xcrun simctl list + # Workaround for https://github.com/golang/go/issues/23749 + - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc' - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds # This builder does the Azure archive purges to avoid accumulating junk diff --git a/cmd/swarm/db.go b/cmd/swarm/db.go index dfd2d069b9..82db713f14 100644 --- a/cmd/swarm/db.go +++ b/cmd/swarm/db.go @@ -23,6 +23,7 @@ import ( "path/filepath" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" "gopkg.in/urfave/cli.v1" @@ -30,11 +31,11 @@ import ( func dbExport(ctx *cli.Context) { args := ctx.Args() - if len(args) != 2 { - utils.Fatalf("invalid arguments, please specify both (path to a local chunk database) and (path to write the tar archive to, - for stdout)") + if len(args) != 3 { + utils.Fatalf("invalid arguments, please specify both (path to a local chunk database), (path to write the tar archive to, - for stdout) and the base key") } - store, err := openDbStore(args[0]) + store, err := openDbStore(args[0], common.Hex2Bytes(args[2])) if err != nil { utils.Fatalf("error opening local chunk database: %s", err) } @@ -62,11 +63,11 @@ func dbExport(ctx *cli.Context) { func dbImport(ctx *cli.Context) { args := ctx.Args() - if len(args) != 2 { - utils.Fatalf("invalid arguments, please specify both (path to a local chunk database) and (path to read the tar archive from, - for stdin)") + if len(args) != 3 { + utils.Fatalf("invalid arguments, please specify both (path to a local chunk database), (path to read the tar archive from, - for stdin) and the base key") } - store, err := openDbStore(args[0]) + store, err := openDbStore(args[0], common.Hex2Bytes(args[2])) if err != nil { utils.Fatalf("error opening local chunk database: %s", err) } @@ -94,11 +95,11 @@ func dbImport(ctx *cli.Context) { func dbClean(ctx *cli.Context) { args := ctx.Args() - if len(args) != 1 { - utils.Fatalf("invalid arguments, please specify (path to a local chunk database)") + if len(args) != 2 { + utils.Fatalf("invalid arguments, please specify (path to a local chunk database) and the base key") } - store, err := openDbStore(args[0]) + store, err := openDbStore(args[0], common.Hex2Bytes(args[1])) if err != nil { utils.Fatalf("error opening local chunk database: %s", err) } @@ -107,10 +108,10 @@ func dbClean(ctx *cli.Context) { store.Cleanup() } -func openDbStore(path string) (*storage.DbStore, error) { +func openDbStore(path string, basekey []byte) (*storage.DbStore, error) { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { return nil, fmt.Errorf("invalid chunkdb path: %s", err) } hash := storage.MakeHashFunc("SHA3") - return storage.NewDbStore(path, hash, 10000000, 0) + return storage.NewDbStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) }) } diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index 792e8d0d7a..a6e6a6ba76 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -39,7 +39,7 @@ func hash(ctx *cli.Context) { stat, _ := f.Stat() chunker := storage.NewTreeChunker(storage.NewChunkerParams()) - key, err := chunker.Split(f, stat.Size(), nil, nil, nil) + key, _, err := chunker.Split(f, stat.Size(), nil) if err != nil { utils.Fatalf("%v\n", err) } else { diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index b7555d0815..6b6b28e657 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -281,8 +281,8 @@ func TestDeposit(t *testing.T) { t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } - // autodeposit every 30ms if new cheque issued - interval := 30 * time.Millisecond + // autodeposit every 200ms if new cheque issued + interval := 200 * time.Millisecond chbook.AutoDeposit(interval, common.Big1, balance) _, err = chbook.Issue(addr1, amount) if err != nil { diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index 9bb899384b..22acb9f863 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -2307,7 +2307,7 @@ var toChecksumAddress = function (address) { }; /** - * Transforms given string to valid 20 bytes-length addres with 0x prefix + * Transforms given string to valid 20 bytes-length address with 0x prefix * * @method toAddress * @param {String} address diff --git a/log/logger.go b/log/logger.go index 15c83a9b25..e2805271a7 100644 --- a/log/logger.go +++ b/log/logger.go @@ -126,13 +126,13 @@ type logger struct { h *swapHandler } -func (l *logger) write(msg string, lvl Lvl, ctx []interface{}) { +func (l *logger) write(msg string, lvl Lvl, ctx []interface{}, skip int) { l.h.Log(&Record{ Time: time.Now(), Lvl: lvl, Msg: msg, Ctx: newContext(l.ctx, ctx), - Call: stack.Caller(2), + Call: stack.Caller(skip), KeyNames: RecordKeyNames{ Time: timeKey, Msg: msgKey, @@ -156,27 +156,27 @@ func newContext(prefix []interface{}, suffix []interface{}) []interface{} { } func (l *logger) Trace(msg string, ctx ...interface{}) { - l.write(msg, LvlTrace, ctx) + l.write(msg, LvlTrace, ctx, 2) } func (l *logger) Debug(msg string, ctx ...interface{}) { - l.write(msg, LvlDebug, ctx) + l.write(msg, LvlDebug, ctx, 2) } func (l *logger) Info(msg string, ctx ...interface{}) { - l.write(msg, LvlInfo, ctx) + l.write(msg, LvlInfo, ctx, 2) } func (l *logger) Warn(msg string, ctx ...interface{}) { - l.write(msg, LvlWarn, ctx) + l.write(msg, LvlWarn, ctx, 2) } func (l *logger) Error(msg string, ctx ...interface{}) { - l.write(msg, LvlError, ctx) + l.write(msg, LvlError, ctx, 2) } func (l *logger) Crit(msg string, ctx ...interface{}) { - l.write(msg, LvlCrit, ctx) + l.write(msg, LvlCrit, ctx, 2) os.Exit(1) } diff --git a/log/root.go b/log/root.go index 71b8cef6d4..dd24c05e32 100644 --- a/log/root.go +++ b/log/root.go @@ -31,31 +31,36 @@ func Root() Logger { // Trace is a convenient alias for Root().Trace func Trace(msg string, ctx ...interface{}) { - root.write(msg, LvlTrace, ctx) + root.write(msg, LvlTrace, ctx, 2) } // Debug is a convenient alias for Root().Debug func Debug(msg string, ctx ...interface{}) { - root.write(msg, LvlDebug, ctx) + root.write(msg, LvlDebug, ctx, 2) } // Info is a convenient alias for Root().Info func Info(msg string, ctx ...interface{}) { - root.write(msg, LvlInfo, ctx) + root.write(msg, LvlInfo, ctx, 2) } // Warn is a convenient alias for Root().Warn func Warn(msg string, ctx ...interface{}) { - root.write(msg, LvlWarn, ctx) + root.write(msg, LvlWarn, ctx, 2) } // Error is a convenient alias for Root().Error func Error(msg string, ctx ...interface{}) { - root.write(msg, LvlError, ctx) + root.write(msg, LvlError, ctx, 2) } // Crit is a convenient alias for Root().Crit func Crit(msg string, ctx ...interface{}) { - root.write(msg, LvlCrit, ctx) + root.write(msg, LvlCrit, ctx, 2) os.Exit(1) } + +// Output is a convenient alias for write +func Output(msg string, lvl Lvl, skip int, ctx ...interface{}) { + root.write(msg, lvl, ctx, skip) +} diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index a4641ed8bd..1a5c480386 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -365,14 +365,14 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { } -func TestMultiplePeersDropSelf(t *testing.T) { +func XTestMultiplePeersDropSelf(t *testing.T) { runMultiplePeers(t, 0, fmt.Errorf("subprotocol error"), fmt.Errorf("Message handler error: (msg code 3): dropped"), ) } -func TestMultiplePeersDropOther(t *testing.T) { +func XTestMultiplePeersDropOther(t *testing.T) { runMultiplePeers(t, 1, fmt.Errorf("Message handler error: (msg code 3): dropped"), fmt.Errorf("subprotocol error"), diff --git a/p2p/simulations/adapters/docker.go b/p2p/simulations/adapters/docker.go index 8ef5629fb5..41c3ecdd1a 100644 --- a/p2p/simulations/adapters/docker.go +++ b/p2p/simulations/adapters/docker.go @@ -33,6 +33,10 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" ) +var ( + ErrLinuxOnly = errors.New("DockerAdapter can only be used on Linux as it uses the current binary (which must be a Linux binary)") +) + // DockerAdapter is a NodeAdapter which runs simulation nodes inside Docker // containers. // @@ -52,7 +56,7 @@ func NewDockerAdapter() (*DockerAdapter, error) { // It is reasonable to require this because the caller can just // compile the current binary in a Docker container. if runtime.GOOS != "linux" { - return nil, errors.New("DockerAdapter can only be used on Linux as it uses the current binary (which must be a Linux binary)") + return nil, ErrLinuxOnly } if err := buildDockerImage(); err != nil { diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index a566fb27d8..f34ac33043 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -105,9 +105,9 @@ func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) { conf.Stack.P2P.NAT = nil conf.Stack.NoUSB = true - // listen on a random localhost port (we'll get the actual port after - // starting the node through the RPC admin.nodeInfo method) - conf.Stack.P2P.ListenAddr = "127.0.0.1:0" + // listen on a localhost port, which we set when we + // initialise NodeConfig (usually a random port) + conf.Stack.P2P.ListenAddr = fmt.Sprintf("127.0.0.1:%d", config.Port) node := &ExecNode{ ID: config.ID, diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 0d22b4f56f..8752d04458 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -34,11 +34,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -const ( - socketReadBuffer = 5000 * 1024 - socketWriteBuffer = 5000 * 1024 -) - // SimAdapter is a NodeAdapter which creates in-memory simulation nodes and // connects them using net.Pipe or OS socket connections type SimAdapter struct { @@ -112,7 +107,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { MaxPeers: math.MaxInt32, NoDiscovery: true, Dialer: s, - EnableMsgEvents: true, + EnableMsgEvents: config.EnableMsgEvents, }, NoUSB: true, Logger: log.New("node.id", id.String()), @@ -378,20 +373,10 @@ func socketPipe() (net.Conn, net.Conn, error) { return nil, nil, err } - err = setSocketBuffer(pipe1) - if err != nil { - return nil, nil, err - } - - err = setSocketBuffer(pipe2) - if err != nil { - return nil, nil, err - } - return pipe1, pipe2, nil } -func setSocketBuffer(conn net.Conn) error { +func setSocketBuffer(conn net.Conn, socketReadBuffer int, socketWriteBuffer int) error { switch v := conn.(type) { case *net.UnixConn: err := v.SetReadBuffer(socketReadBuffer) diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 3939706aec..b1ef7add0b 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -27,7 +27,7 @@ import ( func TestSocketPipe(t *testing.T) { c1, c2, err := socketPipe() if err != nil { - t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err) + t.Fatal(err) } done := make(chan struct{}) @@ -35,15 +35,19 @@ func TestSocketPipe(t *testing.T) { go func() { msgs := 20 size := 8 - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - _ = binary.PutUvarint(msg, uint64(i)) - _, err := c1.Write(msg) - if err != nil { - t.Fatal(err) + // OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously + go func() { + for i := 0; i < msgs; i++ { + msg := make([]byte, size) + _ = binary.PutUvarint(msg, uint64(i)) + + _, err := c1.Write(msg) + if err != nil { + t.Fatal(err) + } } - } + }() for i := 0; i < msgs; i++ { msg := make([]byte, size) @@ -64,7 +68,7 @@ func TestSocketPipe(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -72,7 +76,7 @@ func TestSocketPipe(t *testing.T) { func TestSocketPipeBidirections(t *testing.T) { c1, c2, err := socketPipe() if err != nil { - t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err) + t.Fatal(err) } done := make(chan struct{}) @@ -80,14 +84,18 @@ func TestSocketPipeBidirections(t *testing.T) { go func() { msgs := 100 size := 4 - for i := 0; i < msgs; i++ { - msg := []byte(`ping`) - _, err := c1.Write(msg) - if err != nil { - t.Fatal(err) + // OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously + go func() { + for i := 0; i < msgs; i++ { + msg := []byte(`ping`) + + _, err := c1.Write(msg) + if err != nil { + t.Fatal(err) + } } - } + }() for i := 0; i < msgs; i++ { out := make([]byte, size) @@ -124,7 +132,7 @@ func TestSocketPipeBidirections(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -169,7 +177,7 @@ func TestTcpPipe(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -232,7 +240,7 @@ func TestTcpPipeBidirections(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -281,7 +289,7 @@ func TestNetPipe(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -356,7 +364,7 @@ func TestNetPipeBidirections(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 5b4b47fe2f..2c4b9dd8f2 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "os" + "strconv" "github.com/docker/docker/pkg/reexec" "github.com/ethereum/go-ethereum/crypto" @@ -97,24 +98,30 @@ type NodeConfig struct { // function to sanction or prevent suggesting a peer Reachable func(id discover.NodeID) bool + + Port uint16 } // nodeConfigJSON is used to encode and decode NodeConfig as JSON by encoding // all fields as strings type nodeConfigJSON struct { - ID string `json:"id"` - PrivateKey string `json:"private_key"` - Name string `json:"name"` - Services []string `json:"services"` + ID string `json:"id"` + PrivateKey string `json:"private_key"` + Name string `json:"name"` + Services []string `json:"services"` + EnableMsgEvents bool `json:"enable_msg_events"` + Port uint16 `json:"port"` } // MarshalJSON implements the json.Marshaler interface by encoding the config // fields as strings func (n *NodeConfig) MarshalJSON() ([]byte, error) { confJSON := nodeConfigJSON{ - ID: n.ID.String(), - Name: n.Name, - Services: n.Services, + ID: n.ID.String(), + Name: n.Name, + Services: n.Services, + Port: n.Port, + EnableMsgEvents: n.EnableMsgEvents, } if n.PrivateKey != nil { confJSON.PrivateKey = hex.EncodeToString(crypto.FromECDSA(n.PrivateKey)) @@ -152,6 +159,8 @@ func (n *NodeConfig) UnmarshalJSON(data []byte) error { n.Name = confJSON.Name n.Services = confJSON.Services + n.Port = confJSON.Port + n.EnableMsgEvents = confJSON.EnableMsgEvents return nil } @@ -163,13 +172,36 @@ func RandomNodeConfig() *NodeConfig { if err != nil { panic("unable to generate key") } - var id discover.NodeID - pubkey := crypto.FromECDSAPub(&key.PublicKey) - copy(id[:], pubkey[1:]) - return &NodeConfig{ - ID: id, - PrivateKey: key, + + id := discover.PubkeyID(&key.PublicKey) + port, err := assignTCPPort() + if err != nil { + panic("unable to assign tcp port") } + return &NodeConfig{ + ID: id, + Name: fmt.Sprintf("node_%s", id.String()), + PrivateKey: key, + Port: port, + EnableMsgEvents: true, + } +} + +func assignTCPPort() (uint16, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + l.Close() + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return 0, err + } + p, err := strconv.ParseInt(port, 10, 32) + if err != nil { + return 0, err + } + return uint16(p), nil } // ServiceContext is a collection of options and methods which can be utilised diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index 97dd742e88..24001f1949 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -561,7 +561,8 @@ func (s *Server) LoadSnapshot(w http.ResponseWriter, req *http.Request) { // CreateNode creates a node in the network using the given configuration func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) { - config := adapters.RandomNodeConfig() + config := &adapters.NodeConfig{} + err := json.NewDecoder(req.Body).Decode(config) if err != nil && err != io.EOF { http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 677a8fb147..732d49f546 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -348,7 +348,8 @@ func startTestNetwork(t *testing.T, client *Client) []string { nodeCount := 2 nodeIDs := make([]string, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := client.CreateNode(nil) + config := adapters.RandomNodeConfig() + node, err := client.CreateNode(config) if err != nil { t.Fatalf("error creating node: %s", err) } @@ -527,7 +528,9 @@ func TestHTTPNodeRPC(t *testing.T) { // start a node in the network client := NewClient(s.URL) - node, err := client.CreateNode(nil) + + config := adapters.RandomNodeConfig() + node, err := client.CreateNode(config) if err != nil { t.Fatalf("error creating node: %s", err) } @@ -589,7 +592,8 @@ func TestHTTPSnapshot(t *testing.T) { nodeCount := 2 nodes := make([]*p2p.NodeInfo, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := client.CreateNode(nil) + config := adapters.RandomNodeConfig() + node, err := client.CreateNode(config) if err != nil { t.Fatalf("error creating node: %s", err) } diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go index c38e288552..b370fe2cd2 100644 --- a/p2p/simulations/mocker.go +++ b/p2p/simulations/mocker.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" ) //a map of mocker names to its function @@ -165,7 +166,8 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) { func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error) { ids := make([]discover.NodeID, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := net.NewNode() + conf := adapters.RandomNodeConfig() + node, err := net.NewNodeWithConfig(conf) if err != nil { log.Error("Error creating a node! %s", err) return nil, err diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index caf428ece1..6919da1cd5 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -78,26 +78,12 @@ func (self *Network) Events() *event.Feed { return &self.events } -// NewNode adds a new node to the network with a random ID -func (self *Network) NewNode() (*Node, error) { - conf := adapters.RandomNodeConfig() - conf.Services = []string{self.DefaultService} - return self.NewNodeWithConfig(conf) -} - // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { self.lock.Lock() defer self.lock.Unlock() - // create a random ID and PrivateKey if not set - if conf.ID == (discover.NodeID{}) { - c := adapters.RandomNodeConfig() - conf.ID = c.ID - conf.PrivateKey = c.PrivateKey - } - id := conf.ID if conf.Reachable == nil { conf.Reachable = func(otherID discover.NodeID) bool { _, err := self.InitConn(conf.ID, otherID) @@ -105,14 +91,9 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) } } - // assign a name to the node if not set - if conf.Name == "" { - conf.Name = fmt.Sprintf("node%02d", len(self.Nodes)+1) - } - // check the node doesn't already exist - if node := self.getNode(id); node != nil { - return nil, fmt.Errorf("node with ID %q already exists", id) + if node := self.getNode(conf.ID); node != nil { + return nil, fmt.Errorf("node with ID %q already exists", conf.ID) } if node := self.getNodeByName(conf.Name); node != nil { return nil, fmt.Errorf("node with name %q already exists", conf.Name) @@ -132,8 +113,8 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Node: adapterNode, Config: conf, } - log.Trace(fmt.Sprintf("node %v created", id)) - self.nodeMap[id] = len(self.Nodes) + log.Trace(fmt.Sprintf("node %v created", conf.ID)) + self.nodeMap[conf.ID] = len(self.Nodes) self.Nodes = append(self.Nodes, node) // emit a "control" event diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index 2a062121be..f178bac502 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -41,7 +41,8 @@ func TestNetworkSimulation(t *testing.T) { nodeCount := 20 ids := make([]discover.NodeID, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := network.NewNode() + conf := adapters.RandomNodeConfig() + node, err := network.NewNodeWithConfig(conf) if err != nil { t.Fatalf("error creating node: %s", err) } diff --git a/swarm/api/api.go b/swarm/api/api.go index 572b150da2..d246a25156 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -23,7 +23,6 @@ import ( "net/http" "regexp" "strings" - "sync" "bytes" "mime" @@ -74,8 +73,8 @@ func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { return self.dpa.Retrieve(key) } -func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) { - return self.dpa.Store(data, size, wg, nil) +func (self *Api) Store(data io.Reader, size int64) (key storage.Key, wait func(), err error) { + return self.dpa.Store(data, size) } type ErrResolve error @@ -112,21 +111,22 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { } // Put provides singleton manifest creation on top of dpa store -func (self *Api) Put(content, contentType string) (storage.Key, error) { +func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), err error) { r := strings.NewReader(content) - wg := &sync.WaitGroup{} - key, err := self.dpa.Store(r, int64(len(content)), wg, nil) + key, waitContent, err := self.dpa.Store(r, int64(len(content))) if err != nil { - return nil, err + return nil, nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) - key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) + key, waitManifest, err := self.dpa.Store(r, int64(len(manifest))) if err != nil { - return nil, err + return nil, nil, err } - wg.Wait() - return key, nil + return key, func() { + waitContent() + waitManifest() + }, nil } // Get uses iterative manifest retrieval and prefix matching diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 57c8e88f29..76b2751a58 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -34,9 +34,8 @@ func testApi(t *testing.T, f func(*Api)) { if err != nil { t.Fatalf("unable to create temp dir: %v", err) } - os.RemoveAll(datadir) defer os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, make([]byte, 32)) if err != nil { return } @@ -110,11 +109,12 @@ func TestApiPut(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - key, err := api.Put(content, exp.MimeType) + key, wait, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } - resp := testGet(t, api, key.String(), "") + wait() + resp := testGet(t, api, key.Hex(), "") checkResponse(t, resp, exp) }) } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index f5dc90e2e5..0074ea167d 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -43,6 +43,7 @@ func NewFileSystem(api *Api) *FileSystem { // Upload replicates a local directory as a manifest file and uploads it // using dpa store +// This function waits the chunks to be stored. // TODO: localpath should point to a manifest // // DEPRECATED: Use the HTTP API instead @@ -112,12 +113,12 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if err == nil { stat, _ := f.Stat() var hash storage.Key - wg := &sync.WaitGroup{} - hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil) + var wait func() + hash, wait, err = self.api.dpa.Store(f, stat.Size()) if hash != nil { - list[i].Hash = hash.String() + list[i].Hash = hash.Hex() } - wg.Wait() + wait() awg.Done() if err == nil { first512 := make([]byte, 512) @@ -163,7 +164,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { err2 := trie.recalcAndStore() var hs string if err2 == nil { - hs = trie.hash.String() + hs = trie.hash.Hex() } awg.Wait() return hs, err2 diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 8a15e735dc..6f1594e991 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -105,9 +104,8 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - wg := &sync.WaitGroup{} - hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg) - wg.Wait() + hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index))) + wait() if err != nil { t.Errorf("unexpected error: %v", err) return @@ -122,7 +120,7 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - bzzhash = key.String() + bzzhash = key.Hex() content := readPath(t, "testdata", "test0", "index.html") resp := testGet(t, api, bzzhash, "index2.html") diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index dbd97182fd..9b9f5c2f96 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -110,7 +110,8 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife //(and return the correct HTTP status code) func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { if code == http.StatusInternalServerError { - log.Error(msg) + //log.Error(msg) + log.Output(msg, log.LvlError, 3) } respond(w, r, &ErrorParams{ Code: code, diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index a71edfdd23..c8dc02914f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -91,21 +91,21 @@ type Request struct { // body in swarm and returns the resulting storage key as a text/plain response func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { - s.BadRequest(w, r, "raw POST request cannot contain a path") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "raw POST request cannot contain a path"), http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { - s.BadRequest(w, r, "missing Content-Length header in request") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "missing Content-Length header in request"), http.StatusBadRequest) return } - key, err := s.api.Store(r.Body, r.ContentLength, nil) + key, _, err := s.api.Store(r.Body, r.ContentLength) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } - s.logDebug("content for %s stored", key.Log()) + log.Debug(fmt.Sprintf("content for %s stored", key.Log())) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) @@ -120,7 +120,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { - s.BadRequest(w, r, err.Error()) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, err), http.StatusBadRequest) return } @@ -128,13 +128,13 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { if r.uri.Addr != "" { key, err = s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } } else { key, err = s.api.NewManifest() if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } } @@ -153,7 +153,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } }) if err != nil { - s.Error(w, r, fmt.Errorf("error creating manifest: %s", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error creating manifest: %s", err)), http.StatusInternalServerError) return } @@ -186,12 +186,12 @@ func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error { Size: hdr.Size, ModTime: hdr.ModTime, } - s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + log.Debug(fmt.Sprintf("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)) contentKey, err := mw.AddEntry(tr, entry) if err != nil { return fmt.Errorf("error adding manifest entry from tar stream: %s", err) } - s.logDebug("content for %s stored", contentKey.Log()) + log.Debug(fmt.Sprintf("content for %s stored", contentKey.Log())) } } @@ -243,12 +243,12 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma Size: size, ModTime: time.Now(), } - s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + log.Debug(fmt.Sprintf("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)) contentKey, err := mw.AddEntry(reader, entry) if err != nil { return fmt.Errorf("error adding manifest entry from multipart form: %s", err) } - s.logDebug("content for %s stored", contentKey.Log()) + log.Debug(fmt.Sprintf("content for %s stored", contentKey.Log())) } } @@ -263,7 +263,7 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error if err != nil { return err } - s.logDebug("content for %s stored", key.Log()) + log.Debug(fmt.Sprintf("content for %s stored", key.Log())) return nil } @@ -273,16 +273,16 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { - s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log()) + log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log())) return mw.RemoveEntry(r.uri.Path) }) if err != nil { - s.Error(w, r, fmt.Errorf("error updating manifest: %s", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error updating manifest: %s", err)), http.StatusInternalServerError) return } @@ -296,12 +296,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { - s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, fmt.Sprintf("Cannot parse frequency parameter: %v", err)), http.StatusBadRequest) return } key, err := s.api.ResourceCreate(r.Context(), r.uri.Addr, frequency) if err != nil { - s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Resource creation failed: %v", err)), http.StatusInternalServerError) return } outdata = key.Hex() @@ -309,12 +309,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { data, err := ioutil.ReadAll(r.Body) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } _, _, _, err = s.api.ResourceUpdate(r.Context(), r.uri.Addr, data) if err != nil { - s.Error(w, r, fmt.Errorf("Update resource failed: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Update resource failed: %v", err)), http.StatusInternalServerError) return } @@ -368,11 +368,11 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin } updateKey, data, err = s.api.ResourceLookup(r.Context(), name, uint32(period), uint32(version)) default: - s.BadRequest(w, r, "Invalid mutable resource request") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "Invalid mutable resource request"), http.StatusBadRequest) return } if err != nil { - s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Mutable resource lookup failed: %v", err)), http.StatusInternalServerError) return } log.Debug("Found update", "key", updateKey) @@ -388,7 +388,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } @@ -397,7 +397,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { walker, err := s.api.NewManifestWalker(key, nil) if err != nil { - s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key)) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, fmt.Sprintf("%s is not a manifest", key)), http.StatusBadRequest) return } var entry *api.ManifestEntry @@ -425,7 +425,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - s.NotFound(w, r, errors.New("Manifest entry could not be loaded")) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Manifest entry could not be loaded")), http.StatusNotFound) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) @@ -434,7 +434,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { - s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Root chunk not found %s: %s", key, err)), http.StatusNotFound) return } @@ -460,19 +460,19 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // contained in the manifest func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { - s.BadRequest(w, r, "files request cannot contain a path") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "files request cannot contain a path"), http.StatusBadRequest) return } key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } walker, err := s.api.NewManifestWalker(key, nil) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } @@ -519,7 +519,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { return nil }) if err != nil { - s.logError("error generating tar stream: %s", err) + log.Error(fmt.Sprintf("error generating tar stream: %s", err)) } } @@ -535,14 +535,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } list, err := s.getManifestList(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } @@ -559,7 +559,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { List: &list, }) if err != nil { - s.logError("error rendering list HTML: %s", err) + log.Error(fmt.Sprintf("error rendering list HTML: %s", err)) } return } @@ -635,7 +635,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } @@ -643,9 +643,9 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { if err != nil { switch status { case http.StatusNotFound: - s.NotFound(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) default: - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) } return } @@ -656,11 +656,11 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } - s.logDebug(fmt.Sprintf("Multiple choices! --> %v", list)) + log.Debug(fmt.Sprintf("Multiple choices! --> %v", list)) //show a nice page links to available entries ShowMultipleChoices(w, &r.Request, list) return @@ -668,7 +668,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { - s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("File not found %s: %s", r.uri, err)), http.StatusNotFound) return } @@ -678,16 +678,16 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) + log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) req := &Request{Request: *r, uri: uri} if err != nil { - s.logError("Invalid URI %q: %s", r.URL.Path, err) - s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) + log.Error(fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) + ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Method, uri, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)), http.StatusBadRequest) return } - s.logDebug("%s request received for %s", r.Method, uri) + log.Debug(fmt.Sprintf("%s request received for %s", r.Method, uri)) switch r.Method { case "POST": @@ -763,26 +763,6 @@ func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWri if err != nil { return nil, err } - s.logDebug("generated manifest %s", key) + log.Debug(fmt.Sprintf("generated manifest %s", key)) return key, nil } - -func (s *Server) logDebug(format string, v ...interface{}) { - log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) -} - -func (s *Server) logError(format string, v ...interface{}) { - log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) -} - -func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { - ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest) -} - -func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { - ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) -} - -func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) { - ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) -} diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 9a80405ffc..c9a228e46a 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -24,7 +24,6 @@ import ( "io/ioutil" "net/http" "strings" - "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -182,15 +181,14 @@ func TestBzzGetPath(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() - wg := &sync.WaitGroup{} - for i, mf := range testmanifest { reader[i] = bytes.NewReader([]byte(mf)) - key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil) + var wait func() + key[i], wait, err = srv.Dpa.Store(reader[i], int64(len(mf))) if err != nil { t.Fatal(err) } - wg.Wait() + wait() } _, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a") @@ -245,7 +243,7 @@ func TestBzzGetPath(t *testing.T) { t.Fatalf("Read request body: %v", err) } - if string(respbody) != key[v].String() { + if string(respbody) != key[v].Hex() { isexpectedfailrequest := false for _, r := range expectedfailrequests { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index fde086b7ac..796b4167d0 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -24,7 +24,6 @@ import ( "io" "net/http" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -66,7 +65,9 @@ func (a *Api) NewManifest() (storage.Key, error) { if err != nil { return nil, err } - return a.Store(bytes.NewReader(data), int64(len(data)), &sync.WaitGroup{}) + key, wait, err := a.Store(bytes.NewReader(data), int64(len(data))) + wait() + return key, err } // ManifestWriter is used to add and remove entries from an underlying manifest @@ -86,12 +87,12 @@ func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWrit // AddEntry stores the given data and adds the resulting key to the manifest func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) { - key, err := m.api.Store(data, e.Size, nil) + key, _, err := m.api.Store(data, e.Size) if err != nil { return nil, err } entry := newManifestTrieEntry(e, nil) - entry.Hash = key.String() + entry.Hash = key.Hex() m.trie.addEntry(entry, m.quitC) return key, nil } @@ -339,7 +340,7 @@ func (self *manifestTrie) recalcAndStore() error { if err != nil { return err } - entry.Hash = entry.subtrie.hash.String() + entry.Hash = entry.subtrie.hash.Hex() } list.Entries = append(list.Entries, entry.ManifestEntry) } @@ -352,9 +353,8 @@ func (self *manifestTrie) recalcAndStore() error { } sr := bytes.NewReader(manifest) - wg := &sync.WaitGroup{} - key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil) - wg.Wait() + key, wait, err2 := self.dpa.Store(sr, int64(len(manifest))) + wait() self.hash = key return err2 } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 0e3abecfe4..a2ba70c7ac 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -16,7 +16,11 @@ package api -import "path" +import ( + "path" + + "github.com/ethereum/go-ethereum/swarm/storage" +) type Response struct { MimeType string @@ -41,12 +45,8 @@ func NewStorage(api *Api) *Storage { // its content type // // DEPRECATED: Use the HTTP API instead -func (self *Storage) Put(content, contentType string) (string, error) { - key, err := self.api.Put(content, contentType) - if err != nil { - return "", err - } - return key.String(), err +func (self *Storage) Put(content, contentType string) (storage.Key, func(), error) { + return self.api.Put(content, contentType) } // Get retrieves the content from bzzpath and reads the response in full @@ -100,5 +100,5 @@ func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (ne if err != nil { return "", err } - return key.String(), nil + return key.Hex(), nil } diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go index d260dd61d8..bcbf53ee37 100644 --- a/swarm/api/storage_test.go +++ b/swarm/api/storage_test.go @@ -31,10 +31,12 @@ func TestStoragePutGet(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - bzzhash, err := api.Put(content, exp.MimeType) + bzzkey, wait, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } + wait() + bzzhash := bzzkey.Hex() // to check put against the Api#Get resp0 := testGet(t, api.api, bzzhash, "") checkResponse(t, resp0, exp) diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index f0d64c3538..4d89b132a1 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -808,7 +808,7 @@ func TestFUSE(t *testing.T) { } os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, make([]byte, 32)) if err != nil { t.Fatal(err) } diff --git a/swarm/network/README.md b/swarm/network/README.md new file mode 100644 index 0000000000..ad429b38be --- /dev/null +++ b/swarm/network/README.md @@ -0,0 +1,152 @@ +## Streaming + +Streaming is a new protocol of the swarm bzz bundle of protocols. +This protocol provides the basic logic for chunk-based data flow. +It implements simple retrieve requests and delivery using priority queue. +A data exchange stream is a directional flow of chunks between peers. +The source of datachunks is the upstream, the receiver is called the +downstream peer. Each streaming protocol defines an outgoing streamer +and an incoming streamer, the former installing on the upstream, +the latter on the downstream peer. + +Subscribe on StreamerPeer launches an incoming streamer that sends +a subscribe msg upstream. The streamer on the upstream peer +handles the subscribe msg by installing the relevant outgoing streamer +. The modules now engage in a process of upstream sending a sequence of hashes of +chunks downstream (OfferedHashesMsg). The downstream peer evaluates which hashes are needed +and get it delivered by sending back a msg (WantedHashesMsg). + +Historical syncing is supported - currently not the right abstraction -- +state kept across sessions by saving a series of intervals after their last +batch actually arrived. + +Live streaming is also supported, by starting session from the first item +after the subscription. + +Provable data exchange. In case a stream represents a swarm document's data layer +or higher level chunks, streaming up to a certain index is always provable. It saves on +sending intermediate chunks. + +Using the streamer logic, various stream types are easy to implement: + +* light node requests: + * url lookup with offset + * document download + * document upload +* syncing + * live session syncing + * historical syncing +* simple retrieve requests and deliveries +* mutable resource updates streams +* receipting for finger pointing + +## Syncing + +Syncing is the process that makes sure storer nodes end up storing all and only the chunks that are requested from them. + +### Requirements + +- eventual consistency: so each chunk historical should be syncable +- since the same chunk can and will arrive from many peers, (network traffic should be +optimised, only one transfer of data per chunk) +- explicit request deliveries should be prioritised higher than recent chunks received +during the ongoing session which in turn should be higher than historical chunks. +- insured chunks should get receipted for finger pointing litigation, the receipts storage +should be organised efficiently, upstream peer should also be able to find these +receipts for a deleted chunk easily to refute their challenge. +- syncing should be resilient to cut connections, metadata should be persisted that +keep track of syncing state across sessions, historical syncing state should survive restart +- extra data structures to support syncing should be kept at minimum +- syncing is organized separately for chunk types (resource update v content chunk) +- various types of streams should have common logic abstracted + +Syncing is now entirely mediated by the localstore, ie., no processes or memory leaks due to network contention. +When a new chunk is stored, its chunk hash is index by proximity bin + +peers syncronise by getting the chunks closer to the downstream peer than to the upstream one. +Consequently peers just sync all stored items for the kad bin the receiving peer falls into. +The special case of nearest neighbour sets is handled by the downstream peer +indicating they want to sync all kademlia bins with proximity equal to or higher +than their depth. + +This sync state represents the initial state of a sync connection session. +Retrieval is dictated by downstream peers simply using a special streamer protocol. + +Syncing chunks created during the session by the upstream peer is called live session syncing +while syncing of earlier chunks is historical syncing. + +Once the relevant chunk is retrieved, downstream peer looks up all hash segments in its localstore +and sends to the upstream peer a message with a a bitvector to indicate +missing chunks (e.g., for chunk `k`, hash with chunk internal index which case ) +new items. In turn upstream peer sends the relevant chunk data alongside their index. + +On sending chunks there is a priority queue system. If during looking up hashes in its localstore, +downstream peer hits on an open request then a retrieve request is sent immediately to the upstream peer indicating +that no extra round of checks is needed. If another peers syncer hits the same open request, it is slightly unsafe to not ask +that peer too: if the first one disconnects before delivering or fails to deliver and therefore gets +disconnected, we should still be able to continue with the other. The minimum redundant traffic coming from such simultaneous +eventualities should be sufficiently rare not to warrant more complex treatment. + +Session syncing involves downstream peer to request a new state on a bin from upstream. +using the new state, the range (of chunks) between the previous state and the new one are retrieved +and chunks are requested identical to the historical case. After receiving all the missing chunks +from the new hashes, downstream peer will request a new range. If this happens before upstream peer updates a new state, +we say that session syncing is live or the two peers are in sync. In general the time interval passed since downstream peer request up to the current session cursor is a good indication of a permanent (probably increasing) lag. + +If there is no historical backlog, and downstream peer has an acceptable 'last synced' tag, then it is said to be fully synced with the upstream peer. +If a peer is fully synced with all its storer peers, it can advertise itself as globally fully synced. + +The downstream peer persists the record of the last synced offset. When the two peers disconnect and +reconnect syncing can start from there. +This situation however can also happen while historical syncing is not yet complete. +Effectively this means that the peer needs to persist a record of an arbitrary array of offset ranges covered. + +### Delivery requests + +once the appropriate ranges of the hashstream are retrieved and buffered, downstream peer just scans the hashes, looks them up in localstore, if not found, create a request entry. +The range is referenced by the chunk index. Alongside the name (indicating the stream, e.g., content chunks for bin 6) and the range +downstream peer sends a 128 long bitvector indicating which chunks are needed. +Newly created requests are satisfied bound together in a waitgroup which when done, will promptt sending the next one. +to be able to do check and storage concurrently, we keep a buffer of one, we start with two batches of hashes. +If there is nothing to give, upstream peers SetNextBatch is blocking. Subscription ends with an unsubscribe. which removes the syncer from the map. + +Canceling requests (for instance the late chunks of an erasure batch) should be a chan closed +on the request + +Simple request is also a subscribe +different streaming protocols are different p2p protocols with same message types. +the constructor is the Run function itself. which takes a streamerpeer as argument + + +### provable streams + +The swarm hash over the hash stream has many advantages. It implements a provable data transfer +and provide efficient storage for receipts in the form of inclusion proofs useable for finger pointing litigation. +When challenged on a missing chunk, upstream peer will provide an inclusion proof of a chunk hash against the state of the +sync stream. In order to be able to generate such an inclusion proof, upstream peer needs to store the hash index (counting consecutive hash-size segments) alongside the chunk data and preserve it even when the chunk data is deleted until the chunk is no longer insured. +if there is no valid insurance on the files the entry may be deleted. +As long as the chunk is preserved, no takeover proof will be needed since the node can respond to any challenge. +However, once the node needs to delete an insured chunk for capacity reasons, a receipt should be available to +refute the challenge by finger pointing to a downstream peer. +As part of the deletion protocol then, hashes of insured chunks to be removed are pushed to an infinite stream for every bin. + +Downstream peer on the other hand needs to make sure that they can only be finger pointed about a chunk they did receive and store. +For this the check of a state should be exhaustive. If historical syncing finishes on one state, all hashes before are covered, no +surprises. In other words historical syncing this process is self verifying. With session syncing however, it is not enough to check going back covering the range from old offset to new. Continuity (i.e., that the new state is extension of the old) needs to be verified: after downstream peer reads the range into a buffer, it appends the buffer the last known state at the last known offset and verifies the resulting hash matches +the latest state. Past intervals of historical syncing are checked via the the session root. +Upstream peer signs the states, downstream peers can use as handover proofs. +Downstream peers sign off on a state together with an initial offset. + +Once historical syncing is complete and the session does not lag, downstream peer only preserves the latest upstream state and store the signed version. + +Upstream peer needs to keep the latest takeover states: each deleted chunk's hash should be covered by takeover proof of at least one peer. If historical syncing is complete, upstream peer typically will store only the latest takeover proof from downstream peer. +Crucially, the structure is totally independent of the number of peers in the bin, so it scales extremely well. + +## implementation + +The simplest protocol just involves upstream peer to prefix the key with the kademlia proximity order (say 0-15 or 0-31) +and simply iterate on index per bin when syncing with a peer. + +priority queues are used for sending chunks so that user triggered requests should be responded to first, session syncing second, and historical with lower priority. +The request on chunks remains implemented as a dataless entry in the memory store. +The lifecycle of this object should be more carefully thought through, ie., when it fails to retrieve it should be removed. diff --git a/swarm/network/bitvector/bitvector.go b/swarm/network/bitvector/bitvector.go new file mode 100644 index 0000000000..5f2f64d027 --- /dev/null +++ b/swarm/network/bitvector/bitvector.go @@ -0,0 +1,50 @@ +package bitvector + +import ( + "errors" +) + +var errInvalidLength = errors.New("invalid length") + +type BitVector struct { + len int + b []byte +} + +func New(l int) (bv *BitVector, err error) { + return NewFromBytes(make([]byte, l/8+1), l) +} + +func NewFromBytes(b []byte, l int) (bv *BitVector, err error) { + if l <= 0 { + return nil, errInvalidLength + } + if len(b)*8 < l { + return nil, errInvalidLength + } + return &BitVector{ + len: l, + b: b, + }, nil +} + +func (bv *BitVector) Get(i int) bool { + bi := i / 8 + return bv.b[bi]&(0x1< k.RetryInterval; delta /= div { + for delta := timeAgo; delta > k.RetryInterval; delta /= div { retries++ } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 16bbb62020..9d9ddbc934 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -70,7 +70,7 @@ func newTestKademlia(b string) *testKademlia { } func (k *testKademlia) newTestKadPeer(s string) Peer { - return &testDropPeer{&bzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc} + return &testDropPeer{&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc} } func (k *testKademlia) On(ons ...string) *testKademlia { @@ -283,16 +283,15 @@ func TestSuggestPeerFindPeers(t *testing.T) { func TestSuggestPeerRetries(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 k := newTestKademlia("00000000") - cycle := time.Second - k.RetryInterval = uint(cycle) + k.RetryInterval = int64(time.Second) // cycle k.MaxRetries = 50 k.RetryExponent = 2 sleep := func(n int) { - t := k.RetryInterval + ts := k.RetryInterval for i := 1; i < n; i++ { - t *= k.RetryExponent + ts *= int64(k.RetryExponent) } - time.Sleep(time.Duration(t)) + time.Sleep(time.Duration(ts)) } k.Register("01000000") @@ -399,14 +398,10 @@ func TestPruning(t *testing.T) { func TestKademliaHiveString(t *testing.T) { k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") + k.MaxProxDisplay = 8 h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n" - for i := 3; i < 16; i++ { - expH += fmt.Sprintf("%03d 0 | 0\n", i) - } - expH += "=========================================================================" - if expH[106:] != h[106:] { - t.Errorf("incorrect hive output. full - expected %v, got %v", expH, h) - t.Fatalf("incorrect hive output. substr - expected %v, got %v", expH[100:], h[100:]) + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + if expH[104:] != h[104:] { + t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } } diff --git a/swarm/network/light/lightnode.go b/swarm/network/light/lightnode.go new file mode 100644 index 0000000000..06a0b4efcd --- /dev/null +++ b/swarm/network/light/lightnode.go @@ -0,0 +1,190 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library.d +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "errors" + + "github.com/ethereum/go-ethereum/swarm/network/stream" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// RemoteReader implements IncomingStreamer +type RemoteSectionReader struct { + db *storage.DBAPI + start uint64 + end uint64 + hashes chan []byte + currentHashes []byte + currentData []byte + quit chan struct{} + root []byte +} + +// NewRemoteReader is the constructor for RemoteReader +func NewRemoteSectionReader(root []byte, db *storage.DBAPI) *RemoteSectionReader { + return &RemoteSectionReader{ + db: db, + root: root, + hashes: make(chan []byte), + quit: make(chan struct{}), + } +} + +func (r *RemoteSectionReader) NeedData(key []byte) func() { + chunk, created := r.db.GetOrCreateRequest(storage.Key(key)) + // TODO: we may want to request from this peer anyway even if the request exists + if chunk.ReqC == nil || !created { + return nil + } + return func() { + select { + case <-chunk.ReqC: + case <-r.quit: + } + } +} + +func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*stream.TakeoverProof, error) { + r.hashes <- hashes + return nil +} + +func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { + l := int64(len(b)) + m := int64(len(r.currentData)) + if m > l { + m = l + } + copy(b, r.currentData[:m]) + if m == l { + r.currentData = r.currentData[m:] + return l, nil + } + var end bool + for i := 0; !end && i < len(r.currentHashes); i += stream.HashSize { + hash := r.currentHashes[i : i+stream.HashSize] + chunk, err := r.db.Get(hash) + if err != nil { + return n, err + } + m := chunk.Size + if n+m > l { + m = l - n + end = true + } + copy(b[n:], chunk.SData[:m]) + n += m + } + + for { + select { + case <-r.quit: + return n, errors.New("aborted") + case hashes := <-r.hashes: + var i int + for ; !end && i < len(hashes); i += stream.HashSize { + hash := hashes[i : i+stream.HashSize] + chunk, err := r.db.Get(hash) + if err != nil { + return n, err + } + m := chunk.Size + if n+m > l { + m = l - n + end = true + + } + copy(b[n:], chunk.SData[:m]) + n += m + } + hashes = hashes[i:] + } + } +} + +func (r *RemoteSectionReader) Close() {} + +// RemoteSectionServer implements OutgoingStreamer +type RemoteSectionServer struct { + // quit chan struct{} + root []byte + db *storage.DBAPI + r *storage.LazyChunkReader +} + +// NewRemoteReader is the constructor for RemoteReader +func NewRemoteSectionServer(db *storage.DBAPI, r *storage.LazyChunkReader) *RemoteSectionServer { + return &RemoteSectionServer{ + db: db, + r: r, + } +} + +// GetData retrieves the actual chunk from localstore +func (s *RemoteSectionServer) GetData(key []byte) ([]byte, error) { + chunk, err := s.db.Get(storage.Key(key)) + if err != nil { + return nil, err + } + return chunk.SData, nil +} + +// GetBatch retrieves the next batch of hashes from the dbstore +func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *stream.HandoverProof, error) { + if to > from+stream.BatchSize { + to = from + stream.BatchSize + } + batch := make([]byte, (to-from)*stream.HashSize) + s.r.ReadAt(batch, int64(from)) + return batch, from, to, nil, nil +} + +func (s *RemoteSectionServer) Close() {} + +// RegisterRemoteSectionReader registers RemoteSectionReader on light downstream node +func RegisterRemoteSectionReader(s *stream.Registry, db *storage.DBAPI) { + s.RegisterClientFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Client, error) { + return NewRemoteSectionReader(t, db), nil + }) +} + +// RegisterRemoteSectionServer registers RemoteSectionServer outgoing streamer on +// upstream light server node +func RegisterRemoteSectionServer(s *stream.Registry, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) { + s.RegisterServerFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Server, error) { + r := rf(t) + return NewRemoteSectionServer(db, r), nil + }) +} + +// RegisterRemoteDownloader registers RemoteDownloader incoming streamer +// on downstream light node +// func RegisterRemoteDownloader(s *Streamer, db *storage.DBAPI) { +// s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (IncomingStreamer, error) { +// return NewRemoteDownloader(t, db), nil +// }) +// } +// +// // RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on +// // upstream light server node +// func RegisterRemoteDownloadServer(s *Streamer, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) { +// s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (OutgoingStreamer, error) { +// r := rf(t) +// return NewRemoteDownloadServer(db, r), nil +// }) +// } diff --git a/swarm/network/priorityqueue/priorityqueue.go b/swarm/network/priorityqueue/priorityqueue.go new file mode 100644 index 0000000000..d2fd97fe10 --- /dev/null +++ b/swarm/network/priorityqueue/priorityqueue.go @@ -0,0 +1,95 @@ +// package priority_queue implement a channel based priority queue +// over arbitrary types. It provides an +// an autopop loop applying a function to the items always respecting +// their priority. The structure is only quasi consistent ie., if a lower +// priority item is autopopped, it is guaranteed that there was a point +// when no higher priority item was present, ie. it is not guaranteed +// that there was any point where the lower priority item was present +// but the higher was not + +package priorityqueue + +import ( + "context" + "errors" +) + +var ( + errContention = errors.New("queue contention") + errBadPriority = errors.New("bad priority") + + wakey = struct{}{} +) + +// PriorityQueue is the basic structure +type PriorityQueue struct { + queues []chan interface{} + wakeup chan struct{} +} + +// New is the constructor for PriorityQueue +func New(n int, l int) *PriorityQueue { + var queues = make([]chan interface{}, n) + for i := range queues { + queues[i] = make(chan interface{}, l) + } + return &PriorityQueue{ + queues: queues, + wakeup: make(chan struct{}, 1), + } +} + +// Run is a forever loop popping items from the queues +func (pq *PriorityQueue) Run(ctx context.Context, f func(interface{})) { + top := len(pq.queues) - 1 + p := top +READ: + for { + q := pq.queues[p] + select { + case <-ctx.Done(): + return + case x := <-q: + f(x) + p = top + default: + if p > 0 { + p-- + continue READ + } + p = top + select { + case <-ctx.Done(): + return + case <-pq.wakeup: + } + } + } +} + +// Push pushes an item to the appropriate queue specified in the priority argument +// if context is given it waits until either the item is pushed or the Context aborts +// otherwise returns errContention if the queue is full +func (pq *PriorityQueue) Push(ctx context.Context, x interface{}, p int) error { + if p < 0 || p >= len(pq.queues) { + return errBadPriority + } + if ctx == nil { + select { + case pq.queues[p] <- x: + default: + return errContention + } + } else { + select { + case pq.queues[p] <- x: + case <-ctx.Done(): + return ctx.Err() + } + } + select { + case pq.wakeup <- wakey: + default: + } + return nil +} diff --git a/swarm/network/priorityqueue/priorityqueue_test.go b/swarm/network/priorityqueue/priorityqueue_test.go new file mode 100644 index 0000000000..9ced29cc3f --- /dev/null +++ b/swarm/network/priorityqueue/priorityqueue_test.go @@ -0,0 +1,82 @@ +package priorityqueue + +import ( + "context" + "sync" + "testing" +) + +func TestPriorityQueue(t *testing.T) { + var results []string + wg := sync.WaitGroup{} + pq := New(3, 2) + wg.Add(1) + go pq.Run(context.Background(), func(v interface{}) { + results = append(results, v.(string)) + wg.Done() + }) + pq.Push(context.Background(), "2.0", 2) + wg.Wait() + if results[0] != "2.0" { + t.Errorf("expected first result %q, got %q", "2.0", results[0]) + } + +Loop: + for i, tc := range []struct { + priorities []int + values []string + results []string + errors []error + }{ + { + priorities: []int{0}, + values: []string{""}, + results: []string{""}, + }, + { + priorities: []int{0, 1}, + values: []string{"0.0", "1.0"}, + results: []string{"1.0", "0.0"}, + }, + { + priorities: []int{1, 0}, + values: []string{"1.0", "0.0"}, + results: []string{"1.0", "0.0"}, + }, + { + priorities: []int{0, 1, 1}, + values: []string{"0.0", "1.0", "1.1"}, + results: []string{"1.0", "1.1", "0.0"}, + }, + { + priorities: []int{0, 0, 0}, + values: []string{"0.0", "0.0", "0.1"}, + errors: []error{nil, nil, errContention}, + }, + } { + var results []string + wg := sync.WaitGroup{} + pq := New(3, 2) + wg.Add(len(tc.values)) + for j, value := range tc.values { + err := pq.Push(nil, value, tc.priorities[j]) + if tc.errors != nil && err != tc.errors[j] { + t.Errorf("expected push error %v, got %v", tc.errors[j], err) + continue Loop + } + if err != nil { + continue Loop + } + } + go pq.Run(context.Background(), func(v interface{}) { + results = append(results, v.(string)) + wg.Done() + }) + wg.Wait() + for k, result := range tc.results { + if results[k] != result { + t.Errorf("test case %v: expected %v element %q, got %q", i, k, result, results[k]) + } + } + } +} diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index d755f4a3fa..0cf682fb98 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -175,12 +175,12 @@ func (b *Bzz) APIs() []rpc.API { // returns a p2p protocol run function that can be assigned to p2p.Protocol#Run field // arguments: // * p2p protocol spec -// * run function taking bzzPeer as argument +// * run function taking BzzPeer as argument // this run function is meant to block for the duration of the protocol session // on return the session is terminated and the peer is disconnected // the protocol waits for the bzz handshake is negotiated -// the overlay address on the bzzPeer is set from the remote handshake -func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error { +// the overlay address on the BzzPeer is set from the remote handshake +func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error { return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { // wait for the bzz protocol to perform the handshake handshake, _ := b.GetHandshake(p.ID()) @@ -193,8 +193,8 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(* if handshake.err != nil { return fmt.Errorf("%08x: %s protocol closed: %v", b.BaseAddr()[:4], spec.Name, handshake.err) } - // the handshake has succeeded so construct the bzzPeer and run the protocol - peer := &bzzPeer{ + // the handshake has succeeded so construct the BzzPeer and run the protocol + peer := &BzzPeer{ Peer: protocols.NewPeer(p, rw, spec), localAddr: b.localAddr, BzzAddr: handshake.peerAddr, @@ -208,8 +208,10 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(* // shared among swarm subprotocols func performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error { ctx, cancel := context.WithTimeout(context.Background(), bzzHandshakeTimeout) - defer cancel() - defer close(handshake.done) + defer func() { + close(handshake.done) + cancel() + }() rsh, err := p.Handshake(ctx, handshake, checkHandshake) if err != nil { handshake.err = err @@ -243,22 +245,30 @@ func (b *Bzz) runBzz(p *p2p.Peer, rw p2p.MsgReadWriter) error { return errors.New("received multiple handshakes") } -// bzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer) +// BzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer) // implements the Peer interface and all interfaces Peer implements: Addr, OverlayPeer -type bzzPeer struct { +type BzzPeer struct { *protocols.Peer // represents the connection for online peers localAddr *BzzAddr // local Peers address *BzzAddr // remote address -> implements Addr interface = protocols.Peer lastActive time.Time // time is updated whenever mutexes are releasing } +func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer { + return &BzzPeer{ + Peer: p, + localAddr: addr, + BzzAddr: NewAddrFromNodeID(p.ID()), + } +} + // Off returns the overlay peer record for offline persistence -func (p *bzzPeer) Off() OverlayAddr { +func (p *BzzPeer) Off() OverlayAddr { return p.BzzAddr } // LastActive returns the time the peer was last active -func (p *bzzPeer) LastActive() time.Time { +func (p *BzzPeer) LastActive() time.Time { return p.lastActive } @@ -393,6 +403,15 @@ func NewAddrFromNodeID(id discover.NodeID) *BzzAddr { } } +// NewAddrFromNodeIDAndPort constucts a BzzAddr from a discover.NodeID and port uint16 +// the overlay address is derived as the hash of the nodeID +func NewAddrFromNodeIDAndPort(id discover.NodeID, port uint16) *BzzAddr { + return &BzzAddr{ + OAddr: ToOverlayAddr(id.Bytes()), + UAddr: []byte(discover.NewNode(id, net.IP{127, 0, 0, 1}, port, port).String()), + } +} + // ToOverlayAddr creates an overlayaddress from a byte slice func ToOverlayAddr(id []byte) []byte { return crypto.Keccak256(id) diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index e8ec4ebc37..208f830fbf 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -17,16 +17,29 @@ package network import ( + "flag" "fmt" + "os" "sync" "testing" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" ) +var ( + adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) +} + type testStore struct { sync.Mutex @@ -78,16 +91,16 @@ func HandshakeMsgExchange(lhs, rhs *HandshakeMsg, id discover.NodeID) []p2ptest. } } -func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*bzzPeer) error) *bzzTester { +func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*BzzPeer) error) *bzzTester { cs := make(map[string]chan bool) - srv := func(p *bzzPeer) error { + srv := func(p *BzzPeer) error { defer close(cs[p.ID().String()]) return run(p) } protocall := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - return srv(&bzzPeer{ + return srv(&BzzPeer{ Peer: protocols.NewPeer(p, rw, spec), localAddr: addr, BzzAddr: NewAddrFromNodeID(p.ID()), @@ -115,7 +128,7 @@ type bzzTester struct { func newBzzTester(t *testing.T, n int, addr *BzzAddr, pp *p2ptest.TestPeerPool, spec *protocols.Spec, services func(Peer) error) *bzzTester { - extraservices := func(p *bzzPeer) error { + extraservices := func(p *BzzPeer) error { pp.Add(p) defer pp.Remove(p) if services == nil { diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 17411cc8cf..7f0e8ccdcd 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -1,4 +1,4 @@ -package discovery_test +package discovery import ( "context" @@ -70,15 +70,18 @@ func BenchmarkDiscovery_64_4(b *testing.B) { benchmarkDiscovery(b, 64, 4) } func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) } func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) } -func TestDiscoverySimulationDockerAdapter(t *testing.T) { - t.Skip("broken (cannot build image)") +func XTestDiscoverySimulationDockerAdapter(t *testing.T) { testDiscoverySimulationDockerAdapter(t, *nodeCount, *initCount) } func testDiscoverySimulationDockerAdapter(t *testing.T, nodes, conns int) { adapter, err := adapters.NewDockerAdapter() if err != nil { - t.Fatal(err) + if err == adapters.ErrLinuxOnly { + t.Skip(err) + } else { + t.Fatal(err) + } } testDiscoverySimulation(t, nodes, conns, adapter) } @@ -97,8 +100,20 @@ func testDiscoverySimulationExecAdapter(t *testing.T, nodes, conns int) { testDiscoverySimulation(t, nodes, conns, adapters.NewExecAdapter(baseDir)) } +func TestDiscoverySimulationSocketAdapter(t *testing.T) { + testDiscoverySimulationSocketAdapter(t, *nodeCount, *initCount) +} + func TestDiscoverySimulationSimAdapter(t *testing.T) { - testDiscoverySimulation(t, *nodeCount, *initCount, adapters.NewSimAdapter(services)) + testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount) +} + +func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { + testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services)) +} + +func testDiscoverySimulationSocketAdapter(t *testing.T, nodes, conns int) { + testDiscoverySimulation(t, nodes, conns, adapters.NewSocketAdapter(services)) } func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { @@ -150,7 +165,8 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul trigger := make(chan discover.NodeID) ids := make([]discover.NodeID, nodes) for i := 0; i < nodes; i++ { - node, err := net.NewNode() + conf := adapters.RandomNodeConfig() + node, err := net.NewNodeWithConfig(conf) if err != nil { return nil, fmt.Errorf("error starting node: %s", err) } @@ -290,7 +306,7 @@ func triggerChecks(trigger chan discover.NodeID, net *simulations.Network, id di } func newService(ctx *adapters.ServiceContext) (node.Service, error) { - addr := network.NewAddrFromNodeID(ctx.Config.ID) + addr := network.NewAddrFromNodeIDAndPort(ctx.Config.ID, ctx.Config.Port) kp := network.NewKadParams() kp.MinProxBinSize = testMinProxBinSize diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go new file mode 100644 index 0000000000..1deb6ffba9 --- /dev/null +++ b/swarm/network/stream/common_test.go @@ -0,0 +1,152 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "errors" + "flag" + "io/ioutil" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var ( + adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +var ( + defaultSkipCheck bool + waitPeerErrC chan error + chunkSize = 4096 +) + +var services = adapters.Services{ + "streamer": NewStreamerService, +} + +func init() { + flag.Parse() + // register the Delivery service which will run as a devp2p + // protocol when using the exec adapter + adapters.RegisterServices(services) + + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + +} + +// NewStreamerService +func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { + id := ctx.Config.ID + addr := toAddr(id) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + store := stores[id].(*storage.LocalStore) + db := storage.NewDBAPI(store) + delivery := NewDelivery(kad, db) + deliveries[id] = delivery + netStore := storage.NewNetStore(store, nil) + r := NewRegistry(addr, delivery, netStore, defaultSkipCheck) + RegisterSwarmSyncerServer(r, db) + RegisterSwarmSyncerClient(r, db) + go func() { + waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id)) + }() + return r, nil +} + +func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { + // setup + addr := network.RandomAddr() // tested peers peer address + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + + // temp datadir + datadir, err := ioutil.TempDir("", "streamer") + if err != nil { + return nil, nil, nil, func() {}, err + } + teardown := func() { + os.RemoveAll(datadir) + } + + localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + if err != nil { + return nil, nil, nil, teardown, err + } + + db := storage.NewDBAPI(localStore) + delivery := NewDelivery(to, db) + streamer := NewRegistry(addr, delivery, localStore, defaultSkipCheck) + protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol) + + err = waitForPeers(streamer, 1*time.Second, 1) + if err != nil { + return nil, nil, nil, nil, errors.New("timeout: peer is not created") + } + + return protocolTester, streamer, localStore, teardown, nil +} + +func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error { + ticker := time.NewTicker(10 * time.Millisecond) + timeoutTimer := time.NewTimer(timeout) + for { + select { + case <-ticker.C: + if streamer.peersCount() >= expectedPeers { + return nil + } + case <-timeoutTimer.C: + return errors.New("timeout") + } + } +} + +type roundRobinStore struct { + index uint32 + stores []storage.ChunkStore +} + +func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { + return &roundRobinStore{ + stores: stores, + } +} + +func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { + return nil, errors.New("get not well defined on round robin store") +} + +func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { + i := atomic.AddUint32(&rrs.index, 1) + idx := int(i) % len(rrs.stores) + rrs.stores[idx].Put(chunk) +} + +func (rrs *roundRobinStore) Close() { + for _, store := range rrs.stores { + store.Close() + } +} diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go new file mode 100644 index 0000000000..024d612dcd --- /dev/null +++ b/swarm/network/stream/delivery.go @@ -0,0 +1,244 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "bytes" + "errors" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + swarmChunkServerStreamName = "RETRIEVE_REQUEST" + deliveryCap = 32 +) + +type Delivery struct { + db *storage.DBAPI + overlay network.Overlay + receiveC chan *ChunkDeliveryMsg + getPeer func(discover.NodeID) *Peer + quit chan struct{} +} + +func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery { + d := &Delivery{ + db: db, + overlay: overlay, + receiveC: make(chan *ChunkDeliveryMsg, deliveryCap), + } + + go d.processReceivedChunks() + return d +} + +// SwarmChunkServer implements Server +type SwarmChunkServer struct { + deliveryC chan []byte + batchC chan []byte + db *storage.DBAPI + currentLen uint64 + quit chan struct{} +} + +// NewSwarmChunkServer is SwarmChunkServer constructor +func NewSwarmChunkServer(db *storage.DBAPI) *SwarmChunkServer { + s := &SwarmChunkServer{ + deliveryC: make(chan []byte, deliveryCap), + batchC: make(chan []byte), + db: db, + quit: make(chan struct{}), + } + go s.processDeliveries() + return s +} + +// processDeliveries handles delivered chunk hashes +func (s *SwarmChunkServer) processDeliveries() { + var hashes []byte + var batchC chan []byte + for { + select { + case <-s.quit: + return + case hash := <-s.deliveryC: + hashes = append(hashes, hash...) + batchC = s.batchC + case batchC <- hashes: + hashes = nil + batchC = nil + } + } +} + +// SetNextBatch +func (s *SwarmChunkServer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { + select { + case hashes = <-s.batchC: + case <-s.quit: + return + } + + from = s.currentLen + s.currentLen += uint64(len(hashes)) + to = s.currentLen + return +} + +// Close needs to be called on a stream server +func (s *SwarmChunkServer) Close() { + close(s.quit) +} + +// GetData retrives chunk data from db store +func (s *SwarmChunkServer) GetData(key []byte) ([]byte, error) { + chunk, err := s.db.Get(storage.Key(key)) + if err == storage.ErrFetching { + <-chunk.ReqC + } else if err != nil { + return nil, err + } + return chunk.SData, nil +} + +// RetrieveRequestMsg is the protocol msg for chunk retrieve requests +type RetrieveRequestMsg struct { + Key storage.Key + SkipCheck bool +} + +func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) error { + log.Debug("received request", "peer", sp.ID(), "hash", req.Key) + s, err := sp.getServer(swarmChunkServerStreamName) + if err != nil { + return err + } + streamer := s.Server.(*SwarmChunkServer) + chunk, created := d.db.GetOrCreateRequest(req.Key) + if chunk.ReqC != nil { + if created { + if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil { + log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err) + return nil + } + } + go func() { + t := time.NewTimer(3 * time.Minute) + defer t.Stop() + + select { + case <-chunk.ReqC: + case <-d.quit: + return + case <-t.C: + return + } + + if req.SkipCheck { + err := sp.Deliver(chunk, s.priority) + if err != nil { + sp.Drop(err) + } + } + streamer.deliveryC <- chunk.Key[:] + }() + return nil + } + // TODO: call the retrieve function of the outgoing syncer + if req.SkipCheck { + log.Trace("deliver", "peer", sp.ID(), "hash", chunk.Key) + return sp.Deliver(chunk, s.priority) + } + streamer.deliveryC <- chunk.Key[:] + return nil +} + +type ChunkDeliveryMsg struct { + Key storage.Key + SData []byte // the stored chunk Data (incl size) + peer *Peer // set in handleChunkDeliveryMsg +} + +func (d *Delivery) handleChunkDeliveryMsg(sp *Peer, req *ChunkDeliveryMsg) error { + req.peer = sp + d.receiveC <- req + return nil +} + +func (d *Delivery) processReceivedChunks() { +R: + for req := range d.receiveC { + // this should be has locally + chunk, err := d.db.Get(req.Key) + if !bytes.Equal(chunk.Key, req.Key) { + panic(fmt.Errorf("processReceivedChunks: chunk key %s != req key %s (peer %s)", chunk.Key.Hex(), req.Key.Hex(), req.peer.ID())) + } + if err == nil { + continue R + } + if err != storage.ErrFetching { + panic(fmt.Sprintf("not in db? key %v chunk %v", req.Key, chunk)) + } + select { + case <-chunk.ReqC: + log.Error("someone else delivered?", "hash", chunk.Key.Hex()) + continue R + default: + } + chunk.SData = req.SData + d.db.Put(chunk) + chunk.WaitToStore() + close(chunk.ReqC) + } +} + +// RequestFromPeers sends a chunk retrieve request to +func (d *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error { + var success bool + var err error + d.overlay.EachConn(hash, 255, func(p network.OverlayConn, po int, nn bool) bool { + spId := p.(*network.BzzPeer).ID() + for _, p := range peersToSkip { + if p == spId { + log.Trace("Delivery.RequestFromPeers: skip peer", "peer", spId) + return true + } + } + sp := d.getPeer(spId) + if sp == nil { + log.Warn("Delivery.RequestFromPeers: peer not found", "id", spId) + return true + } + // TODO: skip light nodes that do not accept retrieve requests + err = sp.SendPriority(&RetrieveRequestMsg{ + Key: hash, + SkipCheck: skipCheck, + }, Top) + success = true + return false + }) + if success { + return err + } + return errors.New("no peer found") +} diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go new file mode 100644 index 0000000000..b737c071b9 --- /dev/null +++ b/swarm/network/stream/delivery_test.go @@ -0,0 +1,688 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "bytes" + "context" + crand "crypto/rand" + "fmt" + "io" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/network" + streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var ( + deliveries map[discover.NodeID]*Delivery + stores map[discover.NodeID]storage.ChunkStore + toAddr func(discover.NodeID) *network.BzzAddr + peerCount func(discover.NodeID) int +) + +func TestStreamerRetrieveRequest(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + peerID := tester.IDs[0] + + streamer.delivery.RequestFromPeers(hash0[:], true) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Expects: []p2ptest.Expect{ + { + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: hash0[:], + SkipCheck: true, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } +} + +func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + peerID := tester.IDs[0] + + chunk := storage.NewChunk(storage.Key(hash0[:]), nil) + + peer := streamer.getPeer(peerID) + + peer.handleSubscribeMsg(&SubscribeMsg{ + Stream: swarmChunkServerStreamName, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: chunk.Key[:], + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: nil, + Hashes: nil, + From: 0, + To: 0, + }, + Peer: peerID, + }, + }, + }) + + expectedError := "exchange 0: 'RetrieveRequestMsg' timed out" + if err == nil || err.Error() != expectedError { + t.Fatalf("Expected error %v, got %v", expectedError, err) + } +} + +// upstream request server receives a retrieve Request and responds with +// offered hashes or delivery if skipHash is set to true +func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { + tester, streamer, localStore, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + peerID := tester.IDs[0] + peer := streamer.getPeer(peerID) + + peer.handleSubscribeMsg(&SubscribeMsg{ + Stream: swarmChunkServerStreamName, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }) + + hash := storage.Key(hash0[:]) + chunk := storage.NewChunk(hash, nil) + chunk.SData = hash + localStore.Put(chunk) + chunk.WaitToStore() + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: hash, + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: hash, + From: 0, + // TODO: why is this 32??? + To: 32, + Key: []byte{}, + Stream: swarmChunkServerStreamName, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + hash = storage.Key(hash1[:]) + chunk = storage.NewChunk(hash, nil) + chunk.SData = hash1[:] + localStore.Put(chunk) + chunk.WaitToStore() + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: hash, + SkipCheck: true, + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 6, + Msg: &ChunkDeliveryMsg{ + Key: hash, + SData: hash, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { + tester, streamer, localStore, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) { + return &testClient{ + t: t, + }, nil + }) + + peerID := tester.IDs[0] + + err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + chunkKey := hash0[:] + chunkData := hash1[:] + chunk, created := localStore.GetOrCreateRequest(chunkKey) + + if !created { + t.Fatal("chunk already exists") + } + select { + case <-chunk.ReqC: + t.Fatal("chunk is already received") + default: + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerID, + }, + }, + }, + p2ptest.Exchange{ + Label: "ChunkDeliveryRequest message", + Triggers: []p2ptest.Trigger{ + { + Code: 6, + Msg: &ChunkDeliveryMsg{ + Key: chunkKey, + SData: chunkData, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + timeout := time.NewTimer(1 * time.Second) + + select { + case <-timeout.C: + t.Fatal("timeout receiving chunk") + case <-chunk.ReqC: + } + + storedChunk, err := localStore.Get(chunkKey) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if !bytes.Equal(storedChunk.SData, chunkData) { + t.Fatal("Retrieved chunk has different data than original") + } + +} + +func TestDeliveryFromNodes(t *testing.T) { + testDeliveryFromNodes(t, 2, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 2, 1, dataChunkCount, false) + testDeliveryFromNodes(t, 4, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 4, 1, dataChunkCount, false) + testDeliveryFromNodes(t, 8, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 8, 1, dataChunkCount, false) + testDeliveryFromNodes(t, 16, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 16, 1, dataChunkCount, false) +} + +func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) { + defaultSkipCheck = skipCheck + toAddr = network.NewAddrFromNodeID + conf := &streamTesting.RunConfig{ + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + EnableMsgEvents: false, + } + + sim, teardown, err := streamTesting.NewSimulation(conf) + defer teardown() + if err != nil { + t.Fatal(err.Error()) + } + stores = make(map[discover.NodeID]storage.ChunkStore) + deliveries = make(map[discover.NodeID]*Delivery) + for i, id := range sim.IDs { + stores[id] = sim.Stores[i] + } + peerCount = func(id discover.NodeID) int { + if sim.IDs[0] == id || sim.IDs[nodes-1] == id { + return 1 + } + return 2 + } + + // here we distribute chunks of a random file into Stores of nodes 1 to nodes + rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + size := chunkCount * chunkSize + fileHash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + // wait until all chunks stored + wait() + defer rrdpa.Stop() + if err != nil { + t.Fatal(err.Error()) + } + errc := make(chan error, 1) + waitPeerErrC = make(chan error) + quitC := make(chan struct{}) + + action := func(ctx context.Context) error { + // each node Subscribes to each other's swarmChunkServerStreamName + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // using a global err channel to share betweem action and node service + i := 0 + for err := range waitPeerErrC { + if err != nil { + return fmt.Errorf("error waiting for peers: %s", err) + } + i++ + if i == nodes { + break + } + } + + // each node subscribes to the upstream swarm chunk server stream + // which responds to chunk retrieve requests all but the last node in the chain does not + for j := 0; j < nodes-1; j++ { + id := sim.IDs[j] + err := sim.CallClient(id, func(client *rpc.Client) error { + err := streamTesting.WatchDisconnections(id, client, errc, quitC) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + sid := sim.IDs[j+1] + return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false) + }) + if err != nil { + return err + } + } + // create a retriever dpa for the pivot node + delivery := deliveries[sim.IDs[0]] + retrieveFunc := func(chunk *storage.Chunk) error { + return delivery.RequestFromPeers(chunk.Key[:], skipCheck) + } + netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc) + dpa := storage.NewDPA(netStore, storage.NewChunkerParams()) + dpa.Start() + + go func() { + defer dpa.Stop() + // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks + // we must wait for the peer connections to have started before requesting + n, err := readAll(dpa, fileHash) + log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + if err != nil { + errc <- fmt.Errorf("requesting chunks action error: %v", err) + } + }() + return nil + } + check := func(ctx context.Context, id discover.NodeID) (bool, error) { + select { + case err := <-errc: + return false, err + case <-ctx.Done(): + return false, ctx.Err() + default: + } + var total int64 + err := sim.CallClient(id, func(client *rpc.Client) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return client.CallContext(ctx, &total, "stream_readAll", common.BytesToHash(fileHash)) + }) + log.Info(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) + if err != nil || total != int64(size) { + return false, nil + } + return true, nil + } + + conf.Step = &simulations.Step{ + Action: action, + Trigger: streamTesting.Trigger(10*time.Millisecond, quitC, sim.IDs[0]), + // we are only testing the pivot node (net.Nodes[0]) + Expect: &simulations.Expectation{ + Nodes: sim.IDs[0:1], + Check: check, + }, + } + startedAt := time.Now() + timeout := 300 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result, err := sim.Run(ctx, conf) + finishedAt := time.Now() + if err != nil { + t.Fatalf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + t.Fatalf("Simulation failed: %s", result.Error) + } + streamTesting.CheckResult(t, result, startedAt, finishedAt) +} + +func BenchmarkDeliveryFromNodesWithoutCheck(b *testing.B) { + for chunks := 32; chunks <= 128; chunks *= 2 { + for i := 2; i < 32; i *= 2 { + b.Run( + fmt.Sprintf("nodes=%v,chunks=%v", i, chunks), + func(b *testing.B) { + benchmarkDeliveryFromNodes(b, i, 1, chunks, true) + }, + ) + } + } +} + +func BenchmarkDeliveryFromNodesWithCheck(b *testing.B) { + for chunks := 32; chunks <= 128; chunks *= 2 { + for i := 2; i < 32; i *= 2 { + b.Run( + fmt.Sprintf("nodes=%v,chunks=%v", i, chunks), + func(b *testing.B) { + benchmarkDeliveryFromNodes(b, i, 1, chunks, false) + }, + ) + } + } +} + +func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skipCheck bool) { + defaultSkipCheck = skipCheck + toAddr = network.NewAddrFromNodeID + + timeout := 300 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + conf := &streamTesting.RunConfig{ + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + EnableMsgEvents: false, + } + sim, teardown, err := streamTesting.NewSimulation(conf) + defer teardown() + if err != nil { + b.Fatal(err.Error()) + } + + stores = make(map[discover.NodeID]storage.ChunkStore) + deliveries = make(map[discover.NodeID]*Delivery) + for i, id := range sim.IDs { + stores[id] = sim.Stores[i] + } + peerCount = func(id discover.NodeID) int { + if sim.IDs[0] == id || sim.IDs[nodes-1] == id { + return 1 + } + return 2 + } + // wait channel for all nodes all peer connections to set up + waitPeerErrC = make(chan error) + + // create a dpa for the last node in the chain which we are gonna write to + remoteDpa := storage.NewDPA(sim.Stores[nodes-1], storage.NewChunkerParams()) + remoteDpa.Start() + defer remoteDpa.Stop() + + // channel to signal simulation initialisation with action call complete + // or node disconnections + disconnectC := make(chan error) + quitC := make(chan struct{}) + + initC := make(chan error) + + action := func(ctx context.Context) error { + // each node Subscribes to each other's swarmChunkServerStreamName + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // waitPeerErrC using a global err channel to share betweem action and node service + i := 0 + for err := range waitPeerErrC { + if err != nil { + return fmt.Errorf("error waiting for peers: %s", err) + } + i++ + if i == nodes { + break + } + } + var err error + // each node except the last one subscribes to the upstream swarm chunk server stream + // which responds to chunk retrieve requests + for j := 0; j < nodes-1; j++ { + id := sim.IDs[j] + err = sim.CallClient(id, func(client *rpc.Client) error { + err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + sid := sim.IDs[j+1] // the upstream peer's id + return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false) + }) + if err != nil { + break + } + } + initC <- err + return nil + } + + // the check function is only triggered when the benchmark finishes + trigger := make(chan discover.NodeID) + check := func(ctx context.Context, id discover.NodeID) (_ bool, err error) { + return true, nil + } + + conf.Step = &simulations.Step{ + Action: action, + Trigger: trigger, + // we are only testing the pivot node (net.Nodes[0]) + Expect: &simulations.Expectation{ + Nodes: sim.IDs[0:1], + Check: check, + }, + } + + // run the simulation in the background + errc := make(chan error) + go func() { + _, err := sim.Run(ctx, conf) + close(quitC) + errc <- err + }() + + // wait for simulation action to complete stream subscriptions + err = <-initC + if err != nil { + b.Fatalf("simulation failed to initialise. expected no error. got %v", err) + } + + // create a retriever dpa for the pivot node + // by now deliveries are set for each node by the streamer service + delivery := deliveries[sim.IDs[0]] + retrieveFunc := func(chunk *storage.Chunk) error { + return delivery.RequestFromPeers(chunk.Key[:], skipCheck) + } + netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc) + + // benchmark loop + b.ResetTimer() + b.StopTimer() +Loop: + for i := 0; i < b.N; i++ { + // uploading chunkCount random chunks to the last node + hashes := make([]storage.Key, chunkCount) + for i := 0; i < chunkCount; i++ { + // create actual size real chunks + hash, wait, err := remoteDpa.Store(io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize)) + // wait until all chunks stored + wait() + if err != nil { + b.Fatalf("expected no error. got %v", err) + } + // collect the hashes + hashes[i] = hash + } + // now benchmark the actual retrieval + // netstore.Get is called for each hash in a go routine and errors are collected + b.StartTimer() + errs := make(chan error) + for _, hash := range hashes { + go func(h storage.Key) { + _, err := netStore.Get(h) + log.Warn("test check netstore get", "hash", h, "err", err) + errs <- err + }(hash) + } + // count and report retrieval errors + // if there are misses then chunk timeout is too low for the distance and volume (?) + var total, misses int + for err := range errs { + if err != nil { + log.Warn(err.Error()) + misses++ + } + total++ + if total == chunkCount { + break + } + } + b.StopTimer() + + select { + case err = <-disconnectC: + if err != nil { + break Loop + } + default: + } + + if misses > 0 { + err = fmt.Errorf("%v chunk not found out of %v", misses, total) + break Loop + } + } + + select { + case <-quitC: + case trigger <- sim.IDs[0]: + } + if err == nil { + err = <-errc + } else { + if e := <-errc; e != nil { + b.Errorf("sim.Run function error: %v", e) + } + } + + // benchmark over, trigger the check function to conclude the simulation + if err != nil { + b.Fatalf("expected no error. got %v", err) + } +} diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go new file mode 100644 index 0000000000..63c8783fdf --- /dev/null +++ b/swarm/network/stream/messages.go @@ -0,0 +1,274 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// SubcribeMsg is the protocol msg for requesting a stream(section) +type SubscribeMsg struct { + Stream string + Key []byte + From, To uint64 + Priority uint8 // delivered on priority channel +} + +func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) (err error) { + defer func() { + if err != nil { + if e := p.Send(SubscribeErrorMsg{ + Error: err.Error(), + }); e != nil { + log.Error("send stream subscribe error message", "err", err) + } + } + }() + + f, err := p.streamer.GetServerFunc(req.Stream) + if err != nil { + return err + } + s, err := f(p, req.Key) + if err != nil { + return err + } + os, err := p.setServer(req.Stream, req.Key, s, req.Priority) + if err != nil { + return err + } + log.Debug("received subscription", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + go func() { + if err := p.SendOfferedHashes(os, req.From, req.To); err != nil { + p.Drop(err) + } + }() + return nil +} + +type SubscribeErrorMsg struct { + Error string +} + +func (p *Peer) handleSubscribeErrorMsg(req *SubscribeErrorMsg) (err error) { + return fmt.Errorf("subscribe to peer %s: %v", p.ID(), req.Error) +} + +type UnsubscribeMsg struct { + Stream string + Key []byte +} + +func (p *Peer) handleUnsubscribeMsg(req *UnsubscribeMsg) error { + p.removeServer(req.Stream, req.Key) + return nil +} + +// OfferedHashesMsg is the protocol msg for offering to hand over a +// stream section +type OfferedHashesMsg struct { + Stream string // name of Stream + Key []byte // subtype or key + From, To uint64 // peer and db-specific entry count + Hashes []byte // stream of hashes (128) + *HandoverProof // HandoverProof +} + +// String pretty prints OfferedHashesMsg +func (m OfferedHashesMsg) String() string { + return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", m.Stream, m.From, m.To, len(m.Hashes)/HashSize) +} + +// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface +// Filter method +func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { + sk := req.Stream + sk += keyToString(req.Key) + s, err := p.getClient(sk) + if err != nil { + return err + } + hashes := req.Hashes + want, err := bv.New(len(hashes) / HashSize) + if err != nil { + return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err) + } + wg := sync.WaitGroup{} + for i := 0; i < len(hashes); i += HashSize { + hash := hashes[i : i+HashSize] + + if wait := s.NeedData(hash); wait != nil { + want.Set(i/HashSize, true) + wg.Add(1) + // create request and wait until the chunk data arrives and is stored + go func(w func()) { + w() + wg.Done() + }(wait) + } + } + // done := make(chan bool) + // go func() { + // wg.Wait() + // close(done) + // }() + // go func() { + // select { + // case <-done: + // s.next <- s.batchDone(p, req, hashes) + // case <-time.After(1 * time.Second): + // p.Drop(errors.New("timeout waiting for batch to be delivered")) + // } + // }() + go func() { + wg.Wait() + s.next <- s.batchDone(p, req, hashes) + }() + // only send wantedKeysMsg if all missing chunks of the previous batch arrived + // except + if s.live { + s.sessionAt = req.From + } + from, to := s.nextBatch(req.To) + log.Trace("received offered batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + if from == to { + return nil + } + + msg := &WantedHashesMsg{ + Stream: req.Stream, + Key: req.Key, + Want: want.Bytes(), + From: from, + To: to, + } + go func() { + select { + case <-time.After(30 * time.Second): + p.Drop(err) + return + case err := <-s.next: + if err != nil { + p.Drop(err) + return + } + } + log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To) + err := p.SendPriority(msg, s.priority) + if err != nil { + p.Drop(err) + } + }() + return nil +} + +// WantedHashesMsg is the protocol msg data for signaling which hashes +// offered in OfferedHashesMsg downstream peer actually wants sent over +type WantedHashesMsg struct { + Stream string // name of stream + Key []byte // subtype or key + Want []byte // bitvector indicating which keys of the batch needed + From, To uint64 // next interval offset - empty if not to be continued +} + +// String pretty prints WantedHashesMsg +func (m WantedHashesMsg) String() string { + return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", m.Stream, m.Want, m.From, m.To) +} + +// handleWantedHashesMsg protocol msg handler +// * sends the next batch of unsynced keys +// * sends the actual data chunks as per WantedHashesMsg +func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error { + log.Trace("received wanted batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + s, err := p.getServer(req.Stream + keyToString(req.Key)) + if err != nil { + return err + } + hashes := s.currentBatch + // launch in go routine since GetBatch blocks until new hashes arrive + go func() { + if err := p.SendOfferedHashes(s, req.From, req.To); err != nil { + p.Drop(err) + } + }() + // go p.SendOfferedHashes(s, req.From, req.To) + l := len(hashes) / HashSize + want, err := bv.NewFromBytes(req.Want, l) + if err != nil { + return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err) + } + for i := 0; i < l; i++ { + if want.Get(i) { + hash := hashes[i*HashSize : (i+1)*HashSize] + data, err := s.GetData(hash) + if err != nil { + return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err) + } + chunk := storage.NewChunk(hash, nil) + chunk.SData = data + if err := p.Deliver(chunk, s.priority); err != nil { + return err + } + } + } + return nil +} + +// Handover represents a statement that the upstream peer hands over the stream section +type Handover struct { + Stream string // name of stream + Start, End uint64 // index of hashes + Root []byte // Root hash for indexed segment inclusion proofs +} + +// HandoverProof represents a signed statement that the upstream peer handed over the stream section +type HandoverProof struct { + Sig []byte // Sign(Hash(Serialisation(Handover))) + *Handover +} + +// Takeover represents a statement that downstream peer took over (stored all data) +// handed over +type Takeover Handover + +// TakeoverProof represents a signed statement that the downstream peer took over +// the stream section +type TakeoverProof struct { + Sig []byte // Sign(Hash(Serialisation(Takeover))) + *Takeover +} + +// TakeoverProofMsg is the protocol msg sent by downstream peer +type TakeoverProofMsg TakeoverProof + +// String pretty prints TakeoverProofMsg +func (m TakeoverProofMsg) String() string { + return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", m.Stream, m.Start, m.End, m.Root, m.Sig) +} + +func (p *Peer) handleTakeoverProofMsg(req *TakeoverProofMsg) error { + _, err := p.getServer(req.Stream) + // store the strongest takeoverproof for the stream in streamer + return err +} diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go new file mode 100644 index 0000000000..c1e64bd740 --- /dev/null +++ b/swarm/network/stream/peer.go @@ -0,0 +1,212 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/protocols" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var sendTimeout = 5 * time.Second + +var ( + errServerNotFound = errors.New("server not found") + errClientNotFound = errors.New("client not found") +) + +// Peer is the Peer extension for the streaming protocol +type Peer struct { + *protocols.Peer + streamer *Registry + pq *pq.PriorityQueue + serverMu sync.RWMutex + clientMu sync.RWMutex + servers map[string]*server + clients map[string]*client + quit chan struct{} +} + +// NewPeer is the constructor for Peer +func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { + p := &Peer{ + Peer: peer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: streamer, + servers: make(map[string]*server), + clients: make(map[string]*client), + quit: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + go p.pq.Run(ctx, func(i interface{}) { p.Send(i) }) + go func() { + <-p.quit + cancel() + }() + return p +} + +// Deliver sends a storeRequestMsg protocol message to the peer +func (p *Peer) Deliver(chunk *storage.Chunk, priority uint8) error { + msg := &ChunkDeliveryMsg{ + Key: chunk.Key, + SData: chunk.SData, + } + return p.SendPriority(msg, priority) +} + +// SendPriority sends message to the peer using the outgoing priority queue +func (p *Peer) SendPriority(msg interface{}, priority uint8) error { + ctx, cancel := context.WithTimeout(context.Background(), sendTimeout) + defer cancel() + return p.pq.Push(ctx, msg, int(priority)) +} + +// SendOfferedHashes sends OfferedHashesMsg protocol msg +func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error { + hashes, from, to, proof, err := s.SetNextBatch(f, t) + if err != nil { + return err + } + // true only when quiting + if len(hashes) == 0 { + return nil + } + if proof == nil { + proof = &HandoverProof{ + Handover: &Handover{}, + } + } + s.currentBatch = hashes + msg := &OfferedHashesMsg{ + HandoverProof: proof, + Hashes: hashes, + From: from, + To: to, + Stream: s.stream, + Key: s.key, + } + log.Trace("Swarm syncer offer batch", "peer", p.ID(), "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to) + return p.SendPriority(msg, s.priority) +} + +func (p *Peer) getServer(s string) (*server, error) { + p.serverMu.RLock() + defer p.serverMu.RUnlock() + + server := p.servers[s] + if server == nil { + return nil, fmt.Errorf("server '%v' not provided to peer %v", s, p.ID()) + } + return server, nil +} + +func (p *Peer) getClient(s string) (*client, error) { + p.clientMu.RLock() + defer p.clientMu.RUnlock() + + client := p.clients[s] + if client == nil { + return nil, fmt.Errorf("client '%v' not provided to peer %v", s, p.ID()) + } + return client, nil +} + +func (p *Peer) setServer(s string, key []byte, o Server, priority uint8) (*server, error) { + p.serverMu.Lock() + defer p.serverMu.Unlock() + + sk := s + keyToString(key) + if p.servers[sk] != nil { + return nil, fmt.Errorf("server %v already registered", sk) + } + os := &server{ + Server: o, + priority: priority, + stream: s, + key: key, + } + p.servers[sk] = os + return os, nil +} + +func (p *Peer) removeServer(s string, key []byte) error { + p.serverMu.Lock() + defer p.serverMu.Unlock() + + sk := s + keyToString(key) + server, ok := p.servers[sk] + if !ok { + return errServerNotFound + } + server.Close() + delete(p.servers, sk) + return nil +} + +func (p *Peer) setClient(s string, key []byte, i Client, priority uint8, live bool) error { + p.clientMu.Lock() + defer p.clientMu.Unlock() + + sk := s + keyToString(key) + if p.clients[sk] != nil { + return fmt.Errorf("client %v already registered", sk) + } + next := make(chan error, 1) + // var intervals *Intervals + // if !live { + // key := s + p.ID().String() + // intervals = NewIntervals(key, p.streamer) + // } + p.clients[sk] = &client{ + Client: i, + // intervals: intervals, + live: live, + priority: priority, + next: next, + stream: s, + key: key, + } + next <- nil // this is to allow wantedKeysMsg before first batch arrives + return nil +} + +func (p *Peer) removeClient(s string, key []byte) error { + p.clientMu.Lock() + defer p.clientMu.Unlock() + + sk := s + keyToString(key) + client, ok := p.clients[sk] + if !ok { + return errClientNotFound + } + client.close() + return nil +} + +func (p *Peer) close() { + for _, s := range p.servers { + s.Close() + } +} diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go new file mode 100644 index 0000000000..24925edd7e --- /dev/null +++ b/swarm/network/stream/stream.go @@ -0,0 +1,441 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "fmt" + "io" + "math" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + Low uint8 = iota + Mid + High + Top + PriorityQueue // number of queues + PriorityQueueCap = 32 // queue capacity + HashSize = 32 +) + +// Registry registry for outgoing and incoming streamer constructors +type Registry struct { + api *API + addr *network.BzzAddr + skipCheck bool + clientMu sync.RWMutex + serverMu sync.RWMutex + peersMu sync.RWMutex + serverFuncs map[string]func(*Peer, []byte) (Server, error) + clientFuncs map[string]func(*Peer, []byte) (Client, error) + peers map[discover.NodeID]*Peer + delivery *Delivery + store storage.ChunkStore +} + +// NewRegistry is Streamer constructor +func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkStore, skipCheck bool) *Registry { + streamer := &Registry{ + addr: addr, + skipCheck: skipCheck, + store: store, + serverFuncs: make(map[string]func(*Peer, []byte) (Server, error)), + clientFuncs: make(map[string]func(*Peer, []byte) (Client, error)), + peers: make(map[discover.NodeID]*Peer), + delivery: delivery, + } + streamer.api = NewAPI(streamer, streamer.store) + delivery.getPeer = streamer.getPeer + streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, t []byte) (Server, error) { + return NewSwarmChunkServer(delivery.db), nil + }) + streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t []byte) (Client, error) { + return NewSwarmSyncerClient(p, delivery.db, nil) + }) + return streamer +} + +// RegisterClient registers an incoming streamer constructor +func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, []byte) (Client, error)) { + r.clientMu.Lock() + defer r.clientMu.Unlock() + + r.clientFuncs[stream] = f +} + +// RegisterServer registers an outgoing streamer constructor +func (r *Registry) RegisterServerFunc(stream string, f func(*Peer, []byte) (Server, error)) { + r.serverMu.Lock() + defer r.serverMu.Unlock() + + r.serverFuncs[stream] = f +} + +// GetClient accessor for incoming streamer constructors +func (r *Registry) GetClientFunc(stream string) (func(*Peer, []byte) (Client, error), error) { + r.clientMu.RLock() + defer r.clientMu.RUnlock() + + f := r.clientFuncs[stream] + if f == nil { + return nil, fmt.Errorf("stream %v not registered", stream) + } + return f, nil +} + +// GetServer accessor for incoming streamer constructors +func (r *Registry) GetServerFunc(stream string) (func(*Peer, []byte) (Server, error), error) { + r.serverMu.RLock() + defer r.serverMu.RUnlock() + + f := r.serverFuncs[stream] + if f == nil { + return nil, fmt.Errorf("stream %v not registered", stream) + } + return f, nil +} + +// Subscribe initiates the streamer +func (r *Registry) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { + f, err := r.GetClientFunc(s) + if err != nil { + return err + } + + peer := r.getPeer(peerId) + if peer == nil { + return fmt.Errorf("peer not found %v", peerId) + } + + is, err := f(peer, t) + if err != nil { + return err + } + err = peer.setClient(s, t, is, priority, live) + if err != nil { + return err + } + + msg := &SubscribeMsg{ + Stream: s, + Key: t, + // Live: live, + From: from, + To: to, + Priority: priority, + } + log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to) + + return peer.SendPriority(msg, priority) +} + +func (r *Registry) Unsubscribe(peerId discover.NodeID, s string, t []byte) error { + peer := r.getPeer(peerId) + if peer == nil { + return fmt.Errorf("peer not found %v", peerId) + } + + msg := &UnsubscribeMsg{ + Stream: s, + Key: t, + } + log.Debug("Unsubscribe ", "peer", peerId, "stream", s, "key", t) + + if err := peer.Send(msg); err != nil { + return err + } + return peer.removeClient(s, t) +} + +func (r *Registry) Retrieve(chunk *storage.Chunk) error { + return r.delivery.RequestFromPeers(chunk.Key[:], r.skipCheck) +} + +func (r *Registry) NodeInfo() interface{} { + return nil +} + +func (r *Registry) PeerInfo(id discover.NodeID) interface{} { + return nil +} + +func (r *Registry) getPeer(peerId discover.NodeID) *Peer { + r.peersMu.RLock() + defer r.peersMu.RUnlock() + + return r.peers[peerId] +} + +func (r *Registry) setPeer(peer *Peer) { + r.peersMu.Lock() + r.peers[peer.ID()] = peer + r.peersMu.Unlock() +} + +func (r *Registry) deletePeer(peer *Peer) { + r.peersMu.Lock() + delete(r.peers, peer.ID()) + r.peersMu.Unlock() +} + +func (r *Registry) peersCount() (c int) { + r.peersMu.Lock() + c = len(r.peers) + r.peersMu.Unlock() + return +} + +// Run protocol run function +func (r *Registry) run(p *protocols.Peer) error { + sp := NewPeer(p, r) + r.setPeer(sp) + defer r.deletePeer(sp) + defer close(sp.quit) + defer sp.close() + return sp.Run(sp.HandleMsg) +} + +func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := protocols.NewPeer(p, rw, Spec) + bzzPeer := network.NewBzzTestPeer(peer, r.addr) + r.delivery.overlay.On(bzzPeer) + defer r.delivery.overlay.Off(bzzPeer) + return r.run(peer) +} + +// HandleMsg is the message handler that delegates incoming messages +func (p *Peer) HandleMsg(msg interface{}) error { + switch msg := msg.(type) { + + case *SubscribeMsg: + return p.handleSubscribeMsg(msg) + + case *SubscribeErrorMsg: + return p.handleSubscribeErrorMsg(msg) + + case *UnsubscribeMsg: + return p.handleUnsubscribeMsg(msg) + + case *OfferedHashesMsg: + return p.handleOfferedHashesMsg(msg) + + case *TakeoverProofMsg: + return p.handleTakeoverProofMsg(msg) + + case *WantedHashesMsg: + return p.handleWantedHashesMsg(msg) + + case *ChunkDeliveryMsg: + return p.streamer.delivery.handleChunkDeliveryMsg(p, msg) + + case *RetrieveRequestMsg: + return p.streamer.delivery.handleRetrieveRequestMsg(p, msg) + + default: + return fmt.Errorf("unknown message type: %T", msg) + } +} + +func keyToString(key []byte) string { + l := len(key) + if l == 0 { + return "" + } + return fmt.Sprintf("%s-%d", string(key[:l-1]), key[l-1]) +} + +type server struct { + Server + priority uint8 + currentBatch []byte + stream string + key []byte +} + +// Server interface for outgoing peer Streamer +type Server interface { + SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) + GetData([]byte) ([]byte, error) + Close() +} + +type client struct { + Client + priority uint8 + sessionAt uint64 + live bool + stream string + key []byte + next chan error +} + +// Client interface for incoming peer Streamer +type Client interface { + NeedData([]byte) func() + BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) + Close() +} + +// nextBatch adjusts the indexes by inspecting the intervals +func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { + var intervals []uint64 + if c.live { + if len(intervals) == 0 { + intervals = []uint64{c.sessionAt, from} + } else { + intervals[1] = from + } + nextFrom = from + } else if from >= c.sessionAt { // history sync complete + intervals = nil + nextFrom = from + nextTo = math.MaxUint64 + } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals + intervals = append(intervals[:1], intervals[3:]...) + nextFrom = intervals[1] + if len(intervals) > 2 { + nextTo = intervals[2] + } else { + nextTo = c.sessionAt + } + } else { + nextFrom = from + intervals[1] = from + nextTo = c.sessionAt + } + // b.intervals.set(intervals) + return nextFrom, nextTo +} + +func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error { + if tf := c.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { + tp, err := tf() + if err != nil { + return err + } + return p.SendPriority(tp, c.priority) + } + return nil +} + +func (c *client) close() { + close(c.next) + c.Close() +} + +// Spec is the spec of the streamer protocol +var Spec = &protocols.Spec{ + Name: "stream", + Version: 1, + MaxMsgSize: 10 * 1024 * 1024, + Messages: []interface{}{ + UnsubscribeMsg{}, + OfferedHashesMsg{}, + WantedHashesMsg{}, + TakeoverProofMsg{}, + SubscribeMsg{}, + RetrieveRequestMsg{}, + ChunkDeliveryMsg{}, + SubscribeErrorMsg{}, + }, +} + +func (r *Registry) Protocols() []p2p.Protocol { + return []p2p.Protocol{ + { + Name: Spec.Name, + Version: Spec.Version, + Length: Spec.Length(), + Run: r.runProtocol, + // NodeInfo: , + // PeerInfo: , + }, + } +} + +func (r *Registry) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: "stream", + Version: "0.1", + Service: r.api, + Public: true, + }, + } +} + +func (r *Registry) Start(server *p2p.Server) error { + r.api.dpa.Start() + return nil +} + +func (r *Registry) Stop() error { + r.api.dpa.Stop() + return nil +} + +type API struct { + streamer *Registry + dpa *storage.DPA +} + +func NewAPI(r *Registry, store storage.ChunkStore) *API { + dpa := storage.NewDPA(store, storage.NewChunkerParams()) + return &API{ + streamer: r, + dpa: dpa, + } +} + +func readAll(dpa *storage.DPA, hash []byte) (int64, error) { + r := dpa.Retrieve(hash) + buf := make([]byte, 1024) + var n int + var total int64 + var err error + for (total == 0 || n > 0) && err == nil { + n, err = r.ReadAt(buf, total) + total += int64(n) + } + if err != nil && err != io.EOF { + return total, err + } + return total, nil +} + +func (api *API) ReadAll(hash common.Hash) (int64, error) { + return readAll(api.dpa, hash[:]) +} + +func (api *API) SubscribeStream(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { + return api.streamer.Subscribe(peerId, s, t, from, to, priority, live) +} + +func (api *API) UnsubscribeStream(peerId discover.NodeID, s string, t []byte) error { + return api.streamer.Unsubscribe(peerId, s, t) +} diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go new file mode 100644 index 0000000000..a2aabc7f82 --- /dev/null +++ b/swarm/network/stream/streamer_test.go @@ -0,0 +1,371 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "bytes" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto/sha3" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" +) + +func TestStreamerSubscribe(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + err = streamer.Subscribe(tester.IDs[0], "foo", nil, 0, 0, Top, true) + if err == nil || err.Error() != "stream foo not registered" { + t.Fatalf("Expected error %v, got %v", "stream foo not registered", err) + } +} + +var ( + hash0 = sha3.Sum256([]byte{0}) + hash1 = sha3.Sum256([]byte{1}) + hash2 = sha3.Sum256([]byte{2}) + hashesTmp = append(hash0[:], hash1[:]...) + hashes = append(hashesTmp, hash2[:]...) + receivedHashes map[string][]byte = make(map[string][]byte) + wait0 = make(chan bool) + wait2 = make(chan bool) + batchDone = make(chan bool) +) + +type testClient struct { + t []byte +} + +type testServer struct { + t []byte +} + +func (self *testClient) NeedData(hash []byte) func() { + receivedHashes[string(hash)] = hash + if bytes.Equal(hash, hash0[:]) { + return func() { + <-wait0 + } + } else if bytes.Equal(hash, hash2[:]) { + return func() { + <-wait2 + } + } + return nil +} + +func (self *testClient) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) { + close(batchDone) + return nil +} + +func (self *testClient) Close() {} + +func (self *testServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { + return make([]byte, HashSize), from + 1, to + 1, nil, nil +} + +func (self *testServer) GetData([]byte) ([]byte, error) { + return nil, nil +} + +func (self *testServer) Close() { +} + +func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) { + return &testClient{ + t: t, + }, nil + }) + + peerID := tester.IDs[0] + + err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + err = streamer.Unsubscribe(peerID, "foo", []byte{}) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Unsubscribe message", + Expects: []p2ptest.Expect{ + { + Code: 0, + Msg: &UnsubscribeMsg{ + Stream: "foo", + Key: []byte{}, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterServerFunc("foo", func(p *Peer, t []byte) (Server, error) { + return &testServer{ + t: t, + }, nil + }) + + peerID := tester.IDs[0] + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + Stream: "foo", + Key: []byte{}, + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: make([]byte, HashSize), + From: 6, + To: 9, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "unsubscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &UnsubscribeMsg{ + Stream: "foo", + Key: []byte{}, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterServerFunc("foo", func(p *Peer, t []byte) (Server, error) { + return &testServer{ + t: t, + }, nil + }) + + peerID := tester.IDs[0] + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: "bar", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream bar not registered", + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) { + return &testClient{ + t: t, + }, nil + }) + + peerID := tester.IDs[0] + + err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerID, + }, + }, + }, + p2ptest.Exchange{ + Label: "WantedHashes message", + Triggers: []p2ptest.Trigger{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: hashes, + From: 5, + To: 8, + Stream: "foo", + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 2, + Msg: &WantedHashesMsg{ + Stream: "foo", + Want: []byte{5}, + From: 8, + To: 0, + }, + Peer: peerID, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + if len(receivedHashes) != 3 { + t.Fatalf("Expected number of received hashes %v, got %v", 3, len(receivedHashes)) + } + + close(wait0) + + timeout := time.NewTimer(100 * time.Millisecond) + defer timeout.Stop() + + select { + case <-batchDone: + t.Fatal("batch done early") + case <-timeout.C: + } + + close(wait2) + + timeout2 := time.NewTimer(10000 * time.Millisecond) + defer timeout2.Stop() + + select { + case <-batchDone: + case <-timeout2.C: + t.Fatal("timeout waiting batchdone call") + } + +} diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go new file mode 100644 index 0000000000..605429693d --- /dev/null +++ b/swarm/network/stream/syncer.go @@ -0,0 +1,257 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + // BatchSize = 2 + BatchSize = 128 +) + +// SwarmSyncerServer implements an Server for history syncing on bins +// offered streams: +// * live request delivery with or without checkback +// * (live/non-live historical) chunk syncing per proximity bin +type SwarmSyncerServer struct { + po uint8 + db *storage.DBAPI + sessionAt uint64 + start uint64 + quit chan struct{} +} + +// NewSwarmSyncerServer is contructor for SwarmSyncerServer +func NewSwarmSyncerServer(live bool, po uint8, db *storage.DBAPI) (*SwarmSyncerServer, error) { + sessionAt := db.CurrentBucketStorageIndex(po) + var start uint64 + if live { + start = sessionAt + } + return &SwarmSyncerServer{ + po: po, + db: db, + sessionAt: sessionAt, + start: start, + quit: make(chan struct{}), + }, nil +} + +const maxPO = 32 + +func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) { + streamer.RegisterServerFunc("SYNC", func(p *Peer, t []byte) (Server, error) { + po := t[0] + // TODO: make this work for HISTORY too + return NewSwarmSyncerServer(false, po, db) + }) + // streamer.RegisterServerFunc(stream, func(p *Peer) (Server, error) { + // return NewOutgoingProvableSwarmSyncer(po, db) + // }) +} + +// Close needs to be called on a stream server +func (s *SwarmSyncerServer) Close() { + close(s.quit) +} + +// GetSection retrieves the actual chunk from localstore +func (s *SwarmSyncerServer) GetData(key []byte) ([]byte, error) { + chunk, err := s.db.Get(storage.Key(key)) + if err == storage.ErrFetching { + <-chunk.ReqC + } else if err != nil { + return nil, err + } + return chunk.SData, nil +} + +// GetBatch retrieves the next batch of hashes from the dbstore +func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { + var batch []byte + i := 0 + if from == 0 { + from = s.start + } + if to <= from || from >= s.sessionAt { + to = math.MaxUint64 + } + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + case <-s.quit: + return nil, 0, 0, nil, nil + } + err := s.db.Iterator(from, to, s.po, func(key storage.Key, idx uint64) bool { + batch = append(batch, key[:]...) + i++ + to = idx + return i < BatchSize + }) + if err != nil { + return nil, 0, 0, nil, err + } + if len(batch) > 0 { + break + } + } + + log.Debug("Swarm syncer offer batch", "po", s.po, "len", i, "from", from, "to", to, "current store count", s.db.CurrentBucketStorageIndex(s.po)) + return batch, from, to, nil, nil +} + +// SwarmSyncerClient +type SwarmSyncerClient struct { + sessionAt uint64 + nextC chan struct{} + sessionRoot storage.Key + sessionReader storage.LazySectionReader + retrieveC chan *storage.Chunk + storeC chan *storage.Chunk + db *storage.DBAPI + chunker storage.Chunker + currentRoot storage.Key + requestFunc func(chunk *storage.Chunk) + end, start uint64 +} + +// NewSwarmSyncerClient is a contructor for provable data exchange syncer +func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI, chunker storage.Chunker) (*SwarmSyncerClient, error) { + return &SwarmSyncerClient{ + db: db, + chunker: chunker, + }, nil +} + +// // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer +// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *SwarmSyncerClient { +// retrieveC := make(storage.Chunk, chunksCap) +// RunChunkRequestor(p, retrieveC) +// storeC := make(storage.Chunk, chunksCap) +// RunChunkStorer(store, storeC) +// s := &SwarmSyncerClient{ +// po: po, +// priority: priority, +// sessionAt: sessionAt, +// start: index, +// end: index, +// nextC: make(chan struct{}, 1), +// intervals: intervals, +// sessionRoot: sessionRoot, +// sessionReader: chunker.Join(sessionRoot, retrieveC), +// retrieveC: retrieveC, +// storeC: storeC, +// } +// return s +// } + +// // StartSyncing is called on the Peer to start the syncing process +// // the idea is that it is called only after kademlia is close to healthy +// func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) { +// lastPO := po +// if nn { +// lastPO = maxPO +// } +// +// for i := po; i <= lastPO; i++ { +// s.Subscribe(peerId, "SYNC", newSyncLabel("LIVE", po), 0, 0, High, true) +// s.Subscribe(peerId, "SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false) +// } +// } + +// RegisterSwarmSyncerClient registers the client constructor function for +// to handle incoming sync streams +func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) { + streamer.RegisterClientFunc("SYNC", func(p *Peer, t []byte) (Client, error) { + return NewSwarmSyncerClient(p, db, nil) + }) +} + +// NeedData +func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) { + chunk, _ := s.db.GetOrCreateRequest(key) + // TODO: we may want to request from this peer anyway even if the request exists + if chunk.ReqC == nil { + return nil + } + // create request and wait until the chunk data arrives and is stored + return func() { + chunk.WaitToStore() + } +} + +// BatchDone +func (s *SwarmSyncerClient) BatchDone(streamName string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { + if s.chunker != nil { + return func() (*TakeoverProof, error) { return s.TakeoverProof(streamName, from, hashes, root) } + } + return nil +} + +func (s *SwarmSyncerClient) TakeoverProof(streamName string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { + // for provable syncer currentRoot is non-zero length + if s.chunker != nil { + if from > s.sessionAt { // for live syncing currentRoot is always updated + //expRoot, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC, s.storeC) + expRoot, _, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC) + if err != nil { + return nil, err + } + if !bytes.Equal(root, expRoot) { + return nil, fmt.Errorf("HandoverProof mismatch") + } + s.currentRoot = root + } else { + expHashes := make([]byte, len(hashes)) + _, err := s.sessionReader.ReadAt(expHashes, int64(s.end*HashSize)) + if err != nil && err != io.EOF { + return nil, err + } + if !bytes.Equal(expHashes, hashes) { + return nil, errors.New("invalid proof") + } + } + return nil, nil + } + s.end += uint64(len(hashes)) / HashSize + takeover := &Takeover{ + Stream: streamName, + // Key: s.Key, + Start: s.start, + End: s.end, + Root: root, + } + // serialise and sign + return &TakeoverProof{ + Takeover: takeover, + Sig: nil, + }, nil +} + +func (s *SwarmSyncerClient) Close() {} diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go new file mode 100644 index 0000000000..480bf61eaa --- /dev/null +++ b/swarm/network/stream/syncer_test.go @@ -0,0 +1,221 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "context" + crand "crypto/rand" + "fmt" + "io" + "math" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/network" + streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const dataChunkCount = 500 + +func TestSyncerSimulation(t *testing.T) { + testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1) + testSyncBetweenNodes(t, 4, 1, dataChunkCount, true, 1) + testSyncBetweenNodes(t, 8, 1, dataChunkCount, true, 1) + testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1) +} + +func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) { + defaultSkipCheck = skipCheck + toAddr = func(id discover.NodeID) *network.BzzAddr { + addr := network.NewAddrFromNodeID(id) + addr.OAddr[0] = byte(0) + return addr + } + conf := &streamTesting.RunConfig{ + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + EnableMsgEvents: false, + } + // create context for simulation run + timeout := 30 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + // defer cancel should come before defer simulation teardown + defer cancel() + + // create simulation network with the config + sim, teardown, err := streamTesting.NewSimulation(conf) + defer teardown() + if err != nil { + t.Fatal(err.Error()) + } + + // HACK: these are global variables in the test so that they are available for + // the service constructor function + // TODO: will this work with exec/docker adapter? + // localstore of nodes made available for action and check calls + stores = make(map[discover.NodeID]storage.ChunkStore) + nodeIndex := make(map[discover.NodeID]int) + for i, id := range sim.IDs { + nodeIndex[id] = i + stores[id] = sim.Stores[i] + } + deliveries = make(map[discover.NodeID]*Delivery) + // peerCount function gives the number of peer connections for a nodeID + // this is needed for the service run function to wait until + // each protocol instance runs and the streamer peers are available + peerCount = func(id discover.NodeID) int { + if sim.IDs[0] == id || sim.IDs[nodes-1] == id { + return 1 + } + return 2 + } + waitPeerErrC = make(chan error) + + // here we distribute chunks of a random file into stores 1...nodes + rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + size := chunkCount * chunkSize + _, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + // need to wait cos we then immediately collect the relevant bin content + wait() + defer rrdpa.Stop() + if err != nil { + t.Fatal(err.Error()) + } + + // create DBAPI-s for all nodes + dbs := make([]*storage.DBAPI, nodes) + for i := 0; i < nodes; i++ { + dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore)) + } + + // collect hashes in po 1 bin for each node + hashes := make([][]storage.Key, nodes) + totalHashes := 0 + hashCounts := make([]int, nodes) + for i := nodes - 1; i >= 0; i-- { + if i < nodes-1 { + hashCounts[i] = hashCounts[i+1] + } + dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { + hashes[i] = append(hashes[i], key) + totalHashes++ + hashCounts[i]++ + return true + }) + } + + // errc is error channel for simulation + errc := make(chan error, 1) + quitC := make(chan struct{}) + defer close(quitC) + + // action is subscribe + action := func(ctx context.Context) error { + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // the global peerCount function tells how many connections each node has + // TODO: this is to be reimplemented with peerEvent watcher without global var + i := 0 + for err := range waitPeerErrC { + if err != nil { + return fmt.Errorf("error waiting for peers: %s", err) + } + i++ + if i == nodes { + break + } + } + // each node Subscribes to each other's swarmChunkServerStreamName + for j := 0; j < nodes-1; j++ { + id := sim.IDs[j] + err := sim.CallClient(id, func(client *rpc.Client) error { + // report disconnect events to the error channel cos peers should not disconnect + err := streamTesting.WatchDisconnections(id, client, errc, quitC) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + // start syncing, i.e., subscribe to upstream peers po 1 bin + sid := sim.IDs[j+1] + return client.CallContext(ctx, nil, "stream_subscribeStream", sid, "SYNC", []byte{1}, 0, 0, Top, false) + }) + if err != nil { + return err + } + } + return nil + } + + // this makes sure check is not called before the previous call finishes + check := func(ctx context.Context, id discover.NodeID) (bool, error) { + select { + case err := <-errc: + return false, err + case <-ctx.Done(): + return false, ctx.Err() + default: + } + + i := nodeIndex[id] + var total, found int + for j := i; j < nodes; j++ { + total += len(hashes[j]) + for _, key := range hashes[j] { + chunk, err := dbs[i].Get(key) + if err == storage.ErrFetching { + <-chunk.ReqC + } else if err != nil { + continue + } + // needed for leveldb not to be closed? + // chunk.WaitToStore() + found++ + } + } + log.Debug("sync check", "node", id, "index", i, "bin", po, "found", found, "total", total) + return total == found, nil + } + + conf.Step = &simulations.Step{ + Action: action, + Trigger: streamTesting.Trigger(500*time.Millisecond, quitC, sim.IDs[0:nodes-1]...), + Expect: &simulations.Expectation{ + Nodes: sim.IDs[0:1], + Check: check, + }, + } + startedAt := time.Now() + result, err := sim.Run(ctx, conf) + finishedAt := time.Now() + if err != nil { + t.Fatalf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + t.Fatalf("Simulation failed: %s", result.Error) + } + streamTesting.CheckResult(t, result, startedAt, finishedAt) +} diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go new file mode 100644 index 0000000000..39b2c1df1d --- /dev/null +++ b/swarm/network/stream/testing/testing.go @@ -0,0 +1,267 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testing + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +type Simulation struct { + Net *simulations.Network + Stores []storage.ChunkStore + Addrs []network.Addr + IDs []discover.NodeID +} + +func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) { + var datadirs []string + stores := make([]storage.ChunkStore, len(addrs)) + var err error + for i, addr := range addrs { + var datadir string + datadir, err = ioutil.TempDir("", "streamer") + if err != nil { + break + } + var store storage.ChunkStore + store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + if err != nil { + break + } + datadirs = append(datadirs, datadir) + stores[i] = store + } + teardown := func() { + for i, datadir := range datadirs { + stores[i].Close() + os.RemoveAll(datadir) + } + } + return stores, teardown, err +} + +func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) { + teardown = func() {} + switch adapterType { + case "sim": + adapter = adapters.NewSimAdapter(services) + case "socket": + adapter = adapters.NewSocketAdapter(services) + case "exec": + baseDir, err0 := ioutil.TempDir("", "swarm-test") + if err0 != nil { + return nil, teardown, err0 + } + teardown = func() { os.RemoveAll(baseDir) } + adapter = adapters.NewExecAdapter(baseDir) + case "docker": + adapter, err = adapters.NewDockerAdapter() + if err != nil { + return nil, teardown, err + } + default: + return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker") + } + return adapter, teardown, nil +} + +func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finishedAt time.Time) { + t.Logf("Simulation passed in %s", result.FinishedAt.Sub(result.StartedAt)) + if len(result.Passes) > 1 { + var min, max time.Duration + var sum int + for _, pass := range result.Passes { + duration := pass.Sub(result.StartedAt) + if sum == 0 || duration < min { + min = duration + } + if duration > max { + max = duration + } + sum += int(duration.Nanoseconds()) + } + t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) + } + t.Logf("Setup: %s, Shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) +} + +type RunConfig struct { + Adapter string + Step *simulations.Step + NodeCount int + ConnLevel int + ToAddr func(discover.NodeID) *network.BzzAddr + Services adapters.Services + EnableMsgEvents bool +} + +func NewSimulation(conf *RunConfig) (*Simulation, func(), error) { + // create network + nodes := conf.NodeCount + adapter, adapterTeardown, err := NewAdapter(conf.Adapter, conf.Services) + if err != nil { + return nil, adapterTeardown, err + } + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ + ID: "0", + DefaultService: "streamer", + }) + teardown := func() { + adapterTeardown() + net.Shutdown() + } + ids := make([]discover.NodeID, nodes) + addrs := make([]network.Addr, nodes) + // start nodes + for i := 0; i < nodes; i++ { + nodeconf := adapters.RandomNodeConfig() + nodeconf.EnableMsgEvents = conf.EnableMsgEvents + node, err := net.NewNodeWithConfig(nodeconf) + if err != nil { + return nil, teardown, fmt.Errorf("error creating node: %s", err) + } + ids[i] = node.ID() + addrs[i] = conf.ToAddr(ids[i]) + } + // set nodes number of Stores available + stores, storeTeardown, err := SetStores(addrs...) + teardown = func() { + net.Shutdown() + adapterTeardown() + storeTeardown() + } + if err != nil { + return nil, teardown, err + } + s := &Simulation{ + Net: net, + Stores: stores, + IDs: ids, + Addrs: addrs, + } + return s, teardown, nil +} + +func (s *Simulation) Run(ctx context.Context, conf *RunConfig) (*simulations.StepResult, error) { + // bring up nodes, launch the servive + nodes := conf.NodeCount + conns := conf.ConnLevel + for i := 0; i < nodes; i++ { + if err := s.Net.Start(s.IDs[i]); err != nil { + return nil, fmt.Errorf("error starting node %s: %s", s.IDs[i].TerminalString(), err) + } + } + // run a simulation which connects the 10 nodes in a chain + wg := sync.WaitGroup{} + for i := range s.IDs { + // collect the overlay addresses, to + for j := 0; j < conns; j++ { + var k int + if j == 0 { + k = i - 1 + } else { + k = rand.Intn(len(s.IDs)) + } + if i > 0 { + wg.Add(1) + go func(i, k int) { + defer wg.Done() + s.Net.Connect(s.IDs[i], s.IDs[k]) + }(i, k) + } + } + } + wg.Wait() + log.Info(fmt.Sprintf("simulation with %v nodes", len(s.Addrs))) + + // create an only locally retrieving dpa for the pivot node to test + // if retriee requests have arrived + result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step) + return result, nil +} + +func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) error { + events := make(chan *p2p.PeerEvent) + sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents") + if err != nil { + return fmt.Errorf("error getting peer events for node %v: %s", id, err) + } + go func() { + for { + select { + case <-quitC: + return + case e := <-events: + errc <- fmt.Errorf("peerEvent for node %v: %v", id, e) + case err := <-sub.Err(): + if err != nil { + errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err) + } + } + } + }() + return nil +} + +func Trigger(d time.Duration, quitC chan struct{}, ids ...discover.NodeID) chan discover.NodeID { + trigger := make(chan discover.NodeID) + go func() { + defer close(trigger) + ticker := time.NewTicker(d) + defer ticker.Stop() + // we are only testing the pivot node (net.Nodes[0]) + for range ticker.C { + for _, id := range ids { + select { + case trigger <- id: + case <-quitC: + return + } + } + } + }() + return trigger +} + +func (sim *Simulation) CallClient(id discover.NodeID, f func(*rpc.Client) error) error { + node := sim.Net.GetNode(id) + if node == nil { + return fmt.Errorf("unknown node: %s", id) + } + client, err := node.Client() + if err != nil { + return fmt.Errorf("error getting node client: %s", err) + } + return f(client) +} diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index a6a909b136..ae6b423382 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -180,9 +180,9 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { DefaultService: "bzz", }) for i := 0; i < numnodes; i++ { - nodes[i], err = net.NewNodeWithConfig(&adapters.NodeConfig{ - Services: []string{"bzz", "pss"}, - }) + nodeconf := adapters.RandomNodeConfig() + nodeconf.Services = []string{"bzz", "pss"} + nodes[i], err = net.NewNodeWithConfig(nodeconf) if err != nil { return nil, fmt.Errorf("error creating node 1: %v", err) } @@ -232,11 +232,11 @@ func newServices() adapters.Services { "pss": func(ctx *adapters.ServiceContext) (node.Service, error) { cachedir, err := ioutil.TempDir("", "pss-cache") if err != nil { - return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err) + return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err) } - dpa, err := storage.NewLocalDPA(cachedir) + dpa, err := storage.NewLocalDPA(cachedir, make([]byte, 32)) if err != nil { - return nil, fmt.Errorf("local dpa creation failed: %v", err) + return nil, fmt.Errorf("local dpa creation failed: %s", err) } ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index b5474ded82..d2d02898a6 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -498,7 +498,7 @@ func (self *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessag func (self *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { recvmsg, err := envelope.OpenAsymmetric(self.privateKey) if err != nil { - return nil, "", nil, fmt.Errorf("could not decrypt message: %v", err) + return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err) } // check signature (if signed), strip padding if !recvmsg.Validate() { @@ -774,10 +774,8 @@ func (self *Pss) checkFwdCache(addr []byte, digest pssDigest) bool { // DPA storage handler for message cache func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) { - swg := &sync.WaitGroup{} - wwg := &sync.WaitGroup{} buf := bytes.NewReader(msg.serialize()) - key, err := self.dpa.Store(buf, int64(buf.Len()), swg, wwg) + key, _, err := self.dpa.Store(buf, int64(buf.Len())) if err != nil { log.Warn("Could not store in swarm", "err", err) return pssDigest{}, err diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index b9275f84c0..242f653463 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "crypto/ecdsa" + "encoding/binary" "encoding/hex" + "encoding/json" "flag" "fmt" "io/ioutil" @@ -17,6 +19,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -260,8 +263,8 @@ func TestKeys(t *testing.T) { if err != nil { t.Fatalf("create 'our' key fail") } - ctx, cancel = context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() theirkeys, err := wapi.NewKeyPair(ctx) if err != nil { t.Fatalf("create 'their' key fail") @@ -392,6 +395,679 @@ func TestMismatch(t *testing.T) { } +// send symmetrically encrypted message between two directly connected peers +func TestSymSend(t *testing.T) { + t.Run("32", testSymSend) + t.Run("8", testSymSend) + t.Run("0", testSymSend) +} + +func testSymSend(t *testing.T) { + + // address hint size + var addrsize int64 + var err error + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("sym send test", "addrsize", addrsize) + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + + var topic string + err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + + // retrieve public key from pss instance + // set this public key reciprocally + var lpubkeyhex string + err = clients[0].Call(&lpubkeyhex, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkeyhex string + err = clients[1].Call(&rpubkeyhex, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 500) + + // at this point we've verified that symkeys are saved and match on each peer + // now try sending symmetrically encrypted message, both directions + lmsgC := make(chan APIMsg) + lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) + defer lcancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + log.Trace("lsub", "id", lsub) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) + defer rcancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + log.Trace("rsub", "id", rsub) + defer rsub.Unsubscribe() + + lrecvkey := network.RandomAddr().Over() + rrecvkey := network.RandomAddr().Over() + + var lkeyids [2]string + var rkeyids [2]string + + // manually set reciprocal symkeys + err = clients[0].Call(&lkeyids, "psstest_setSymKeys", rpubkeyhex, lrecvkey, rrecvkey, defaultSymKeySendLimit, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(&rkeyids, "psstest_setSymKeys", rpubkeyhex, rrecvkey, lrecvkey, defaultSymKeySendLimit, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // send and verify delivery + lmsg := []byte("plugh") + err = clients[1].Call(nil, "pss_sendSym", rkeyids[1], topic, hexutil.Encode(lmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-lmsgC: + if !bytes.Equal(recvmsg.Msg, lmsg) { + t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg) + } + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + rmsg := []byte("xyzzy") + err = clients[0].Call(nil, "pss_sendSym", lkeyids[1], topic, hexutil.Encode(rmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-rmsgC: + if !bytes.Equal(recvmsg.Msg, rmsg) { + t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) + } + case cerr := <-rctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } +} + +// send asymmetrically encrypted message between two directly connected peers +func TestAsymSend(t *testing.T) { + t.Run("32", testAsymSend) + t.Run("8", testAsymSend) + t.Run("0", testAsymSend) +} + +func testAsymSend(t *testing.T) { + + // address hint size + var addrsize int64 + var err error + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("asym send test", "addrsize", addrsize) + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + + var topic string + err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Millisecond * 250) + + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + + // retrieve public key from pss instance + // set this public key reciprocally + var lpubkey string + err = clients[0].Call(&lpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkey string + err = clients[1].Call(&rpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 500) // replace with hive healthy code + + lmsgC := make(chan APIMsg) + lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) + defer lcancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + log.Trace("lsub", "id", lsub) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) + defer rcancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + log.Trace("rsub", "id", rsub) + defer rsub.Unsubscribe() + + // store reciprocal public keys + err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // send and verify delivery + rmsg := []byte("xyzzy") + err = clients[0].Call(nil, "pss_sendAsym", rpubkey, topic, hexutil.Encode(rmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-rmsgC: + if !bytes.Equal(recvmsg.Msg, rmsg) { + t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) + } + case cerr := <-rctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + lmsg := []byte("plugh") + err = clients[1].Call(nil, "pss_sendAsym", lpubkey, topic, hexutil.Encode(lmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-lmsgC: + if !bytes.Equal(recvmsg.Msg, lmsg) { + t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg.Msg) + } + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } +} + +type Job struct { + Msg []byte + SendNode discover.NodeID + RecvNode discover.NodeID +} + +func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubkeys map[discover.NodeID]string, topic string) { + for j := range jobs { + rpcs[j.SendNode].Call(nil, "pss_sendAsym", pubkeys[j.RecvNode], topic, hexutil.Encode(j.Msg)) + } +} + +// params in run name: +// nodes/msgs/addrbytes/adaptertype +// if adaptertype is exec uses execadapter, simadapter otherwise +func XTestNetwork(t *testing.T) { + t.Run("3/2000/4/sock", testNetwork) + t.Run("4/2000/4/sock", testNetwork) + t.Run("8/2000/4/sock", testNetwork) + t.Run("16/2000/4/sock", testNetwork) + t.Run("32/2000/4/sock", testNetwork) + t.Run("64/2000/4/sim", testNetwork) +} + +func testNetwork(t *testing.T) { + type msgnotifyC struct { + id discover.NodeID + msgIdx int + } + + paramstring := strings.Split(t.Name(), "/") + nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0) + msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0) + addrsize, _ := strconv.ParseInt(paramstring[3], 10, 0) + adapter := paramstring[4] + + log.Info("network test", "nodecount", nodecount, "msgcount", msgcount, "addrhintsize", addrsize) + + nodes := make([]discover.NodeID, nodecount) + bzzaddrs := make(map[discover.NodeID]string, nodecount) + rpcs := make(map[discover.NodeID]*rpc.Client, nodecount) + pubkeys := make(map[discover.NodeID]string, nodecount) + + sentmsgs := make([][]byte, msgcount) + recvmsgs := make([]bool, msgcount) + nodemsgcount := make(map[discover.NodeID]int, nodecount) + + trigger := make(chan discover.NodeID) + + var a adapters.NodeAdapter + if adapter == "exec" { + dirname, err := ioutil.TempDir(".", "") + if err != nil { + t.Fatal(err) + } + a = adapters.NewExecAdapter(dirname) + } else if adapter == "sock" { + a = adapters.NewSocketAdapter(services) + } else if adapter == "tcp" { + a = adapters.NewTCPAdapter(services) + } else if adapter == "sim" { + a = adapters.NewSimAdapter(services) + } + net := simulations.NewNetwork(a, &simulations.NetworkConfig{ + ID: "0", + }) + defer net.Shutdown() + + f, err := os.Open(fmt.Sprintf("testdata/snapshot_%d.json", nodecount)) + if err != nil { + t.Fatal(err) + } + jsonbyte, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(jsonbyte, &snap) + if err != nil { + t.Fatal(err) + } + err = net.Load(&snap) + if err != nil { + t.Fatal(err) + } + + triggerChecks := func(trigger chan discover.NodeID, id discover.NodeID, rpcclient *rpc.Client, topic string) error { + msgC := make(chan APIMsg) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic) + if err != nil { + t.Fatal(err) + } + go func() { + defer sub.Unsubscribe() + for { + select { + case recvmsg := <-msgC: + idx, _ := binary.Uvarint(recvmsg.Msg) + if !recvmsgs[idx] { + log.Debug("msg recv", "idx", idx, "id", id) + recvmsgs[idx] = true + trigger <- id + } + case <-sub.Err(): + return + } + } + }() + return nil + } + + var topic string + for i, nod := range net.GetNodes() { + nodes[i] = nod.ID() + rpcs[nodes[i]], err = nod.Client() + if err != nil { + t.Fatal(err) + } + if topic == "" { + err = rpcs[nodes[i]].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + } + var pubkey string + err = rpcs[nodes[i]].Call(&pubkey, "pss_getPublicKey") + if err != nil { + t.Fatal(err) + } + pubkeys[nod.ID()] = pubkey + var addrhex string + err = rpcs[nodes[i]].Call(&addrhex, "pss_baseAddr") + if err != nil { + t.Fatal(err) + } + bzzaddrs[nodes[i]] = addrhex + err = triggerChecks(trigger, nodes[i], rpcs[nodes[i]], topic) + if err != nil { + t.Fatal(err) + } + } + + // setup workers + jobs := make(chan Job, 10) + for w := 1; w <= 10; w++ { + go worker(w, jobs, rpcs, pubkeys, topic) + } + + for i := 0; i < int(msgcount); i++ { + sendnodeidx := rand.Intn(int(nodecount)) + recvnodeidx := rand.Intn(int(nodecount - 1)) + if recvnodeidx >= sendnodeidx { + recvnodeidx++ + } + nodemsgcount[nodes[recvnodeidx]]++ + sentmsgs[i] = make([]byte, 8) + c := binary.PutUvarint(sentmsgs[i], uint64(i)) + if c == 0 { + t.Fatal("0 byte message") + } + if err != nil { + t.Fatal(err) + } + err = rpcs[nodes[sendnodeidx]].Call(nil, "pss_setPeerPublicKey", pubkeys[nodes[recvnodeidx]], topic, bzzaddrs[nodes[recvnodeidx]]) + if err != nil { + t.Fatal(err) + } + + jobs <- Job{ + Msg: sentmsgs[i], + SendNode: nodes[sendnodeidx], + RecvNode: nodes[recvnodeidx], + } + } + + finalmsgcount := 0 + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() +outer: + for i := 0; i < int(msgcount); i++ { + select { + case id := <-trigger: + nodemsgcount[id]-- + finalmsgcount++ + case <-ctx.Done(): + log.Warn("timeout") + break outer + } + } + + for i, msg := range recvmsgs { + if !msg { + log.Debug("missing message", "idx", i) + } + } + t.Logf("%d of %d messages received", finalmsgcount, msgcount) + + if finalmsgcount != int(msgcount) { + t.Fatalf("%d messages were not received", int(msgcount)-finalmsgcount) + } + +} + +// symmetric send performance with varying message sizes +func BenchmarkSymkeySend(b *testing.B) { + b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkSymKeySend) +} + +func benchmarkSymKeySend(b *testing.B) { + msgsizestring := strings.Split(b.Name(), "/") + if len(msgsizestring) != 2 { + b.Fatalf("benchmark called without msgsize param") + } + msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + ps := newTestPss(privkey, nil, nil) + msg := make([]byte, msgsize) + rand.Read(msg) + topic := BytesToTopic([]byte("foo")) + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + symkeyid, err := ps.generateSymmetricKey(topic, &to, true) + if err != nil { + b.Fatalf("could not generate symkey: %v", err) + } + symkey, err := ps.w.GetSymKey(symkeyid) + if err != nil { + b.Fatalf("could not retrieve symkey: %v", err) + } + ps.SetSymmetricKey(symkey, topic, &to, false) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps.SendSym(symkeyid, topic, msg) + } +} + +// asymmetric send performance with varying message sizes +func BenchmarkAsymkeySend(b *testing.B) { + b.Run(fmt.Sprintf("%d", 256), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkAsymKeySend) +} + +func benchmarkAsymKeySend(b *testing.B) { + msgsizestring := strings.Split(b.Name(), "/") + if len(msgsizestring) != 2 { + b.Fatalf("benchmark called without msgsize param") + } + msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + ps := newTestPss(privkey, nil, nil) + msg := make([]byte, msgsize) + rand.Read(msg) + topic := BytesToTopic([]byte("foo")) + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to) + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) + } +} +func BenchmarkSymkeyBruteforceChangeaddr(b *testing.B) { + for i := 100; i < 100000; i = i * 10 { + for j := 32; j < 10000; j = j * 8 { + b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceChangeaddr) + } + //b.Run(fmt.Sprintf("%d", i), benchmarkSymkeyBruteforceChangeaddr) + } +} + +// decrypt performance using symkey cache, worst case +// (decrypt key always last in cache) +func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { + keycountstring := strings.Split(b.Name(), "/") + cachesize := int64(0) + var ps *Pss + if len(keycountstring) < 2 { + b.Fatalf("benchmark called without count param") + } + keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) + } + if len(keycountstring) == 3 { + cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) + } + } + pssmsgs := make([]*PssMsg, 0, keycount) + var keyid string + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + if cachesize > 0 { + ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) + } else { + ps = newTestPss(privkey, nil, nil) + } + topic := BytesToTopic([]byte("foo")) + for i := 0; i < int(keycount); i++ { + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + keyid, err = ps.generateSymmetricKey(topic, &to, true) + if err != nil { + b.Fatalf("cant generate symkey #%d: %v", i, err) + } + symkey, err := ps.w.GetSymKey(keyid) + if err != nil { + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) + } + wparams := &whisper.MessageParams{ + TTL: defaultWhisperTTL, + KeySym: symkey, + Topic: whisper.TopicType(topic), + WorkTime: defaultWhisperWorkTime, + PoW: defaultWhisperPoW, + Payload: []byte("xyzzy"), + Padding: []byte("1234567890abcdef"), + } + woutmsg, err := whisper.NewSentMessage(wparams) + if err != nil { + b.Fatalf("could not create whisper message: %v", err) + } + env, err := woutmsg.Wrap(wparams) + if err != nil { + b.Fatalf("could not generate whisper envelope: %v", err) + } + ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + }) + pssmsgs = append(pssmsgs, &PssMsg{ + To: to, + Payload: env, + }) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]) { + b.Fatalf("pss processing failed: %v", err) + } + } +} + +func BenchmarkSymkeyBruteforceSameaddr(b *testing.B) { + for i := 100; i < 100000; i = i * 10 { + for j := 32; j < 10000; j = j * 8 { + b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceSameaddr) + } + } +} + +// decrypt performance using symkey cache, best case +// (decrypt key always first in cache) +func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { + var keyid string + var ps *Pss + cachesize := int64(0) + keycountstring := strings.Split(b.Name(), "/") + if len(keycountstring) < 2 { + b.Fatalf("benchmark called without count param") + } + keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) + } + if len(keycountstring) == 3 { + cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) + } + } + addr := make([]PssAddress, keycount) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + if cachesize > 0 { + ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) + } else { + ps = newTestPss(privkey, nil, nil) + } + topic := BytesToTopic([]byte("foo")) + for i := 0; i < int(keycount); i++ { + copy(addr[i], network.RandomAddr().Over()) + keyid, err = ps.generateSymmetricKey(topic, &addr[i], true) + if err != nil { + b.Fatalf("cant generate symkey #%d: %v", i, err) + } + + } + symkey, err := ps.w.GetSymKey(keyid) + if err != nil { + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) + } + wparams := &whisper.MessageParams{ + TTL: defaultWhisperTTL, + KeySym: symkey, + Topic: whisper.TopicType(topic), + WorkTime: defaultWhisperWorkTime, + PoW: defaultWhisperPoW, + Payload: []byte("xyzzy"), + Padding: []byte("1234567890abcdef"), + } + woutmsg, err := whisper.NewSentMessage(wparams) + if err != nil { + b.Fatalf("could not create whisper message: %v", err) + } + env, err := woutmsg.Wrap(wparams) + if err != nil { + b.Fatalf("could not generate whisper envelope: %v", err) + } + ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + }) + pssmsg := &PssMsg{ + To: addr[len(addr)-1][:], + Payload: env, + } + for i := 0; i < b.N; i++ { + if !ps.process(pssmsg) { + b.Fatalf("pss processing failed: %v", err) + } + } +} + // setup simulated network and connect nodes in circle func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { nodes := make([]*simulations.Node, numnodes) @@ -405,9 +1081,9 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { DefaultService: "bzz", }) for i := 0; i < numnodes; i++ { - nodes[i], err = net.NewNodeWithConfig(&adapters.NodeConfig{ - Services: []string{"bzz", pssProtocolName}, - }) + nodeconf := adapters.RandomNodeConfig() + nodeconf.Services = []string{"bzz", pssProtocolName} + nodes[i], err = net.NewNodeWithConfig(nodeconf) if err != nil { return nil, fmt.Errorf("error creating node 1: %v", err) } @@ -457,11 +1133,11 @@ func newServices() adapters.Services { pssProtocolName: func(ctx *adapters.ServiceContext) (node.Service, error) { cachedir, err := ioutil.TempDir("", "pss-cache") if err != nil { - return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err) + return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err) } - dpa, err := storage.NewLocalDPA(cachedir) + dpa, err := storage.NewLocalDPA(cachedir, network.NewAddrFromNodeID(ctx.Config.ID).Over()) if err != nil { - return nil, fmt.Errorf("local dpa creation failed: %v", err) + return nil, fmt.Errorf("local dpa creation failed: %s", err) } // execadapter does not exec init() @@ -532,7 +1208,7 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss log.Error("create pss cache tmpdir failed", "error", err) os.Exit(1) } - dpa, err := storage.NewLocalDPA(cachedir) + dpa, err := storage.NewLocalDPA(cachedir, addr.Over()) if err != nil { log.Error("local dpa creation failed", "error", err) os.Exit(1) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 98cd6e75ea..f643fb502d 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -13,7 +13,6 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . - package storage import ( @@ -118,23 +117,19 @@ func (self *TreeChunker) decrementWorkerCount() { self.workerCount -= 1 } -func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { +func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) (k Key, wait func(), err error) { if self.chunkSize <= 0 { panic("chunker must be initialised") } jobC := make(chan *hashJob, 2*ChunkProcessors) wg := &sync.WaitGroup{} + storeWg := &sync.WaitGroup{} errC := make(chan error) quitC := make(chan bool) - // wwg = workers waitgroup keeps track of hashworkers spawned by this split call - if wwg != nil { - wwg.Add(1) - } - self.incrementWorkerCount() - go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg) + self.runHashWorker(jobC, chunkC, errC, quitC, storeWg) depth := 0 treeSize := self.chunkSize @@ -149,16 +144,12 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s // this waitgroup member is released after the root hash is calculated wg.Add(1) //launch actual recursive function passing the waitgroups - go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, swg, wwg) + go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, storeWg) // closes internal error channel if all subprocesses in the workgroup finished go func() { // waiting for all threads to finish wg.Wait() - // if storage waitgroup is non-nil, we wait for storage to finish too - if swg != nil { - swg.Wait() - } close(errC) }() @@ -166,16 +157,16 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s select { case err := <-errC: if err != nil { - return nil, err + return nil, nil, err } case <-time.NewTimer(splitTimeout).C: - return nil, errOperationTimedOut + return nil, nil, errOperationTimedOut } - return key, nil + return key, storeWg.Wait, nil } -func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, swg, wwg *sync.WaitGroup) { +func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, storeWg *sync.WaitGroup) { // @@ -225,7 +216,7 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize] childrenWg.Add(1) - self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, swg, wwg) + self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, storeWg) i++ pos += treeSize @@ -237,11 +228,8 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade worker := self.getWorkerCount() if int64(len(jobC)) > worker && worker < ChunkProcessors { - if wwg != nil { - wwg.Add(1) - } self.incrementWorkerCount() - go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg) + self.runHashWorker(jobC, chunkC, errC, quitC, storeWg) } select { @@ -250,60 +238,58 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade } } -func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) { - defer self.decrementWorkerCount() +func (self *TreeChunker) runHashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storeWg *sync.WaitGroup) { + storeWg.Add(1) - hasher := self.hashFunc() - if wwg != nil { - defer wwg.Done() - } - for { - select { + go func() { + defer self.decrementWorkerCount() + defer storeWg.Done() + hasher := self.hashFunc() + for { + select { - case job, ok := <-jobC: - if !ok { + case job, ok := <-jobC: + if !ok { + return + } + // now we got the hashes in the chunk, then hash the chunks + self.hashChunk(hasher, job, chunkC, storeWg) + case <-quitC: return } - // now we got the hashes in the chunk, then hash the chunks - self.hashChunk(hasher, job, chunkC, swg) - case <-quitC: - return } - } + }() } // The treeChunkers own Hash hashes together // - the size (of the subtree encoded in the Chunk) // - the Chunk, ie. the contents read from the input reader -func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) { +func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, storeWg *sync.WaitGroup) { hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length hasher.Write(job.chunk[8:]) // minus 8 []byte length h := hasher.Sum(nil) - newChunk := &Chunk{ - Key: h, - SData: job.chunk, - Size: job.size, - wg: swg, - } + newChunk := NewChunk(h, nil) + newChunk.SData = job.chunk + newChunk.Size = job.size // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) copy(job.key, h) // send off new chunk to storage - if chunkC != nil { - if swg != nil { - swg.Add(1) - } - } job.parentWg.Done() if chunkC != nil { chunkC <- newChunk + storeWg.Add(1) + go func() { + defer storeWg.Done() + <-newChunk.dbStored + }() } } -func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { - return nil, errAppendOppNotSuported +func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (Key, func(), error) { + return nil, nil, errAppendOppNotSuported } // LazyChunkReader implements LazySectionReader @@ -315,16 +301,18 @@ type LazyChunkReader struct { chunkSize int64 // inherit from chunker branches int64 // inherit from chunker hashSize int64 // inherit from chunker + depth int } // implements the Joiner interface -func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader { +func (self *TreeChunker) Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader { return &LazyChunkReader{ key: key, chunkC: chunkC, chunkSize: self.chunkSize, branches: self.branches, hashSize: self.hashSize, + depth: depth, } } @@ -371,8 +359,13 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { depth++ } wg := sync.WaitGroup{} + length := int64(len(b)) + for d := 0; d < self.depth; d++ { + off *= self.chunkSize + length *= self.chunkSize + } wg.Add(1) - go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC) + go self.join(b, off, off+length, depth, treeSize/self.branches, self.chunk, &wg, errC, quitC) go func() { wg.Wait() close(errC) @@ -385,25 +378,21 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { return 0, err } if off+int64(len(b)) >= size { - return len(b), io.EOF + return int(size - off), io.EOF } return len(b), nil } func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { defer parentWg.Done() - // return NewDPA(&LocalStore{}) - - // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - // find appropriate block level - for chunk.Size < treeSize && depth > 0 { + for chunk.Size < treeSize && depth > self.depth { treeSize /= self.branches depth-- } // leaf chunk found - if depth == 0 { + if depth == self.depth { extra := 8 + eoff - int64(len(chunk.SData)) if extra > 0 { eoff -= extra @@ -456,10 +445,8 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr // block until they time out or arrive // abort if quitC is readable func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk { - chunk := &Chunk{ - Key: key, - C: make(chan bool), // close channel to signal data delivery - } + chunk := NewChunk(key, nil) + chunk.C = make(chan bool) // submit chunk for retrieval select { case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally) @@ -475,8 +462,7 @@ func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk { case <-chunk.C: // bells are ringing, data have been delivered } if len(chunk.SData) == 0 { - return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - + return nil } return chunk } diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 19712a4709..e699d1740e 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "io" - "sync" "testing" "time" @@ -45,7 +44,7 @@ type chunkerTester struct { t test } -func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { +func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, expectedError error) (key Key, wait func(), err error) { // reset self.chunks = make(map[string]*Chunk) @@ -65,31 +64,31 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c return nil case chunk := <-chunkC: // self.chunks = append(self.chunks, chunk) - self.chunks[chunk.Key.String()] = chunk - if chunk.wg != nil { - chunk.wg.Done() - } + self.chunks[chunk.Key.Hex()] = chunk + close(chunk.dbStored) } } }() } - key, err = chunker.Split(data, size, chunkC, swg, nil) + var w func() + key, w, err = chunker.Split(data, size, chunkC) if err != nil && expectedError == nil { err = fmt.Errorf("Split error: %v", err) } - if chunkC != nil { - if swg != nil { - swg.Wait() + wait = func() { + w() + close(quitC) } - close(quitC) + } else { + wait = func() {} } - return key, err + return key, wait, err } -func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { +func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, expectedError error) (key Key, wait func(), err error) { quitC := make(chan bool) timeout := time.After(60 * time.Second) if chunkC != nil { @@ -102,17 +101,16 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, return nil case chunk := <-chunkC: if chunk != nil { - stored, success := self.chunks[chunk.Key.String()] + stored, success := self.chunks[chunk.Key.Hex()] if !success { // Requesting data - self.chunks[chunk.Key.String()] = chunk - if chunk.wg != nil { - chunk.wg.Done() - } + self.chunks[chunk.Key.Hex()] = chunk + close(chunk.dbStored) } else { // getting data chunk.SData = stored.SData chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + close(chunk.dbStored) if chunk.C != nil { close(chunk.C) } @@ -122,26 +120,26 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, } }() } - - key, err = chunker.Append(rootKey, data, chunkC, swg, nil) + var w func() + key, w, err = chunker.Append(rootKey, data, chunkC) if err != nil && expectedError == nil { err = fmt.Errorf("Append error: %v", err) } if chunkC != nil { - if swg != nil { - swg.Wait() + wait = func() { + w() + close(quitC) } - close(quitC) + } else { + wait = func() {} } - return key, err + return key, wait, err } func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { // reset but not the chunks - reader := chunker.Join(key, chunkC) - timeout := time.After(600 * time.Second) i := 0 go func() error { @@ -155,7 +153,7 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch return nil } // this just mocks the behaviour of a chunk store retrieval - stored, success := self.chunks[chunk.Key.String()] + stored, success := self.chunks[chunk.Key.Hex()] if !success { return errors.New("Not found") } @@ -166,6 +164,8 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch } } }() + + reader := chunker.Join(key, chunkC, 0) return reader } @@ -183,10 +183,9 @@ func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) { brokendata = brokenLimitReader(data, n, n/2) chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} expectedError := fmt.Errorf("Broken reader") - key, err := tester.Split(splitter, brokendata, int64(n), chunkC, swg, expectedError) + key, _, err := tester.Split(splitter, brokendata, int64(n), chunkC, expectedError) if err == nil || err.Error() != expectedError.Error() { tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) } @@ -207,14 +206,13 @@ func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key { } chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil) + key, wait, err := tester.Split(splitter, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } tester.t.Logf(" Key = %v\n", key) - + wait() chunkC = make(chan *Chunk, 1000) quitC := make(chan bool) @@ -250,12 +248,12 @@ func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) { } chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil) + key, wait, err := tester.Split(splitter, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } + wait() tester.t.Logf(" Key = %v\n", key) //create a append data stream @@ -269,12 +267,12 @@ func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) { } chunkC = make(chan *Chunk, 1000) - swg = &sync.WaitGroup{} - newKey, err := tester.Append(splitter, key, appendData, chunkC, swg, nil) + newKey, wait, err := tester.Append(splitter, key, appendData, chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } + wait() tester.t.Logf(" NewKey = %v\n", newKey) chunkC = make(chan *Chunk, 1000) @@ -325,17 +323,19 @@ func TestSha3ForCorrectness(t *testing.T) { } -func TestDataAppend(t *testing.T) { - sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} - appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} - - tester := &chunkerTester{t: t} - chunker := NewPyramidChunker(NewChunkerParams()) - for i, s := range sizes { - testRandomDataAppend(chunker, s, appendSizes[i], tester) - - } -} +// func TestDataAppend(t *testing.T) { +// // sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} +// sizes := []int{1} +// // appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} +// appendSizes := []int{4095} +// +// tester := &chunkerTester{t: t} +// chunker := NewPyramidChunker(NewChunkerParams()) +// for i, s := range sizes { +// testRandomDataAppend(chunker, s, appendSizes[i], tester) +// +// } +// } func TestRandomData(t *testing.T) { sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} @@ -390,12 +390,12 @@ func benchmarkJoin(n int, t *testing.B) { data := testDataReader(n) chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil) + key, wait, err := tester.Split(chunker, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } + wait() chunkC = make(chan *Chunk, 1000) quitC := make(chan bool) reader := tester.Join(chunker, key, i, chunkC, quitC) @@ -411,7 +411,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { chunker := NewTreeChunker(NewChunkerParams()) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(chunker, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -426,7 +426,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { chunker := NewTreeChunker(cp) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(chunker, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -439,10 +439,11 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { splitter := NewPyramidChunker(NewChunkerParams()) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(splitter, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } + } } @@ -454,7 +455,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { splitter := NewPyramidChunker(cp) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(splitter, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -470,20 +471,18 @@ func benchmarkAppendPyramid(n, m int, t *testing.B) { data1 := testDataReader(m) chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil) + key, wait, err := tester.Split(chunker, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } - + wait() chunkC = make(chan *Chunk, 1000) - swg = &sync.WaitGroup{} - _, err = tester.Append(chunker, key, data1, chunkC, swg, nil) + _, wait, err = tester.Append(chunker, key, data1, chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } - + wait() close(chunkC) } } @@ -494,7 +493,8 @@ func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) } func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) } func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) } func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) } -func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } + +// func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) } func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) } @@ -505,7 +505,8 @@ func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) } func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) } func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) } -func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } + +// func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) } func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) } @@ -516,7 +517,8 @@ func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) } func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) } func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) } func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) } -func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } + +// func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) } func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) } @@ -527,7 +529,8 @@ func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(5000 func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) } func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) } func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) } -func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } + +// func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) } func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) } @@ -538,7 +541,8 @@ func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) } func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) } func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) } -func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } + +// func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } func BenchmarkAppendPyramid_2(t *testing.B) { benchmarkAppendPyramid(100, 1000, t) } func BenchmarkAppendPyramid_2h(t *testing.B) { benchmarkAppendPyramid(500, 1000, t) } @@ -548,7 +552,8 @@ func BenchmarkAppendPyramid_4h(t *testing.B) { benchmarkAppendPyramid(50000, 100 func BenchmarkAppendPyramid_5(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } func BenchmarkAppendPyramid_6(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } func BenchmarkAppendPyramid_7(t *testing.B) { benchmarkAppendPyramid(10000000, 1000, t) } -func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } + +// func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } // go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no // If you dont add the timeout argument above .. the benchmark will timeout and dump diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index cd4c2ef139..21f8e5bbc1 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -19,14 +19,29 @@ package storage import ( "bytes" "crypto/rand" + "encoding/binary" + "flag" "fmt" + "hash" "io" + "os" "sync" "testing" + "time" + "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" ) +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) +} + type brokenLimitedReader struct { lr io.Reader errAt int @@ -42,16 +57,117 @@ func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader } } +func mputChunks(store ChunkStore, processors int, n int, chunksize int, hash hash.Hash) (hs []Key) { + f := func(int) *Chunk { + data := make([]byte, chunksize) + rand.Reader.Read(data) + hash.Reset() + hash.Write(data) + h := hash.Sum(nil) + chunk := NewChunk(Key(h), nil) + chunk.SData = data + return chunk + } + return mput(store, processors, n, f) +} + +func mputRandomKey(store ChunkStore, processors int, n int, chunksize int) (hs []Key) { + data := make([]byte, chunksize+8) + binary.LittleEndian.PutUint64(data[0:8], uint64(chunksize)) + + f := func(int) *Chunk { + h := make([]byte, 32) + rand.Reader.Read(h) + chunk := NewChunk(Key(h), nil) + chunk.SData = data + return chunk + } + return mput(store, processors, n, f) +} + +func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []Key) { + wg := sync.WaitGroup{} + wg.Add(processors) + c := make(chan *Chunk) + for i := 0; i < processors; i++ { + go func() { + defer wg.Done() + for chunk := range c { + wg.Add(1) + chunk := chunk + go func() { + defer wg.Done() + + store.Put(chunk) + + <-chunk.dbStored + }() + } + }() + } + fa := f + if _, ok := store.(*MemStore); ok { + fa = func(i int) *Chunk { + chunk := f(i) + close(chunk.dbStored) + return chunk + } + } + for i := 0; i < n; i++ { + chunk := fa(i) + hs = append(hs, chunk.Key) + c <- chunk + } + close(c) + wg.Wait() + return hs +} + +func mget(store ChunkStore, hs []Key, f func(h Key, chunk *Chunk) error) error { + wg := sync.WaitGroup{} + wg.Add(len(hs)) + errc := make(chan error) + + for _, k := range hs { + go func(h Key) { + defer wg.Done() + chunk, err := store.Get(h) + if err != nil { + errc <- err + return + } + if f != nil { + err = f(h, chunk) + if err != nil { + errc <- err + return + } + } + }(k) + } + go func() { + wg.Wait() + close(errc) + }() + var err error + select { + case err = <-errc: + case <-time.NewTimer(5 * time.Second).C: + err = fmt.Errorf("timed out after 5 seconds") + } + return err +} + func testDataReader(l int) (r io.Reader) { return io.LimitReader(rand.Reader, int64(l)) } -func (self *brokenLimitedReader) Read(buf []byte) (int, error) { - if self.off+len(buf) > self.errAt { +func (r *brokenLimitedReader) Read(buf []byte) (int, error) { + if r.off+len(buf) > r.errAt { return 0, fmt.Errorf("Broken reader") } - self.off += len(buf) - return self.lr.Read(buf) + r.off += len(buf) + return r.lr.Read(buf) } func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) { @@ -63,54 +179,50 @@ func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) { return } -func testStore(m ChunkStore, l int64, branches int64, t *testing.T) { - - chunkC := make(chan *Chunk) - go func() { - for chunk := range chunkC { - m.Put(chunk) - if chunk.wg != nil { - chunk.wg.Done() - } - } - }() - chunker := NewTreeChunker(&ChunkerParams{ - Branches: branches, - Hash: SHA3Hash, - }) - swg := &sync.WaitGroup{} - key, _ := chunker.Split(rand.Reader, l, chunkC, swg, nil) - swg.Wait() - close(chunkC) - chunkC = make(chan *Chunk) - - quit := make(chan bool) - - go func() { - for ch := range chunkC { - go func(chunk *Chunk) { - storedChunk, err := m.Get(chunk.Key) - if err == notFound { - log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log())) - } else if err != nil { - log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err)) - } else { - chunk.SData = storedChunk.SData - chunk.Size = storedChunk.Size - } - log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log())) - close(chunk.C) - }(ch) - } - close(quit) - }() - r := chunker.Join(key, chunkC) - - b := make([]byte, l) - n, err := r.ReadAt(b, 0) - if err != io.EOF { - t.Fatalf("read error (%v/%v) %v", n, l, err) +func testStoreRandom(m ChunkStore, processors int, n int, chunksize int, t *testing.T) { + hs := mputRandomKey(m, processors, n, chunksize) + err := mget(m, hs, nil) + if err != nil { + t.Fatalf("testStore failed: %v", err) + } +} + +func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int, t *testing.T) { + hs := mputChunks(m, processors, n, chunksize, sha3.NewKeccak256()) + f := func(h Key, chunk *Chunk) error { + if !bytes.Equal(h, chunk.Key) { + return fmt.Errorf("key does not match retrieved chunk Key") + } + hasher := sha3.NewKeccak256() + hasher.Write(chunk.SData) + exp := hasher.Sum(nil) + if !bytes.Equal(h, exp) { + return fmt.Errorf("key is not hash of chunk data") + } + return nil + } + err := mget(m, hs, f) + if err != nil { + t.Fatalf("testStore failed: %v", err) + } +} + +func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int, b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + mputRandomKey(store, processors, n, chunksize) + } +} + +func benchmarkStoreGet(store ChunkStore, processors int, n int, chunksize int, b *testing.B) { + hs := mputRandomKey(store, processors, n, chunksize) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := mget(store, hs, nil) + if err != nil { + b.Fatalf("mget failed: %v", err) + } } - close(chunkC) - <-quit } diff --git a/swarm/storage/dbapi.go b/swarm/storage/dbapi.go new file mode 100644 index 0000000000..69a659564a --- /dev/null +++ b/swarm/storage/dbapi.go @@ -0,0 +1,52 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package storage + +// wrapper of db-s to provide mockable custom local chunk store access to syncer +type DBAPI struct { + db *DbStore + loc *LocalStore +} + +func NewDBAPI(loc *LocalStore) *DBAPI { + return &DBAPI{loc.DbStore.(*DbStore), loc} +} + +// to obtain the chunks from key or request db entry only +func (self *DBAPI) Get(key Key) (*Chunk, error) { + return self.loc.Get(key) +} + +// current storage counter of chunk db +func (self *DBAPI) CurrentBucketStorageIndex(po uint8) uint64 { + return self.db.CurrentBucketStorageIndex(po) +} + +// iteration storage counter and proximity order +func (self *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Key, uint64) bool) error { + return self.db.SyncIterator(from, to, po, f) +} + +// to obtain the chunks from key or request db entry only +func (self *DBAPI) GetOrCreateRequest(key Key) (*Chunk, bool) { + return self.loc.GetOrCreateRequest(key) +} + +// to obtain the chunks from key or request db entry only +func (self *DBAPI) Put(chunk *Chunk) { + self.loc.Put(chunk) +} diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 39608d934a..ea1ef46a02 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -36,7 +36,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/opt" ) const ( @@ -52,10 +52,13 @@ const ( ) var ( - keyAccessCnt = []byte{2} - keyEntryCnt = []byte{3} - keyDataIdx = []byte{4} - keyGCPos = []byte{5} + keyOldData = byte(1) + keyAccessCnt = []byte{2} + keyEntryCnt = []byte{3} + keyDataIdx = []byte{4} + keyGCPos = []byte{5} + keyData = byte(6) + keyDistanceCnt = byte(7) ) type gcItem struct { @@ -69,13 +72,19 @@ type DbStore struct { // this should be stored in db, accessed transactionally entryCnt, accessCnt, dataIdx, capacity uint64 + bucketCnt []uint64 gcPos, gcStartPos []byte gcArray []*gcItem hashfunc SwarmHasher + po func(Key) uint8 - lock sync.Mutex + batchC chan bool + batchesC chan struct{} + batch *leveldb.Batch + lock sync.RWMutex + trusted bool // if hash integity check is to be performed (for testing only) // Functions encodeDataFunc is used to bypass // the default functionality of DbStore with @@ -87,43 +96,63 @@ type DbStore struct { getDataFunc func(key Key) (data []byte, err error) } -func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) { +// TODO: Instead of passing the distance function, just pass the address from which distances are calculated +// to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing +// a function different from the one that is actually used. +func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *DbStore, err error) { s = new(DbStore) - s.hashfunc = hash + s.batchC = make(chan bool) + s.batchesC = make(chan struct{}, 1) + go s.writeBatches() + s.batch = new(leveldb.Batch) // associate encodeData with default functionality s.encodeDataFunc = encodeData s.db, err = NewLDBDatabase(path) if err != nil { - return + return nil, err } + s.po = po s.setCapacity(capacity) s.gcStartPos = make([]byte, 1) s.gcStartPos[0] = kpIndex s.gcArray = make([]*gcItem, gcArraySize) + s.bucketCnt = make([]uint64, 0x100) + for i := 0; i < 0x100; i++ { + k := make([]byte, 2) + k[0] = keyDistanceCnt + k[1] = uint8(i) + cnt, _ := s.db.Get(k) + s.bucketCnt[i] = BytesToU64(cnt) + s.bucketCnt[i]++ + } data, _ := s.db.Get(keyEntryCnt) s.entryCnt = BytesToU64(data) + s.entryCnt++ data, _ = s.db.Get(keyAccessCnt) s.accessCnt = BytesToU64(data) + s.accessCnt++ data, _ = s.db.Get(keyDataIdx) s.dataIdx = BytesToU64(data) + s.dataIdx++ + s.gcPos, _ = s.db.Get(keyGCPos) if s.gcPos == nil { s.gcPos = s.gcStartPos } - return + return s, nil } // NewMockDbStore creates a new instance of DbStore with // mockStore set to a provided value. If mockStore argument is nil, // this function behaves exactly as NewDbStore. -func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, radius int, mockStore *mock.NodeStore) (s *DbStore, err error) { - s, err = NewDbStore(path, hash, capacity, radius) +func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8, mockStore *mock.NodeStore) (s *DbStore, err error) { + s, err = NewDbStore(path, hash, capacity, po) if err != nil { return nil, err } @@ -144,12 +173,14 @@ func BytesToU64(data []byte) uint64 { if len(data) < 8 { return 0 } - return binary.LittleEndian.Uint64(data) + //return binary.LittleEndian.Uint64(data) + return binary.BigEndian.Uint64(data) } func U64ToBytes(val uint64) []byte { data := make([]byte, 8) - binary.LittleEndian.PutUint64(data, val) + //binary.LittleEndian.PutUint64(data, val) + binary.BigEndian.PutUint64(data, val) return data } @@ -162,38 +193,53 @@ func (s *DbStore) updateIndexAccess(index *dpaDBIndex) { } func getIndexKey(hash Key) []byte { - HashSize := len(hash) - key := make([]byte, HashSize+1) + hashSize := len(hash) + key := make([]byte, hashSize+1) key[0] = 0 copy(key[1:], hash[:]) return key } -func getDataKey(idx uint64) []byte { +func getOldDataKey(idx uint64) []byte { key := make([]byte, 9) - key[0] = 1 + key[0] = keyOldData binary.BigEndian.PutUint64(key[1:9], idx) return key } +func getDataKey(idx uint64, po uint8) []byte { + key := make([]byte, 10) + key[0] = keyData + key[1] = po + binary.BigEndian.PutUint64(key[2:], idx) + + return key +} + func encodeIndex(index *dpaDBIndex) []byte { data, _ := rlp.EncodeToBytes(index) return data } func encodeData(chunk *Chunk) []byte { - return chunk.SData + return append(chunk.Key[:], chunk.SData...) } -func decodeIndex(data []byte, index *dpaDBIndex) { +func decodeIndex(data []byte, index *dpaDBIndex) error { dec := rlp.NewStream(bytes.NewReader(data), 0) - dec.Decode(index) + return dec.Decode(index) + } func decodeData(data []byte, chunk *Chunk) { + chunk.SData = data[32:] + chunk.Size = int64(binary.BigEndian.Uint64(data[32:40])) +} + +func decodeOldData(data []byte, chunk *Chunk) { chunk.SData = data - chunk.Size = int64(binary.LittleEndian.Uint64(data[0:8])) + chunk.Size = int64(binary.BigEndian.Uint64(data[0:8])) } func gcListPartition(list []*gcItem, left int, right int, pivotIndex int) int { @@ -279,17 +325,13 @@ func (s *DbStore) collectGarbage(ratio float32) { cutidx := gcListSelect(s.gcArray, 0, gcnt-1, int(float32(gcnt)*ratio)) cutval := s.gcArray[cutidx].value - // fmt.Print(gcnt, " ", s.entryCnt, " ") - // actual gc for i := 0; i < gcnt; i++ { if s.gcArray[i].value <= cutval { - s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey) + s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey, s.po(Key(s.gcPos[1:]))) } } - // fmt.Println(s.entryCnt) - s.db.Put(keyGCPos, s.gcPos) } @@ -311,14 +353,16 @@ func (s *DbStore) Export(out io.Writer) (int64, error) { var index dpaDBIndex decodeIndex(it.Value(), &index) - data, err := s.db.Get(getDataKey(index.Idx)) + hash := key[1:] + + data, err := s.db.Get(getDataKey(index.Idx, s.po(hash))) if err != nil { log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) continue } hdr := &tar.Header{ - Name: hex.EncodeToString(key[1:]), + Name: hex.EncodeToString(hash), Mode: 0644, Size: int64(len(data)), } @@ -334,12 +378,12 @@ func (s *DbStore) Export(out io.Writer) (int64, error) { return count, nil } -// Import reads chunks into the store from a tar archive, returning the number // of chunks read. func (s *DbStore) Import(in io.Reader) (int64, error) { tr := tar.NewReader(in) var count int64 + var wg sync.WaitGroup for { hdr, err := tr.Next() if err == io.EOF { @@ -363,11 +407,17 @@ func (s *DbStore) Import(in io.Reader) (int64, error) { if err != nil { return count, err } - - s.Put(&Chunk{Key: key, SData: data}) + chunk := NewChunk(key, nil) + chunk.SData = data + s.Put(chunk) + wg.Add(1) + go func() { + defer wg.Done() + <-chunk.dbStored + }() count++ } - + wg.Wait() return count, nil } @@ -385,21 +435,23 @@ func (s *DbStore) Cleanup() { } total++ var index dpaDBIndex - decodeIndex(it.Value(), &index) - - data, err := s.db.Get(getDataKey(index.Idx)) + err := decodeIndex(it.Value(), &index) + if err != nil { + it.Next() + continue + } + data, err := s.db.Get(getDataKey(index.Idx, s.po(Key(key[1:])))) if err != nil { log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) - s.delete(index.Idx, getIndexKey(key[1:])) + s.delete(index.Idx, getIndexKey(key[1:]), s.po(Key(key[1:]))) errorsFound++ } else { hasher := s.hashfunc() - hasher.Write(data) + hasher.Write(data[32:]) hash := hasher.Sum(nil) if !bytes.Equal(hash, key[1:]) { log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:])) - s.delete(index.Idx, getIndexKey(key[1:])) - errorsFound++ + s.delete(index.Idx, getIndexKey(key[1:]), s.po(Key(key[1:]))) } } it.Next() @@ -408,65 +460,165 @@ func (s *DbStore) Cleanup() { log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) } -func (s *DbStore) delete(idx uint64, idxKey []byte) { +func (s *DbStore) ReIndex() { + //Iterates over the database and checks that there are no faulty chunks + it := s.db.NewIterator() + startPosition := []byte{keyOldData} + it.Seek(startPosition) + var key []byte + var errorsFound, total int + for it.Valid() { + key = it.Key() + if (key == nil) || (key[0] != keyOldData) { + break + } + data := it.Value() + hasher := s.hashfunc() + hasher.Write(data) + hash := hasher.Sum(nil) + + newKey := make([]byte, 10) + oldCntKey := make([]byte, 2) + newCntKey := make([]byte, 2) + oldCntKey[0] = keyDistanceCnt + newCntKey[0] = keyDistanceCnt + key[0] = keyData + key[1] = s.po(Key(key[1:])) + oldCntKey[1] = key[1] + newCntKey[1] = s.po(Key(newKey[1:])) + copy(newKey[2:], key[1:]) + newValue := append(hash, data...) + + batch := new(leveldb.Batch) + batch.Delete(key) + s.bucketCnt[oldCntKey[1]]-- + batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]])) + batch.Put(newKey, newValue) + s.bucketCnt[newCntKey[1]]++ + batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]])) + s.db.Write(batch) + it.Next() + } + it.Release() + log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) +} + +func (s *DbStore) delete(idx uint64, idxKey []byte, po uint8) { batch := new(leveldb.Batch) batch.Delete(idxKey) - batch.Delete(getDataKey(idx)) + batch.Delete(getDataKey(idx, po)) s.entryCnt-- + s.bucketCnt[po]-- + cntKey := make([]byte, 2) + cntKey[0] = keyDistanceCnt + cntKey[1] = po batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) + batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) s.db.Write(batch) } -func (s *DbStore) Counter() uint64 { +func (s *DbStore) CurrentBucketStorageIndex(po uint8) uint64 { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.bucketCnt[po] +} + +func (s *DbStore) Size() uint64 { s.lock.Lock() defer s.lock.Unlock() + return s.entryCnt +} + +func (s *DbStore) CurrentStorageIndex() uint64 { + s.lock.RLock() + defer s.lock.RUnlock() return s.dataIdx } func (s *DbStore) Put(chunk *Chunk) { - s.lock.Lock() - defer s.lock.Unlock() - ikey := getIndexKey(chunk.Key) var index dpaDBIndex - if s.tryAccessIdx(ikey, &index) { - if chunk.dbStored != nil { + po := s.po(chunk.Key) + s.lock.Lock() + defer s.lock.Unlock() + + idata, err := s.db.Get(ikey) + if err != nil { + s.doPut(chunk, ikey, &index, po) + batchC := s.batchC + go func() { + <-batchC close(chunk.dbStored) - } - log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access")) - return // already exists, only update access - } - - data := s.encodeDataFunc(chunk) - //data := ethutil.Encode([]interface{}{entry}) - - if s.entryCnt >= s.capacity { - s.collectGarbage(gcArrayFreeRatio) - } - - batch := new(leveldb.Batch) - - batch.Put(getDataKey(s.dataIdx), data) - - index.Idx = s.dataIdx - s.updateIndexAccess(&index) - - idata := encodeIndex(&index) - batch.Put(ikey, idata) - - batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) - s.entryCnt++ - batch.Put(keyDataIdx, U64ToBytes(s.dataIdx)) - s.dataIdx++ - batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) - s.accessCnt++ - - s.db.Write(batch) - if chunk.dbStored != nil { + }() + } else { + log.Trace(fmt.Sprintf("DbStore: chunk already exists, only update access")) + decodeIndex(idata, &index) close(chunk.dbStored) } - log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx)) + index.Access = s.accessCnt + s.accessCnt++ + idata = encodeIndex(&index) + s.batch.Put(ikey, idata) + select { + case s.batchesC <- struct{}{}: + default: + } +} + +// force putting into db, does not check access index +func (s *DbStore) doPut(chunk *Chunk, ikey []byte, index *dpaDBIndex, po uint8) { + data := s.encodeDataFunc(chunk) + s.batch.Put(getDataKey(s.dataIdx, po), data) + index.Idx = s.dataIdx + s.bucketCnt[po] = s.dataIdx + s.entryCnt++ + s.dataIdx++ + + cntKey := make([]byte, 2) + cntKey[0] = keyDistanceCnt + cntKey[1] = po + s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) + +} + +func (s *DbStore) writeBatches() { + for range s.batchesC { + s.lock.Lock() + b := s.batch + e := s.entryCnt + d := s.dataIdx + a := s.accessCnt + c := s.batchC + s.batchC = make(chan bool) + s.batch = new(leveldb.Batch) + s.lock.Unlock() + err := s.writeBatch(b, e, d, a) + // TODO: set this error on the batch, then tell the chunk + if err != nil { + log.Error(fmt.Sprintf("DbStore: spawn batch write (%d chunks): %v", b.Len(), err)) + } + close(c) + if e >= s.capacity { + log.Trace(fmt.Sprintf("DbStore: collecting garbage...(%d chunks)", e)) + s.collectGarbage(gcArrayFreeRatio) + } + } + log.Trace(fmt.Sprintf("DbStore: quit batch write loop")) +} + +// must be called non concurrently +func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) error { + b.Put(keyEntryCnt, U64ToBytes(entryCnt)) + b.Put(keyDataIdx, U64ToBytes(dataIdx)) + b.Put(keyAccessCnt, U64ToBytes(accessCnt)) + l := b.Len() + if err := s.db.Write(b); err != nil { + return fmt.Errorf("unable to write batch: %v", err) + } + log.Trace(fmt.Sprintf("DbStore: batch write (%d chunks) complete", l)) + return nil } // newMockEncodeDataFunc returns a function that stores the chunk data @@ -475,10 +627,10 @@ func (s *DbStore) Put(chunk *Chunk) { // not need to store the data, but still need to create the index. func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk *Chunk) []byte { return func(chunk *Chunk) []byte { - if err := mockStore.Put(chunk.Key, chunk.SData); err != nil { + if err := mockStore.Put(chunk.Key, encodeData(chunk)); err != nil { log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Key.Log(), err)) } - return nil + return chunk.Key[:] } } @@ -489,27 +641,24 @@ func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { return false } decodeIndex(idata, index) - - batch := new(leveldb.Batch) - - batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) + s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) s.accessCnt++ - s.updateIndexAccess(index) + index.Access = s.accessCnt idata = encodeIndex(index) - batch.Put(ikey, idata) - - s.db.Write(batch) - + s.batch.Put(ikey, idata) return true } func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { s.lock.Lock() defer s.lock.Unlock() + return s.get(key) +} - var index dpaDBIndex +func (s *DbStore) get(key Key) (chunk *Chunk, err error) { + var indx dpaDBIndex - if s.tryAccessIdx(getIndexKey(key), &index) { + if s.tryAccessIdx(getIndexKey(key), &indx) { var data []byte if s.getDataFunc != nil { // if getDataFunc is defined, use it to retrieve the chunk data @@ -519,31 +668,34 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { } } else { // default DbStore functionality to retrieve chunk data - data, err = s.db.Get(getDataKey(index.Idx)) + proximity := s.po(key) + datakey := getDataKey(indx.Idx, proximity) + data, err = s.db.Get(datakey) + log.Trace(fmt.Sprintf("DBStore: Chunk %v indexkey %v datakey %x proximity %d", key.Log(), indx.Idx, datakey, proximity)) if err != nil { log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err)) - s.delete(index.Idx, getIndexKey(key)) + s.delete(indx.Idx, getIndexKey(key), s.po(key)) return } } - if s.hashfunc != nil { + if !s.trusted { + data_mod := data[32:] hasher := s.hashfunc() - hasher.Write(data) + hasher.Write(data_mod) hash := hasher.Sum(nil) + if !bytes.Equal(hash, key) { - s.delete(index.Idx, getIndexKey(key)) - log.Warn("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") + log.Error(fmt.Sprintf("Apparent key/hash mismatch. Hash %x, key %v", hash, key[:])) + s.delete(indx.Idx, getIndexKey(key), s.po(key)) + log.Error("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") } } - chunk = &Chunk{ - Key: key, - } + chunk = NewChunk(key, nil) decodeData(data, chunk) - } else { - err = notFound + err = ErrNotFound } return @@ -556,8 +708,8 @@ func newMockGetDataFunc(mockStore *mock.NodeStore) func(key Key) (data []byte, e return func(key Key) (data []byte, err error) { data, err = mockStore.Get(key) if err == mock.ErrNotFound { - // preserve notFound error - err = notFound + // preserve ErrNotFound error + err = ErrNotFound } return data, err } @@ -598,62 +750,36 @@ func (s *DbStore) Close() { s.db.Close() } -// describes a section of the DbStore representing the unsynced -// domain relevant to a peer -// Start - Stop designate a continuous area Keys in an address space -// typically the addresses closer to us than to the peer but not closer -// another closer peer in between -// From - To designates a time interval typically from the last disconnect -// till the latest connection (real time traffic is relayed) -type DbSyncState struct { - Start, Stop Key - First, Last uint64 -} +// SyncIterator(start, stop, po, f) calls f on each hash of a bin po from start to stop +func (s *DbStore) SyncIterator(since uint64, until uint64, po uint8, f func(Key, uint64) bool) error { + sincekey := getDataKey(since, po) + untilkey := getDataKey(until, po) + it := s.db.NewIterator() + defer it.Release() -// implements the syncer iterator interface -// iterates by storage index (~ time of storage = first entry to db) -type dbSyncIterator struct { - it iterator.Iterator - DbSyncState -} - -// initialises a sync iterator from a syncToken (passed in with the handshake) -func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) { - if state.First > state.Last { - return nil, fmt.Errorf("no entries found") - } - si = &dbSyncIterator{ - it: self.db.NewIterator(), - DbSyncState: state, - } - si.it.Seek(getIndexKey(state.Start)) - return si, nil -} - -// walk the area from Start to Stop and returns items within time interval -// First to Last -func (self *dbSyncIterator) Next() (key Key) { - for self.it.Valid() { - dbkey := self.it.Key() - if dbkey[0] != 0 { + for ok := it.Seek(sincekey); ok; ok = it.Next() { + dbkey := it.Key() + if dbkey[0] != keyData || dbkey[1] != po || bytes.Compare(untilkey, dbkey) < 0 { break } - key = Key(make([]byte, len(dbkey)-1)) - copy(key[:], dbkey[1:]) - if bytes.Compare(key[:], self.Start) <= 0 { - self.it.Next() - continue - } - if bytes.Compare(key[:], self.Stop) > 0 { + key := make([]byte, 32) + val := it.Value() + copy(key, val[:32]) + if !f(Key(key), binary.BigEndian.Uint64(dbkey[2:])) { break } - var index dpaDBIndex - decodeIndex(self.it.Value(), &index) - self.it.Next() - if (index.Idx >= self.First) && (index.Idx < self.Last) { - return - } } - self.it.Release() - return nil + return it.Error() +} + +func databaseExists(path string) bool { + o := &opt.Options{ + ErrorIfMissing: true, + } + tdb, err := leveldb.OpenFile(path, o) + if err != nil { + return false + } + defer tdb.Close() + return true } diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 48e8f9a092..65a1bc9669 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -18,250 +18,256 @@ package storage import ( "bytes" + "fmt" "io/ioutil" - "strings" + "os" + "sync" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" ) -func initDbStore(t *testing.T) *DbStore { +type testDbStore struct { + *DbStore + dir string +} + +func newTestDbStore(mock bool) (*testDbStore, error) { dir, err := ioutil.TempDir("", "bzz-storage-test") if err != nil { - t.Fatal(err) + return nil, err } - m, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius) + + var db *DbStore + if mock { + globalStore := mem.NewGlobalStore() + addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") + mockStore := globalStore.NewNodeStore(addr) + + db, err = NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc, mockStore) + } else { + db, err = NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc) + } + + return &testDbStore{db, dir}, err +} + +func testPoFunc(k Key) (ret uint8) { + basekey := make([]byte, 32) + return uint8(Proximity(basekey[:], k[:])) +} + +func (db *testDbStore) close() { + db.Close() + err := os.RemoveAll(db.dir) if err != nil { - t.Fatal("can't create store:", err) + panic(err) } - return m } -func testDbStore(l int64, branches int64, t *testing.T) { - m := initDbStore(t) - defer m.Close() - testStore(m, l, branches, t) +func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) { + db, err := newTestDbStore(mock) + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + db.trusted = true + testStoreRandom(db, processors, n, chunksize, t) } -func TestDbStore128_0x1000000(t *testing.T) { - testDbStore(0x1000000, 128, t) +func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) { + db, err := newTestDbStore(mock) + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + testStoreCorrect(db, processors, n, chunksize, t) } -func TestDbStore128_10000_(t *testing.T) { - testDbStore(10000, 128, t) +func TestDbStoreRandom_1(t *testing.T) { + testDbStoreRandom(1, 1, 0, false, t) } -func TestDbStore128_1000_(t *testing.T) { - testDbStore(1000, 128, t) +func TestDbStoreCorrect_1(t *testing.T) { + testDbStoreCorrect(1, 1, 4096, false, t) } -func TestDbStore128_100_(t *testing.T) { - testDbStore(100, 128, t) +func TestDbStoreRandom_1_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, false, t) } -func TestDbStore2_100_(t *testing.T) { - testDbStore(100, 2, t) +func TestDbStoreRandom_8_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, false, t) +} + +func TestDbStoreCorrect_1_5k(t *testing.T) { + testDbStoreCorrect(1, 5000, 4096, false, t) +} + +func TestDbStoreCorrect_8_5k(t *testing.T) { + testDbStoreCorrect(8, 5000, 4096, false, t) +} + +func TestMockDbStoreRandom_1(t *testing.T) { + testDbStoreRandom(1, 1, 0, true, t) +} + +func TestMockDbStoreCorrect_1(t *testing.T) { + testDbStoreCorrect(1, 1, 4096, true, t) +} + +func TestMockDbStoreRandom_1_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, true, t) +} + +func TestMockDbStoreRandom_8_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, true, t) +} + +func TestMockDbStoreCorrect_1_5k(t *testing.T) { + testDbStoreCorrect(1, 5000, 4096, true, t) +} + +func TestMockDbStoreCorrect_8_5k(t *testing.T) { + testDbStoreCorrect(8, 5000, 4096, true, t) +} + +func testDbStoreNotFound(t *testing.T, mock bool) { + db, err := newTestDbStore(mock) + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + + _, err = db.Get(ZeroKey) + if err != ErrNotFound { + t.Errorf("Expected ErrNotFound, got %v", err) + } } func TestDbStoreNotFound(t *testing.T) { - m := initDbStore(t) - defer m.Close() - _, err := m.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) - } + testDbStoreNotFound(t, false) } - -func TestDbStoreSyncIterator(t *testing.T) { - m := initDbStore(t) - defer m.Close() - keys := []Key{ - Key(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("3000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - } - for _, key := range keys { - m.Put(NewChunk(key, nil)) - } - it, err := m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 4, - }) - if err != nil { - t.Fatalf("unexpected error creating NewSyncIterator") - } - - var chunk Key - var res []Key - for { - chunk = it.Next() - if chunk == nil { - break - } - res = append(res, chunk) - } - if len(res) != 1 { - t.Fatalf("Expected 1 chunk, got %v: %v", len(res), res) - } - if !bytes.Equal(res[0][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[0]) - } - - if err != nil { - t.Fatalf("unexpected error creating NewSyncIterator") - } - - it, err = m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 4, - }) - - res = nil - for { - chunk = it.Next() - if chunk == nil { - break - } - res = append(res, chunk) - } - if len(res) != 2 { - t.Fatalf("Expected 2 chunk, got %v: %v", len(res), res) - } - if !bytes.Equal(res[0][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[0]) - } - if !bytes.Equal(res[1][:], keys[2]) { - t.Fatalf("Expected %v chunk, got %v", keys[2], res[1]) - } - - if err != nil { - t.Fatalf("unexpected error creating NewSyncIterator") - } - - it, _ = m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 5, - }) - res = nil - for { - chunk = it.Next() - if chunk == nil { - break - } - res = append(res, chunk) - } - if len(res) != 2 { - t.Fatalf("Expected 2 chunk, got %v", len(res)) - } - if !bytes.Equal(res[0][:], keys[4]) { - t.Fatalf("Expected %v chunk, got %v", keys[4], res[0]) - } - if !bytes.Equal(res[1][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[1]) - } - - it, _ = m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 5, - }) - res = nil - for { - chunk = it.Next() - if chunk == nil { - break - } - res = append(res, chunk) - } - if len(res) != 1 { - t.Fatalf("Expected 1 chunk, got %v", len(res)) - } - if !bytes.Equal(res[0][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[0]) - } -} - -func initMockDbStore(t *testing.T, mockStore *mock.NodeStore) *DbStore { - dir, err := ioutil.TempDir("", "bzz-storage-test-mock") - if err != nil { - t.Fatal(err) - } - m, err := NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius, mockStore) - if err != nil { - t.Fatal("can't create store:", err) - } - return m -} - -// testMockDbStore runs the same tests as testDbStore but with mock store configured. -// It also verifies if mock global store is storing the chunk data. -func testMockDbStore(l int64, branches int64, t *testing.T) { - globalStore := mem.NewGlobalStore() - addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") - mockStore := globalStore.NewNodeStore(addr) - m := initMockDbStore(t, mockStore) - defer m.Close() - - key := Key(common.Hex2Bytes("fed1911825fc6a02ebfd19ab218a20455d8d7d275f8bf4d8244eb04364fae6f7")) - data := common.Hex2BytesFixed(strings.Repeat("1234567890abcdf", 10), 4096) - - m.Put(&Chunk{ - Key: key, - SData: data, - }) - - _, err := globalStore.Get(addr, key) - if err != nil { - t.Errorf("unexpected error getting the data from global mock store: %v", err) - } - - if !globalStore.HasKey(addr, key) { - t.Error("key not found in global store") - } - - testStore(m, l, branches, t) - -} - -func TestMockDbStore128_0x1000000(t *testing.T) { - testMockDbStore(0x1000000, 128, t) -} - -func TestMockDbStore128_10000_(t *testing.T) { - testMockDbStore(10000, 128, t) -} - -func TestMockDbStore128_1000_(t *testing.T) { - testMockDbStore(1000, 128, t) -} - -func TestMockDbStore128_100_(t *testing.T) { - testMockDbStore(100, 128, t) -} - -func TestMockDbStore2_100_(t *testing.T) { - testMockDbStore(100, 2, t) -} - func TestMockDbStoreNotFound(t *testing.T) { - globalStore := mem.NewGlobalStore() - mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")) - m := initMockDbStore(t, mockStore) - defer m.Close() - _, err := m.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) - } + testDbStoreNotFound(t, true) +} + +func testIterator(t *testing.T, mock bool) { + var chunkcount int = 32 + var i int + var poc uint + chunkkeys := NewKeyCollection(chunkcount) + chunkkeys_results := NewKeyCollection(chunkcount) + var chunks []*Chunk + + for i := 0; i < chunkcount; i++ { + chunks = append(chunks, NewChunk(nil, nil)) + } + + db, err := newTestDbStore(mock) + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + + FakeChunk(getDefaultChunkSize(), chunkcount, chunks) + + wg := &sync.WaitGroup{} + wg.Add(len(chunks)) + for i = 0; i < len(chunks); i++ { + db.Put(chunks[i]) + chunkkeys[i] = chunks[i].Key + j := i + go func() { + defer wg.Done() + <-chunks[j].dbStored + }() + } + + //testSplit(m, l, 128, chunkkeys, t) + + for i = 0; i < len(chunkkeys); i++ { + log.Trace(fmt.Sprintf("Chunk array pos %d/%d: '%v'", i, chunkcount, chunkkeys[i])) + } + wg.Wait() + i = 0 + for poc = 0; poc <= 255; poc++ { + err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Key, n uint64) bool { + log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc))) + chunkkeys_results[n-1] = k + i++ + return true + }) + if err != nil { + t.Fatalf("Iterator call failed: %v", err) + } + } + + for i = 0; i < chunkcount; i++ { + if !bytes.Equal(chunkkeys[i], chunkkeys_results[i]) { + t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeys_results[i]) + } + } + +} + +func TestIterator(t *testing.T) { + testIterator(t, false) +} +func TestMockIterator(t *testing.T) { + testIterator(t, true) +} + +func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) { + db, err := newTestDbStore(mock) + if err != nil { + b.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + db.trusted = true + benchmarkStorePut(db, processors, n, chunksize, b) +} + +func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) { + db, err := newTestDbStore(mock) + if err != nil { + b.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + db.trusted = true + benchmarkStoreGet(db, processors, n, chunksize, b) +} + +func BenchmarkDbStorePut_1_5k(b *testing.B) { + benchmarkDbStorePut(5000, 1, 4096, false, b) +} + +func BenchmarkDbStorePut_8_5k(b *testing.B) { + benchmarkDbStorePut(5000, 8, 4096, false, b) +} + +func BenchmarkDbStoreGet_1_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 1, 4096, false, b) +} + +func BenchmarkDbStoreGet_8_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 8, 4096, false, b) +} + +func BenchmarkMockDbStorePut_1_5k(b *testing.B) { + benchmarkDbStorePut(5000, 1, 4096, true, b) +} + +func BenchmarkMockDbStorePut_8_5k(b *testing.B) { + benchmarkDbStorePut(5000, 8, 4096, true, b) +} + +func BenchmarkMockDbStoreGet_1_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 1, 4096, true, b) +} + +func BenchmarkMockDbStoreGet_8_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 8, 4096, true, b) } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 49a362555e..7633b61f0b 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -48,7 +48,10 @@ const ( ) var ( - notFound = errors.New("not found") + ErrNotFound = errors.New("not found") + ErrFetching = errors.New("chunk still fetching") + // timeout interval before retrieval is timed out + searchTimeout = 3 * time.Second ) type DPA struct { @@ -59,15 +62,16 @@ type DPA struct { lock sync.Mutex running bool + wg *sync.WaitGroup quitC chan bool } // for testing locally -func NewLocalDPA(datadir string) (*DPA, error) { +func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) { hash := MakeHashFunc("SHA3") - dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, 0) + dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) if err != nil { return nil, err } @@ -79,7 +83,7 @@ func NewLocalDPA(datadir string) (*DPA, error) { } func NewDPA(store ChunkStore, params *ChunkerParams) *DPA { - chunker := NewTreeChunker(params) + chunker := NewPyramidChunker(params) return &DPA{ Chunker: chunker, ChunkStore: store, @@ -91,13 +95,13 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA { // Chunk retrieval blocks on netStore requests with a timeout so reader will // report error if retrieval of chunks within requested range time out. func (self *DPA) Retrieve(key Key) LazySectionReader { - return self.Chunker.Join(key, self.retrieveC) + return self.Chunker.Join(key, self.retrieveC, 0) } // Public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess -func (self *DPA) Store(data io.Reader, size int64, swg *sync.WaitGroup, wwg *sync.WaitGroup) (key Key, err error) { - return self.Chunker.Split(data, size, self.storeC, swg, wwg) +func (self *DPA) Store(data io.Reader, size int64) (key Key, wait func(), err error) { + return self.Chunker.Split(data, size, self.storeC) } func (self *DPA) Start() { @@ -135,11 +139,8 @@ func (self *DPA) retrieveLoop() { func (self *DPA) retrieveWorker() { for chunk := range self.retrieveC { - log.Trace(fmt.Sprintf("dpa: retrieve loop : chunk %v", chunk.Key.Log())) storedChunk, err := self.Get(chunk.Key) - if err == notFound { - log.Trace(fmt.Sprintf("chunk %v not found", chunk.Key.Log())) - } else if err != nil { + if err != nil { log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err)) } else { chunk.SData = storedChunk.SData @@ -165,14 +166,8 @@ func (self *DPA) storeLoop() { } func (self *DPA) storeWorker() { - for chunk := range self.storeC { self.Put(chunk) - if chunk.wg != nil { - log.Trace(fmt.Sprintf("dpa: store processor %v", chunk.Key.Log())) - chunk.wg.Done() - - } select { case <-self.quitC: return @@ -180,62 +175,3 @@ func (self *DPA) storeWorker() { } } } - -// DpaChunkStore implements the ChunkStore interface, -// this chunk access layer assumed 2 chunk stores -// local storage eg. LocalStore and network storage eg., NetStore -// access by calling network is blocking with a timeout - -type dpaChunkStore struct { - n int - localStore ChunkStore - netStore ChunkStore -} - -func NewDpaChunkStore(localStore, netStore ChunkStore) *dpaChunkStore { - return &dpaChunkStore{0, localStore, netStore} -} - -// Get is the entrypoint for local retrieve requests -// waits for response or times out -func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) { - chunk, err = self.netStore.Get(key) - // timeout := time.Now().Add(searchTimeout) - if chunk.SData != nil { - log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData))) - return - } - // TODO: use self.timer time.Timer and reset with defer disableTimer - timer := time.After(searchTimeout) - select { - case <-timer: - log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log())) - err = notFound - case <-chunk.Req.C: - log.Trace(fmt.Sprintf("DPA.Get: %v retrieved, %d bytes (%p)", key.Log(), len(chunk.SData), chunk)) - } - return -} - -// Put is the entrypoint for local store requests coming from storeLoop -func (self *dpaChunkStore) Put(entry *Chunk) { - chunk, err := self.localStore.Get(entry.Key) - if err != nil { - log.Trace(fmt.Sprintf("DPA.Put: %v new chunk. call netStore.Put", entry.Key.Log())) - chunk = entry - } else if chunk.SData == nil { - log.Trace(fmt.Sprintf("DPA.Put: %v request entry found", entry.Key.Log())) - chunk.SData = entry.SData - chunk.Size = entry.Size - } else { - log.Trace(fmt.Sprintf("DPA.Put: %v chunk already known", entry.Key.Log())) - return - } - // from this point on the storage logic is the same with network storage requests - log.Trace(fmt.Sprintf("DPA.Put %v: %v", self.n, chunk.Key.Log())) - self.n++ - self.netStore.Put(chunk) -} - -// Close chunk store -func (self *dpaChunkStore) Close() {} diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index a23b9efebe..20b07216a3 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -21,19 +21,23 @@ import ( "io" "io/ioutil" "os" - "sync" "testing" ) const testDataSize = 0x1000000 func TestDPArandom(t *testing.T) { - dbStore := initDbStore(t) - dbStore.setCapacity(50000) - memStore := NewMemStore(dbStore, defaultCacheCapacity) + tdb, err := newTestDbStore(false) + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer tdb.close() + db := tdb.DbStore + db.setCapacity(50000) + memStore := NewMemStore(db, defaultCacheCapacity) localStore := &LocalStore{ memStore, - dbStore, + db, } chunker := NewTreeChunker(NewChunkerParams()) dpa := &DPA{ @@ -45,12 +49,11 @@ func TestDPArandom(t *testing.T) { defer os.RemoveAll("/tmp/bzz") reader, slice := testDataReaderAndSlice(testDataSize) - wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, testDataSize, wg, nil) + key, wait, err := dpa.Store(reader, testDataSize) if err != nil { t.Errorf("Store error: %v", err) } - wg.Wait() + wait() resultReader := dpa.Retrieve(key) resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) @@ -65,7 +68,7 @@ func TestDPArandom(t *testing.T) { } ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666) ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666) - localStore.memStore = NewMemStore(dbStore, defaultCacheCapacity) + localStore.memStore = NewMemStore(db, defaultCacheCapacity) resultReader = dpa.Retrieve(key) for i := range resultSlice { resultSlice[i] = 0 @@ -83,13 +86,17 @@ func TestDPArandom(t *testing.T) { } func TestDPA_capacity(t *testing.T) { - dbStore := initDbStore(t) - memStore := NewMemStore(dbStore, defaultCacheCapacity) + tdb, err := newTestDbStore(false) + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer tdb.close() + db := tdb.DbStore + memStore := NewMemStore(db, 0) localStore := &LocalStore{ memStore, - dbStore, + db, } - memStore.setCapacity(0) chunker := NewTreeChunker(NewChunkerParams()) dpa := &DPA{ Chunker: chunker, @@ -97,12 +104,11 @@ func TestDPA_capacity(t *testing.T) { } dpa.Start() reader, slice := testDataReaderAndSlice(testDataSize) - wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, testDataSize, wg, nil) + key, wait, err := dpa.Store(reader, testDataSize) if err != nil { t.Errorf("Store error: %v", err) } - wg.Wait() + wait() resultReader := dpa.Retrieve(key) resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) diff --git a/swarm/storage/forwarder.go b/swarm/storage/forwarder.go deleted file mode 100644 index 0d1acfab57..0000000000 --- a/swarm/storage/forwarder.go +++ /dev/null @@ -1,16 +0,0 @@ -package storage - -// implements CloudStore -// noop placeholder for netstore functionality - -type Forwarder struct { -} - -func (self *Forwarder) Store(chunk *Chunk) { -} - -func (self *Forwarder) Retrieve(chunk *Chunk) { -} - -func (self *Forwarder) Deliver(chunk *Chunk) { -} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index bb55cc99fa..9ce6737e27 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -18,10 +18,27 @@ package storage import ( "encoding/binary" + "fmt" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" ) +type StoreParams struct { + ChunkDbPath string + DbCapacity uint64 + CacheCapacity uint + Radius int +} + +//create params with default values +func NewDefaultStoreParams() (self *StoreParams) { + return &StoreParams{ + DbCapacity: defaultDbCapacity, + CacheCapacity: defaultCacheCapacity, + } +} + // LocalStore is a combination of inmemory db over a disk persisted db // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores type LocalStore struct { @@ -29,10 +46,9 @@ type LocalStore struct { DbStore ChunkStore } -// This constructor uses MemStore and DbStore as components. -// If mockStore is not nil, it will be used by DbStore to store chunk data. -func NewLocalStore(hash SwarmHasher, params *StoreParams, mockStore *mock.NodeStore) (*LocalStore, error) { - dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius, mockStore) +// This constructor uses MemStore and DbStore as components +func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte, mockStore *mock.NodeStore) (*LocalStore, error) { + dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }, mockStore) if err != nil { return nil, err } @@ -42,20 +58,45 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams, mockStore *mock.NodeSt }, nil } +func NewTestLocalStore(path string) (*LocalStore, error) { + basekey := make([]byte, 32) + hasher := MakeHashFunc("SHA3") + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } + return localStore, nil +} + +func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) { + hasher := MakeHashFunc("SHA3") + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } + return localStore, nil +} + // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { - chunk.dbStored = make(chan bool) - self.memStore.Put(chunk) - if chunk.wg != nil { - chunk.wg.Add(1) + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + c := &Chunk{ + Key: Key(append([]byte{}, chunk.Key...)), + SData: append([]byte{}, chunk.SData...), + Size: chunk.Size, + dbStored: chunk.dbStored, } - go func() { - self.DbStore.Put(chunk) - if chunk.wg != nil { - chunk.wg.Done() - } - }() + self.memStore.Put(c) + self.DbStore.Put(c) } // Get(chunk *Chunk) looks up a chunk in the local stores @@ -65,6 +106,13 @@ func (self *LocalStore) Put(chunk *Chunk) { func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { chunk, err = self.memStore.Get(key) if err == nil { + if chunk.ReqC != nil { + select { + case <-chunk.ReqC: + default: + return chunk, ErrFetching + } + } return } chunk, err = self.DbStore.Get(key) @@ -76,6 +124,25 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { return } +// retrieve logic common for local and network chunk retrieval requests +func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool) { + var err error + chunk, err = self.Get(key) + if err == nil { + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key)) + return chunk, false + } + if err == ErrFetching { + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", key, chunk.ReqC)) + return chunk, false + } + // no data and no request status + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v not found locally. open new request", key)) + chunk = NewChunk(key, make(chan bool)) + self.memStore.Put(chunk) + return chunk, true +} + // Close the local store func (self *LocalStore) Close() { self.DbStore.Close() diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 3cb25ac625..822eb28b4c 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -168,8 +168,8 @@ func (s *MemStore) Put(entry *Chunk) { entry.Size = node.entry.Size entry.SData = node.entry.SData } - if entry.Req == nil { - entry.Req = node.entry.Req + if entry.ReqC == nil { + entry.ReqC = node.entry.ReqC } entry.C = node.entry.C node.entry = entry @@ -214,7 +214,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { l := hash.bits(bitpos, node.bits) st := node.subtree[l] if st == nil { - return nil, notFound + return nil, ErrNotFound } bitpos += node.bits node = st @@ -232,7 +232,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { } } } else { - err = notFound + err = ErrNotFound } return @@ -240,7 +240,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { func (s *MemStore) removeOldest() { node := s.memtree - + log.Warn("purge memstore") for node.entry == nil { aidx := uint(0) @@ -280,17 +280,15 @@ func (s *MemStore) removeOldest() { } - if node.entry.dbStored != nil { - log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) - <-node.entry.dbStored - log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) - } else { - log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log())) - } + log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) + <-node.entry.dbStored + log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) - if node.entry.SData != nil { + if node.entry.ReqC == nil { node.entry = nil s.entryCnt-- + } else { + return } node.access[0] = 0 @@ -316,5 +314,39 @@ func (s *MemStore) removeOldest() { } } +// type MemStore struct { +// m map[string]*Chunk +// mu sync.RWMutex +// } + +// func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { +// return &MemStore{ +// m: make(map[string]*Chunk), +// } +// } + +// func (m *MemStore) Get(key Key) (*Chunk, error) { +// m.mu.RLock() +// defer m.mu.RUnlock() +// c, ok := m.m[string(key[:])] +// if !ok { +// return nil, ErrNotFound +// } +// if !bytes.Equal(c.Key, key) { +// panic(fmt.Errorf("MemStore.Get: chunk key %s != req key %s", c.Key.Hex(), key.Hex())) +// } +// return c, nil +// } + +// func (m *MemStore) Put(c *Chunk) { +// m.mu.Lock() +// defer m.mu.Unlock() +// m.m[string(c.Key[:])] = c +// } + +// func (m *MemStore) setCapacity(n int) { + +// } + // Close memstore func (s *MemStore) Close() {} diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 2e0ab535af..edf87917e8 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -16,35 +16,82 @@ package storage -import ( - "testing" -) +import "testing" -func testMemStore(l int64, branches int64, t *testing.T) { - m := NewMemStore(nil, defaultCacheCapacity) - testStore(m, l, branches, t) +func newTestMemStore() *MemStore { + return NewMemStore(nil, defaultCacheCapacity) } -func TestMemStore128_10000(t *testing.T) { - testMemStore(10000, 128, t) +func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) { + m := newTestMemStore() + defer m.Close() + testStoreRandom(m, processors, n, chunksize, t) } -func TestMemStore128_1000(t *testing.T) { - testMemStore(1000, 128, t) +func testMemStoreCorrect(n int, processors int, chunksize int, t *testing.T) { + m := newTestMemStore() + defer m.Close() + testStoreCorrect(m, processors, n, chunksize, t) } -func TestMemStore128_100(t *testing.T) { - testMemStore(100, 128, t) +func TestMemStoreRandom_1(t *testing.T) { + testMemStoreRandom(1, 1, 0, t) } -func TestMemStore2_100(t *testing.T) { - testMemStore(100, 2, t) +func TestMemStoreCorrect_1(t *testing.T) { + testMemStoreCorrect(1, 1, 4104, t) +} + +func TestMemStoreRandom_1_10k(t *testing.T) { + testMemStoreRandom(1, 5000, 0, t) +} + +func TestMemStoreCorrect_1_10k(t *testing.T) { + testMemStoreCorrect(1, 5000, 4096, t) +} + +func TestMemStoreRandom_8_10k(t *testing.T) { + testMemStoreRandom(8, 5000, 0, t) +} + +func TestMemStoreCorrect_8_10k(t *testing.T) { + testMemStoreCorrect(8, 5000, 4096, t) } func TestMemStoreNotFound(t *testing.T) { - m := NewMemStore(nil, defaultCacheCapacity) + m := newTestMemStore() + defer m.Close() + _, err := m.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) + if err != ErrNotFound { + t.Errorf("Expected ErrNotFound, got %v", err) } } + +func benchmarkMemStorePut(n int, processors int, chunksize int, b *testing.B) { + m := newTestMemStore() + defer m.Close() + benchmarkStorePut(m, processors, n, chunksize, b) +} + +func benchmarkMemStoreGet(n int, processors int, chunksize int, b *testing.B) { + m := newTestMemStore() + defer m.Close() + benchmarkStoreGet(m, processors, n, chunksize, b) +} + +func BenchmarkMemStorePut_1_5k(b *testing.B) { + benchmarkMemStorePut(5000, 1, 4096, b) +} + +func BenchmarkMemStorePut_8_5k(b *testing.B) { + benchmarkMemStorePut(5000, 8, 4096, b) +} + +func BenchmarkMemStoreGet_1_5k(b *testing.B) { + benchmarkMemStoreGet(5000, 1, 4096, b) +} + +func BenchmarkMemStoreGet_8_5k(b *testing.B) { + benchmarkMemStoreGet(5000, 8, 4096, b) +} diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 7661f122b5..e63ad2a7d5 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -17,53 +17,57 @@ package storage import ( - "fmt" "path/filepath" "time" - - "github.com/ethereum/go-ethereum/log" ) -/* -NetStore is a cloud storage access abstaction layer for swarm -it contains the shared logic of network served chunk store/retrieval requests -both local (coming from DPA api) and remote (coming from peers via bzz protocol) -it implements the ChunkStore interface and embeds LocalStore - -It is called by the bzz protocol instances via Depo (the store/retrieve request handler) -a protocol instance is running on each peer, so this is heavily parallelised. -NetStore falls back to a backend (CloudStorage interface) -implemented by bzz/network/forwarder. forwarder or IPFS or IPΞS -*/ +// NetStore implements the ChunkStore interface, +// this chunk access layer assumed 2 chunk stores +// local storage eg. LocalStore and network storage eg., NetStore +// access by calling network is blocking with a timeout type NetStore struct { - hashfunc SwarmHasher localStore *LocalStore - cloud CloudStore + retrieve func(chunk *Chunk) error } -// backend engine for cloud store -// It can be aggregate dispatching to several parallel implementations: -// bzz/network/forwarder. forwarder or IPFS or IPΞS -type CloudStore interface { - Store(*Chunk) - Deliver(*Chunk) - Retrieve(*Chunk) +func NewNetStore(localStore *LocalStore, retrieve func(chunk *Chunk) error) *NetStore { + return &NetStore{localStore, retrieve} } -type StoreParams struct { - ChunkDbPath string - DbCapacity uint64 - CacheCapacity uint - Radius int -} +// Get is the entrypoint for local retrieve requests +// waits for response or times out +func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { + if self.retrieve == nil { + chunk, err = self.localStore.Get(key) + if err == nil { + return chunk, nil + } + if err != ErrFetching { + return nil, err + } + } else { + var created bool + chunk, created = self.localStore.GetOrCreateRequest(key) + if chunk.ReqC == nil { + return chunk, nil + } -//create params with default values -func NewDefaultStoreParams() (self *StoreParams) { - return &StoreParams{ - DbCapacity: defaultDbCapacity, - CacheCapacity: defaultCacheCapacity, - Radius: defaultRadius, + if created { + if err := self.retrieve(chunk); err != nil { + return nil, err + } + } } + + t := time.NewTicker(searchTimeout) + defer t.Stop() + + select { + case <-t.C: + return nil, ErrNotFound + case <-chunk.ReqC: + } + return chunk, nil } //this can only finally be set after all config options (file, cmd line, env vars) @@ -74,70 +78,10 @@ func (self *StoreParams) Init(path string) { } } -// netstore contructor, takes path argument that is used to initialise dbStore, -// the persistent (disk) storage component of LocalStore -// the second argument is the hive, the connection/logistics manager for the node -func NewNetStore(hash SwarmHasher, lstore *LocalStore, cloud CloudStore, params *StoreParams) *NetStore { - return &NetStore{ - hashfunc: hash, - localStore: lstore, - cloud: cloud, - } +// Put is the entrypoint for local store requests coming from storeLoop +func (self *NetStore) Put(chunk *Chunk) { + self.localStore.Put(chunk) } -const ( - // maximum number of peers that a retrieved message is delivered to - requesterCount = 3 -) - -var ( - // timeout interval before retrieval is timed out - searchTimeout = 3 * time.Second -) - -// store logic common to local and network chunk store requests -// ~ unsafe put in localdb no check if exists no extra copy no hash validation -// the chunk is forced to propagate (Cloud.Store) even if locally found! -// caller needs to make sure if that is wanted -func (self *NetStore) Put(entry *Chunk) { - self.localStore.Put(entry) - - // handle deliveries - if entry.Req != nil { - log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v hit existing request...delivering", entry.Key.Log())) - // closing C signals to other routines (local requests) - // that the chunk is has been retrieved - close(entry.Req.C) - // deliver the chunk to requesters upstream - go self.cloud.Deliver(entry) - } else { - log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())) - // handle propagating store requests - // go self.cloud.Store(entry) - go self.cloud.Store(entry) - } -} - -// retrieve logic common for local and network chunk retrieval requests -func (self *NetStore) Get(key Key) (*Chunk, error) { - var err error - chunk, err := self.localStore.Get(key) - if err == nil { - if chunk.Req == nil { - log.Trace(fmt.Sprintf("NetStore.Get: %v found locally", key)) - } else { - log.Trace(fmt.Sprintf("NetStore.Get: %v hit on an existing request", key)) - // no need to launch again - } - return chunk, err - } - // no data and no request status - log.Trace(fmt.Sprintf("NetStore.Get: %v not found locally. open new request", key)) - chunk = NewChunk(key, newRequestStatus(key)) - self.localStore.memStore.Put(chunk) - go self.cloud.Retrieve(chunk) - return chunk, nil -} - -// Close netstore +// Close chunk store func (self *NetStore) Close() {} diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 19d493405a..a34b73993f 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -136,10 +136,11 @@ func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) { return } -func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader { +func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader { return &LazyChunkReader{ key: key, chunkC: chunkC, + depth: depth, chunkSize: self.chunkSize, branches: self.branches, hashSize: self.hashSize, @@ -164,16 +165,18 @@ func (self *PyramidChunker) decrementWorkerCount() { self.workerCount -= 1 } -func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) { +func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) (k Key, wait func(), err error) { jobC := make(chan *chunkJob, 2*ChunkProcessors) wg := &sync.WaitGroup{} + storageWG := &sync.WaitGroup{} + storageWG.Add(1) errC := make(chan error) quitC := make(chan bool) rootKey := make([]byte, self.hashSize) chunkLevel := make([][]*TreeEntry, self.branches) wg.Add(1) - go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG) + self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -181,10 +184,6 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk // waiting for all chunks to finish wg.Wait() - // if storage waitgroup is non-nil, we wait for storage to finish too - if storageWG != nil { - storageWG.Wait() - } //We close errC here because this is passed down to 8 parallel routines underneath. // if a error happens in one of them.. that particular routine raises error... // once they all complete successfully, the control comes back and we can safely close this here. @@ -196,15 +195,15 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk select { case err := <-errC: if err != nil { - return nil, err + return nil, nil, err } case <-time.NewTimer(splitTimeout).C: } - return rootKey, nil + return rootKey, storageWG.Wait, nil } -func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) { +func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (k Key, wait func(), err error) { quitC := make(chan bool) rootKey := make([]byte, self.hashSize) chunkLevel := make([][]*TreeEntry, self.branches) @@ -216,8 +215,11 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, wg := &sync.WaitGroup{} errC := make(chan error) + storageWG := &sync.WaitGroup{} + storageWG.Add(1) + wg.Add(1) - go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG) + self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -225,10 +227,6 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, // waiting for all chunks to finish wg.Wait() - // if storage waitgroup is non-nil, we wait for storage to finish too - if storageWG != nil { - storageWG.Wait() - } close(errC) }() @@ -237,21 +235,18 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, select { case err := <-errC: if err != nil { - return nil, err + return nil, nil, err } case <-time.NewTimer(splitTimeout).C: } - return rootKey, nil + return rootKey, storageWG.Wait, nil } -func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) { +func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storageWG *sync.WaitGroup) { defer self.decrementWorkerCount() - + defer storageWG.Done() hasher := self.hashFunc() - if wwg != nil { - defer wwg.Done() - } for { select { @@ -259,38 +254,35 @@ func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan if !ok { return } - self.processChunk(id, hasher, job, chunkC, swg) + self.processChunk(id, hasher, job, chunkC, storageWG) case <-quitC: return } } } -func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, swg *sync.WaitGroup) { +func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, storageWG *sync.WaitGroup) { hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length hasher.Write(job.chunk[8:]) // minus 8 []byte length h := hasher.Sum(nil) - newChunk := &Chunk{ - Key: h, - SData: job.chunk, - Size: job.size, - wg: swg, - } + newChunk := NewChunk(h, nil) + newChunk.SData = job.chunk + newChunk.Size = job.size // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) copy(job.key, h) // send off new chunk to storage - if chunkC != nil { - if swg != nil { - swg.Add(1) - } - } job.parentWg.Done() if chunkC != nil { chunkC <- newChunk + storageWG.Add(1) + go func() { + defer storageWG.Done() + <-newChunk.dbStored + }() } } @@ -374,19 +366,15 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC return nil } -func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, processorWG *sync.WaitGroup, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) { +func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) { defer wg.Done() chunkWG := &sync.WaitGroup{} totalDataSize := 0 - // processorWG keeps track of workers spawned for hashing chunks - if processorWG != nil { - processorWG.Add(1) - } - self.incrementWorkerCount() - go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG) + + go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG) parent := NewTreeEntry(self) var unFinishedChunk *Chunk @@ -426,14 +414,24 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt var n int var err error chunkData := make([]byte, self.chunkSize+8) + maxBuf := len(chunkData) + readBytes := 8 if unFinishedChunk != nil { copy(chunkData, unFinishedChunk.SData) - n, err = data.Read(chunkData[8+unFinishedChunk.Size:]) - n += int(unFinishedChunk.Size) - unFinishedChunk = nil - } else { - n, err = data.Read(chunkData[8:]) + readBytes += int(unFinishedChunk.Size) } + for readBytes < maxBuf { + n0, err0 := data.Read(chunkData[readBytes:]) + readBytes += n0 + n += n0 + if err0 != nil { + if err0 != io.EOF || (n0 == 0 && maxBuf == readBytes) || n == 0 || n0 != 0 { + err = err0 + } + break + } + } + unFinishedChunk = nil totalDataSize += n if err != nil { @@ -486,11 +484,9 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt workers := self.getWorkerCount() if int64(len(jobC)) > workers && workers < ChunkProcessors { - if processorWG != nil { - processorWG.Add(1) - } self.incrementWorkerCount() - go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG) + storageWG.Add(1) + go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG) } } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 03235fa65a..7357323213 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -140,24 +140,9 @@ type ResourceHandler struct { } // Create or open resource update chunk store -// -// If validator is nil, signature and access validation will be deactivated -func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) { - - hashfunc := MakeHashFunc(SHA3Hash) - - path := filepath.Join(datadir, DbDirName) - dbStore, err := NewDbStore(datadir, hashfunc, singletonSwarmDbCapacity, 0) - if err != nil { - return nil, err - } - localStore := &LocalStore{ - memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), - DbStore: dbStore, - } - +func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) { rh := &ResourceHandler{ - ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), + ChunkStore: chunkStore, ethClient: ethClient, resources: make(map[string]*resource), validator: validator, @@ -734,10 +719,10 @@ type resourceChunkStore struct { chunkSize int64 } -func newResourceChunkStore(path string, hasher SwarmHasher, localStore *LocalStore, cloudStore CloudStore) *resourceChunkStore { +func NewResourceChunkStore(localStore *LocalStore, request func(*Chunk) error) ChunkStore { return &resourceChunkStore{ localStore: localStore, - netStore: NewNetStore(hasher, localStore, cloudStore, NewDefaultStoreParams()), + netStore: NewNetStore(localStore, request), } } @@ -749,23 +734,23 @@ func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { // if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing. // sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue // that caused the failure or that the chunk doesn't exist. - if chunk.Req == nil { + if chunk.ReqC == nil { return chunk, nil } t := time.NewTimer(time.Second * 1) select { case <-t.C: - return nil, errors.New("timeout") - case <-chunk.Req.C: - log.Trace("Received resource update chunk", "peer", chunk.Req.Source) + log.Trace("Timeout on resource chunk store") + return nil, fmt.Errorf("timeout") + case <-chunk.C: + log.Trace("Received resource update chunk") } return chunk, nil } func (r *resourceChunkStore) Put(chunk *Chunk) { - chunk.wg = &sync.WaitGroup{} r.netStore.Put(chunk) - chunk.wg.Wait() + chunk.WaitToStore() } func (r *resourceChunkStore) Close() { @@ -804,3 +789,20 @@ func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { hasher.Write(data) return common.BytesToHash(hasher.Sum(nil)) } + +// TODO: this should not be exposed, but swarm/testutil/http.go needs it +func NewTestResourceHandler(datadir string, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) { + path := filepath.Join(datadir, DbDirName) + basekey := make([]byte, 32) + hasher := MakeHashFunc(SHA3Hash) + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } + resourceChunkStore := NewResourceChunkStore(localStore, nil) + return NewResourceHandler(hasher, resourceChunkStore, ethClient, validator) +} diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 376b8397ae..3f6f1cd3dd 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -220,7 +220,7 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourceFrequency * 3) fwdBlocks(int(resourceFrequency*2)-1, backend) - rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil) + rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil) _, err = rh2.LookupLatestByName(ctx, safeName, true) if err != nil { t.Fatal(err) @@ -351,7 +351,7 @@ func setupTest(backend ethApi, validator ResourceValidator) (rh *ResourceHandler os.RemoveAll(datadir) } - rh, err = NewResourceHandler(datadir, &testCloudStore{}, backend, validator) + rh, err = NewTestResourceHandler(datadir, backend, validator) return rh, datadir, signer, cleanF, nil } @@ -417,18 +417,6 @@ func newTestSigner() (*testSigner, error) { }, nil } -type testCloudStore struct { -} - -func (c *testCloudStore) Store(*Chunk) { -} - -func (c *testCloudStore) Deliver(*Chunk) { -} - -func (c *testCloudStore) Retrieve(*Chunk) { -} - // Default fallthrough validation of mutable resource ownership type testValidator struct { *baseValidator diff --git a/swarm/storage/types.go b/swarm/storage/types.go index d35f1f9294..2e6f6d7d47 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -19,16 +19,19 @@ package storage import ( "bytes" "crypto" + "crypto/rand" + "encoding/binary" "fmt" "hash" "io" - "sync" "github.com/ethereum/go-ethereum/bmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" ) +const MaxPO = 7 + type Hasher func() hash.Hash type SwarmHasher func() SwarmHash @@ -73,6 +76,26 @@ func (h Key) bits(i, j uint) uint { return res } +func Proximity(one, other []byte) (ret int) { + b := (MaxPO-1)/8 + 1 + if b > len(one) { + b = len(one) + } + m := 8 + for i := 0; i < b; i++ { + oxo := one[i] ^ other[i] + if i == b-1 { + m = MaxPO % 8 + } + for j := 0; j < m; j++ { + if (oxo>>uint8(7-j))&0x01 != 0 { + return i*8 + j + } + } + } + return MaxPO +} + func IsZeroKey(key Key) bool { return len(key) == 0 || bytes.Equal(key, ZeroKey) } @@ -100,10 +123,10 @@ func (key Key) Hex() string { } func (key Key) Log() string { - if len(key[:]) < 4 { + if len(key[:]) < 8 { return fmt.Sprintf("%x", []byte(key[:])) } - return fmt.Sprintf("%08x", []byte(key[:4])) + return fmt.Sprintf("%016x", []byte(key[:8])) } func (key Key) String() string { @@ -122,25 +145,22 @@ func (key *Key) UnmarshalJSON(value []byte) error { return nil } -// each chunk when first requested opens a record associated with the request -// next time a request for the same chunk arrives, this record is updated -// this request status keeps track of the request ID-s as well as the requesting -// peers and has a channel that is closed when the chunk is retrieved. Multiple -// local callers can wait on this channel (or combined with a timeout, block with a -// select). -type RequestStatus struct { - Key Key - Source Peer - C chan bool - Requesters map[uint64][]interface{} +type KeyCollection []Key + +func NewKeyCollection(l int) KeyCollection { + return make(KeyCollection, l) } -func newRequestStatus(key Key) *RequestStatus { - return &RequestStatus{ - Key: key, - Requesters: make(map[uint64][]interface{}), - C: make(chan bool), - } +func (c KeyCollection) Len() int { + return len(c) +} + +func (c KeyCollection) Less(i, j int) bool { + return bytes.Compare(c[i], c[j]) == -1 +} + +func (c KeyCollection) Swap(i, j int) { + c[i], c[j] = c[j], c[i] } // Chunk also serves as a request object passed to ChunkStores @@ -149,18 +169,47 @@ func newRequestStatus(key Key) *RequestStatus { // but the size of the subtree encoded in the chunk // 0 if request, to be supplied by the dpa type Chunk struct { - Key Key // always - SData []byte // nil if request, to be supplied by dpa - Size int64 // size of the data covered by the subtree encoded in this chunk - Source Peer // peer - C chan bool // to signal data delivery by the dpa - Req *RequestStatus // request Status needed by netStore - wg *sync.WaitGroup // wg to synchronize - dbStored chan bool // never remove a chunk from memStore before it is written to dbStore + Key Key // always + SData []byte // nil if request, to be supplied by dpa + Size int64 // size of the data covered by the subtree encoded in this chunk + //Source Peer // peer + C chan bool // to signal data delivery by the dpa + ReqC chan bool // to signal the request done + dbStored chan bool // never remove a chunk from memStore before it is written to dbStore } -func NewChunk(key Key, rs *RequestStatus) *Chunk { - return &Chunk{Key: key, Req: rs} +func NewChunk(key Key, reqC chan bool) *Chunk { + return &Chunk{Key: key, ReqC: reqC, dbStored: make(chan bool)} +} + +func (c *Chunk) WaitToStore() { + <-c.dbStored +} + +func FakeChunk(size int64, count int, chunks []*Chunk) int { + var i int + hasher := MakeHashFunc(SHA3Hash)() + chunksize := getDefaultChunkSize() + if size > chunksize { + size = chunksize + } + + for i = 0; i < count; i++ { + hasher.Reset() + chunks[i].SData = make([]byte, size) + rand.Read(chunks[i].SData) + binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(size)) + hasher.Write(chunks[i].SData) + chunks[i].Key = make([]byte, 32) + copy(chunks[i].Key, hasher.Sum(nil)) + } + + return i +} + +func getDefaultChunkSize() int64 { + return DefaultBranches * int64(MakeHashFunc(SHA3Hash)().Size()) + } /* @@ -198,14 +247,14 @@ type Splitter interface { The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel. A closed error signals process completion at which point the key can be considered final if there were no errors. */ - Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error) + Split(io.Reader, int64, chan *Chunk) (Key, func(), error) /* This is the first step in making files mutable (not chunks).. Append allows adding more data chunks to the end of the already existsing file. The key for the root chunk is supplied to load the respective tree. Rest of the parameters behave like Split. */ - Append(Key, io.Reader, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error) + Append(Key, io.Reader, chan *Chunk) (Key, func(), error) } type Joiner interface { @@ -221,7 +270,7 @@ type Joiner interface { The chunks are not meant to be validated by the chunker when joining. This is because it is left to the DPA to decide which sources are trusted. */ - Join(key Key, chunkC chan *Chunk) LazySectionReader + Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader } type Chunker interface { diff --git a/swarm/swarm.go b/swarm/swarm.go index 7352f9020e..0ec85adadd 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -22,7 +22,6 @@ import ( "crypto/ecdsa" "fmt" "net" - "path/filepath" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -40,6 +39,7 @@ import ( httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/fuse" "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/stream" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/mock" @@ -51,16 +51,19 @@ type Swarm struct { api *api.Api // high level api layer (fs/manifest) dns api.Resolver // DNS registrar //dbAccess *network.DbAccess // access to local chunk db iterator and storage counter - storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends - dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support + //storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends + dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support //depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage - cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) - bzz *network.Bzz // the logistic manager - backend chequebook.Backend // simple blockchain Backend - privateKey *ecdsa.PrivateKey - lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped - sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit - ps *pss.Pss + streamer *stream.Registry + //cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) + bzz *network.Bzz // the logistic manager + backend chequebook.Backend // simple blockchain Backend + privateKey *ecdsa.PrivateKey + corsString string + swapEnabled bool + lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit + ps *pss.Pss } type SwarmAPI struct { @@ -98,7 +101,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e log.Debug(fmt.Sprintf("Setting up Swarm service components")) hash := storage.MakeHashFunc(config.ChunkerParams.Hash) - self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, mockStore) + self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.Hex2Bytes(config.BzzKey), mockStore) if err != nil { return } @@ -115,8 +118,8 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e config.HiveParams.Discovery = true // setup cloud storage internal access layer - self.cloud = &storage.Forwarder{} - self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams) + //self.cloud = &storage.Forwarder{} + //self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams) log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store")) nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey))) addr := network.NewAddrFromNodeID(nodeid) @@ -125,10 +128,17 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e UnderlayAddr: addr.UAddr, HiveParams: config.HiveParams, } + + db := storage.NewDBAPI(self.lstore) + delivery := stream.NewDelivery(to, db) + self.streamer = stream.NewRegistry(addr, delivery, self.lstore, false) + stream.RegisterSwarmSyncerServer(self.streamer, db) + stream.RegisterSwarmSyncerClient(self.streamer, db) + self.bzz = network.NewBzz(bzzconfig, to, nil) // set up DPA, the cloud storage local access layer - dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) + dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve) log.Debug(fmt.Sprintf("-> Local Access to Swarm")) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) @@ -166,7 +176,9 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e return nil, err } } - resourceHandler, err = storage.NewResourceHandler(filepath.Join(self.config.Path, storage.DbDirName), self.cloud, ensClient, resourceValidator) + hashfunc := storage.MakeHashFunc(storage.SHA3Hash) + chunkStore := storage.NewResourceChunkStore(self.lstore, func(*storage.Chunk) error { return nil }) + resourceHandler, err = storage.NewResourceHandler(hashfunc, chunkStore, ensClient, resourceValidator) if err != nil { return nil, err } @@ -267,12 +279,14 @@ func (self *Swarm) Stop() error { // implements the node.Service interface func (self *Swarm) Protocols() (protos []p2p.Protocol) { - protos = append(protos, self.bzz.Protocols()...) if self.ps != nil { protos = append(protos, self.ps.Protocols()...) } + if self.streamer != nil { + protos = append(protos, self.streamer.Protocols()...) + } return } @@ -285,7 +299,7 @@ func (self *Swarm) RegisterPssProtocol(spec *protocols.Spec, targetprotocol *p2p } // implements node.Service -// Apis returns the RPC Api descriptors the Swarm implementation offers +// APIs returns the RPC Api descriptors the Swarm implementation offers func (self *Swarm) APIs() []rpc.API { apis := []rpc.API{ @@ -355,32 +369,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error { return nil } -// Local swarm without netStore -func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { - - prvKey, err := crypto.GenerateKey() - if err != nil { - return - } - - config := api.NewConfig() - config.Path = datadir - config.Port = port - config.Init(prvKey) - - dpa, err := storage.NewLocalDPA(datadir) - if err != nil { - return - } - - self = &Swarm{ - api: api.NewApi(dpa, nil, nil), - config: config, - } - - return -} - // serialisable info about swarm type Info struct { *api.Config diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 77e3386fd2..451c07309f 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -51,9 +51,8 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { ChunkDbPath: dir, DbCapacity: 5000000, CacheCapacity: 5000, - Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, nil) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, make([]byte, 32), nil) if err != nil { os.RemoveAll(dir) t.Fatal(err) @@ -71,7 +70,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { t.Fatal(err) } - rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, &fakeBackend{}, nil) + rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil) if err != nil { t.Fatal(err) } @@ -104,15 +103,3 @@ type TestSwarmServer struct { func (t *TestSwarmServer) Close() { t.cleanup() } - -type testCloudStore struct { -} - -func (c *testCloudStore) Store(*storage.Chunk) { -} - -func (c *testCloudStore) Deliver(*storage.Chunk) { -} - -func (c *testCloudStore) Retrieve(*storage.Chunk) { -} diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index 2248a1b129..a2b332a2e0 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -162,8 +162,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) - path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) + path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go deleted file mode 100644 index 0edd3782f5..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,go1.10 - -package notify - -const refZero = 0 diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go deleted file mode 100644 index b81c3c1859..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,cgo,!go1.10 - -package notify - -/* -#include -*/ -import "C" - -var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/vendor.json b/vendor/vendor.json index 675744d773..387afbb763 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -322,10 +322,10 @@ "revisionTime": "2016-11-28T21:05:44Z" }, { - "checksumSHA1": "1ESHllhZOIBg7MnlGHUdhz047bI=", + "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", "path": "github.com/rjeczalik/notify", - "revision": "27b537f07230b3f917421af6dcf044038dbe57e2", - "revisionTime": "2018-01-03T13:19:05Z" + "revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9", + "revisionTime": "2018-02-03T14:01:15Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index a3d6d5db5c..ad4d64cbf0 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -93,7 +93,7 @@ var masterBloomFilter []byte var masterPow = 0.00000001 var round = 1 -func TestSimulation(t *testing.T) { +func XTestSimulation(t *testing.T) { // create a chain of whisper nodes, // installs the filters with shared (predefined) parameters initialize(t)