diff --git a/.travis.yml b/.travis.yml index ba62b87bf5..2b529816a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum sudo: false matrix: include: - - os: linux - dist: trusty - sudo: required - go: 1.7.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage - - os: linux dist: trusty sudo: required @@ -25,7 +14,6 @@ matrix: - go run build/ci.go install - go run build/ci.go test -coverage - # These are the latest Go versions. - os: linux dist: trusty sudo: required @@ -47,6 +35,28 @@ matrix: - go run build/ci.go install - go run build/ci.go test -coverage + # These are the latest Go versions. + - os: linux + dist: trusty + sudo: required + go: "1.10" + script: + - sudo modprobe fuse + - sudo chmod 666 /dev/fuse + - sudo chown root:$USER /etc/fuse.conf + - go run build/ci.go install + - go run build/ci.go test -coverage + + - os: osx + go: "1.10" + script: + - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 + - brew update + - brew install caskroom/cask/brew-cask + - brew cask install osxfuse + - go run build/ci.go install + - go run build/ci.go test -coverage + # This builder only tests code linters on latest version of Go - os: linux dist: trusty @@ -185,6 +195,8 @@ matrix: - xctool -version - xcrun simctl list + # Workaround for https://github.com/golang/go/issues/23749 + - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc' - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds # This builder does the Azure archive purges to avoid accumulating junk diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index b7555d0815..6b6b28e657 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -281,8 +281,8 @@ func TestDeposit(t *testing.T) { t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } - // autodeposit every 30ms if new cheque issued - interval := 30 * time.Millisecond + // autodeposit every 200ms if new cheque issued + interval := 200 * time.Millisecond chbook.AutoDeposit(interval, common.Big1, balance) _, err = chbook.Issue(addr1, amount) if err != nil { diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index 9bb899384b..22acb9f863 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -2307,7 +2307,7 @@ var toChecksumAddress = function (address) { }; /** - * Transforms given string to valid 20 bytes-length addres with 0x prefix + * Transforms given string to valid 20 bytes-length address with 0x prefix * * @method toAddress * @param {String} address diff --git a/log/logger.go b/log/logger.go index 15c83a9b25..e2805271a7 100644 --- a/log/logger.go +++ b/log/logger.go @@ -126,13 +126,13 @@ type logger struct { h *swapHandler } -func (l *logger) write(msg string, lvl Lvl, ctx []interface{}) { +func (l *logger) write(msg string, lvl Lvl, ctx []interface{}, skip int) { l.h.Log(&Record{ Time: time.Now(), Lvl: lvl, Msg: msg, Ctx: newContext(l.ctx, ctx), - Call: stack.Caller(2), + Call: stack.Caller(skip), KeyNames: RecordKeyNames{ Time: timeKey, Msg: msgKey, @@ -156,27 +156,27 @@ func newContext(prefix []interface{}, suffix []interface{}) []interface{} { } func (l *logger) Trace(msg string, ctx ...interface{}) { - l.write(msg, LvlTrace, ctx) + l.write(msg, LvlTrace, ctx, 2) } func (l *logger) Debug(msg string, ctx ...interface{}) { - l.write(msg, LvlDebug, ctx) + l.write(msg, LvlDebug, ctx, 2) } func (l *logger) Info(msg string, ctx ...interface{}) { - l.write(msg, LvlInfo, ctx) + l.write(msg, LvlInfo, ctx, 2) } func (l *logger) Warn(msg string, ctx ...interface{}) { - l.write(msg, LvlWarn, ctx) + l.write(msg, LvlWarn, ctx, 2) } func (l *logger) Error(msg string, ctx ...interface{}) { - l.write(msg, LvlError, ctx) + l.write(msg, LvlError, ctx, 2) } func (l *logger) Crit(msg string, ctx ...interface{}) { - l.write(msg, LvlCrit, ctx) + l.write(msg, LvlCrit, ctx, 2) os.Exit(1) } diff --git a/log/root.go b/log/root.go index 71b8cef6d4..dd24c05e32 100644 --- a/log/root.go +++ b/log/root.go @@ -31,31 +31,36 @@ func Root() Logger { // Trace is a convenient alias for Root().Trace func Trace(msg string, ctx ...interface{}) { - root.write(msg, LvlTrace, ctx) + root.write(msg, LvlTrace, ctx, 2) } // Debug is a convenient alias for Root().Debug func Debug(msg string, ctx ...interface{}) { - root.write(msg, LvlDebug, ctx) + root.write(msg, LvlDebug, ctx, 2) } // Info is a convenient alias for Root().Info func Info(msg string, ctx ...interface{}) { - root.write(msg, LvlInfo, ctx) + root.write(msg, LvlInfo, ctx, 2) } // Warn is a convenient alias for Root().Warn func Warn(msg string, ctx ...interface{}) { - root.write(msg, LvlWarn, ctx) + root.write(msg, LvlWarn, ctx, 2) } // Error is a convenient alias for Root().Error func Error(msg string, ctx ...interface{}) { - root.write(msg, LvlError, ctx) + root.write(msg, LvlError, ctx, 2) } // Crit is a convenient alias for Root().Crit func Crit(msg string, ctx ...interface{}) { - root.write(msg, LvlCrit, ctx) + root.write(msg, LvlCrit, ctx, 2) os.Exit(1) } + +// Output is a convenient alias for write +func Output(msg string, lvl Lvl, skip int, ctx ...interface{}) { + root.write(msg, lvl, ctx, skip) +} diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index a4641ed8bd..1a5c480386 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -365,14 +365,14 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { } -func TestMultiplePeersDropSelf(t *testing.T) { +func XTestMultiplePeersDropSelf(t *testing.T) { runMultiplePeers(t, 0, fmt.Errorf("subprotocol error"), fmt.Errorf("Message handler error: (msg code 3): dropped"), ) } -func TestMultiplePeersDropOther(t *testing.T) { +func XTestMultiplePeersDropOther(t *testing.T) { runMultiplePeers(t, 1, fmt.Errorf("Message handler error: (msg code 3): dropped"), fmt.Errorf("subprotocol error"), diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 6ecacd87a7..8752d04458 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -34,11 +34,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -const ( - socketReadBuffer = 5000 * 1024 - socketWriteBuffer = 5000 * 1024 -) - // SimAdapter is a NodeAdapter which creates in-memory simulation nodes and // connects them using net.Pipe or OS socket connections type SimAdapter struct { @@ -112,7 +107,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { MaxPeers: math.MaxInt32, NoDiscovery: true, Dialer: s, - EnableMsgEvents: false, + EnableMsgEvents: config.EnableMsgEvents, }, NoUSB: true, Logger: log.New("node.id", id.String()), @@ -378,20 +373,10 @@ func socketPipe() (net.Conn, net.Conn, error) { return nil, nil, err } - err = setSocketBuffer(pipe1) - if err != nil { - return nil, nil, err - } - - err = setSocketBuffer(pipe2) - if err != nil { - return nil, nil, err - } - return pipe1, pipe2, nil } -func setSocketBuffer(conn net.Conn) error { +func setSocketBuffer(conn net.Conn, socketReadBuffer int, socketWriteBuffer int) error { switch v := conn.(type) { case *net.UnixConn: err := v.SetReadBuffer(socketReadBuffer) diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 3939706aec..b1ef7add0b 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -27,7 +27,7 @@ import ( func TestSocketPipe(t *testing.T) { c1, c2, err := socketPipe() if err != nil { - t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err) + t.Fatal(err) } done := make(chan struct{}) @@ -35,15 +35,19 @@ func TestSocketPipe(t *testing.T) { go func() { msgs := 20 size := 8 - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - _ = binary.PutUvarint(msg, uint64(i)) - _, err := c1.Write(msg) - if err != nil { - t.Fatal(err) + // OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously + go func() { + for i := 0; i < msgs; i++ { + msg := make([]byte, size) + _ = binary.PutUvarint(msg, uint64(i)) + + _, err := c1.Write(msg) + if err != nil { + t.Fatal(err) + } } - } + }() for i := 0; i < msgs; i++ { msg := make([]byte, size) @@ -64,7 +68,7 @@ func TestSocketPipe(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -72,7 +76,7 @@ func TestSocketPipe(t *testing.T) { func TestSocketPipeBidirections(t *testing.T) { c1, c2, err := socketPipe() if err != nil { - t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err) + t.Fatal(err) } done := make(chan struct{}) @@ -80,14 +84,18 @@ func TestSocketPipeBidirections(t *testing.T) { go func() { msgs := 100 size := 4 - for i := 0; i < msgs; i++ { - msg := []byte(`ping`) - _, err := c1.Write(msg) - if err != nil { - t.Fatal(err) + // OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously + go func() { + for i := 0; i < msgs; i++ { + msg := []byte(`ping`) + + _, err := c1.Write(msg) + if err != nil { + t.Fatal(err) + } } - } + }() for i := 0; i < msgs; i++ { out := make([]byte, size) @@ -124,7 +132,7 @@ func TestSocketPipeBidirections(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -169,7 +177,7 @@ func TestTcpPipe(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -232,7 +240,7 @@ func TestTcpPipeBidirections(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -281,7 +289,7 @@ func TestNetPipe(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } @@ -356,7 +364,7 @@ func TestNetPipeBidirections(t *testing.T) { select { case <-done: - case <-time.After(1 * time.Second): + case <-time.After(5 * time.Second): t.Fatal("test timeout") } } diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 2169d68308..2c4b9dd8f2 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -105,21 +105,23 @@ type NodeConfig struct { // nodeConfigJSON is used to encode and decode NodeConfig as JSON by encoding // all fields as strings type nodeConfigJSON struct { - ID string `json:"id"` - PrivateKey string `json:"private_key"` - Name string `json:"name"` - Services []string `json:"services"` - Port uint16 `json:"port"` + ID string `json:"id"` + PrivateKey string `json:"private_key"` + Name string `json:"name"` + Services []string `json:"services"` + EnableMsgEvents bool `json:"enable_msg_events"` + Port uint16 `json:"port"` } // MarshalJSON implements the json.Marshaler interface by encoding the config // fields as strings func (n *NodeConfig) MarshalJSON() ([]byte, error) { confJSON := nodeConfigJSON{ - ID: n.ID.String(), - Name: n.Name, - Services: n.Services, - Port: n.Port, + ID: n.ID.String(), + Name: n.Name, + Services: n.Services, + Port: n.Port, + EnableMsgEvents: n.EnableMsgEvents, } if n.PrivateKey != nil { confJSON.PrivateKey = hex.EncodeToString(crypto.FromECDSA(n.PrivateKey)) @@ -158,6 +160,7 @@ func (n *NodeConfig) UnmarshalJSON(data []byte) error { n.Name = confJSON.Name n.Services = confJSON.Services n.Port = confJSON.Port + n.EnableMsgEvents = confJSON.EnableMsgEvents return nil } @@ -176,9 +179,11 @@ func RandomNodeConfig() *NodeConfig { panic("unable to assign tcp port") } return &NodeConfig{ - ID: id, - PrivateKey: key, - Port: port, + ID: id, + Name: fmt.Sprintf("node_%s", id.String()), + PrivateKey: key, + Port: port, + EnableMsgEvents: true, } } diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index 97dd742e88..24001f1949 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -561,7 +561,8 @@ func (s *Server) LoadSnapshot(w http.ResponseWriter, req *http.Request) { // CreateNode creates a node in the network using the given configuration func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) { - config := adapters.RandomNodeConfig() + config := &adapters.NodeConfig{} + err := json.NewDecoder(req.Body).Decode(config) if err != nil && err != io.EOF { http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 677a8fb147..732d49f546 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -348,7 +348,8 @@ func startTestNetwork(t *testing.T, client *Client) []string { nodeCount := 2 nodeIDs := make([]string, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := client.CreateNode(nil) + config := adapters.RandomNodeConfig() + node, err := client.CreateNode(config) if err != nil { t.Fatalf("error creating node: %s", err) } @@ -527,7 +528,9 @@ func TestHTTPNodeRPC(t *testing.T) { // start a node in the network client := NewClient(s.URL) - node, err := client.CreateNode(nil) + + config := adapters.RandomNodeConfig() + node, err := client.CreateNode(config) if err != nil { t.Fatalf("error creating node: %s", err) } @@ -589,7 +592,8 @@ func TestHTTPSnapshot(t *testing.T) { nodeCount := 2 nodes := make([]*p2p.NodeInfo, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := client.CreateNode(nil) + config := adapters.RandomNodeConfig() + node, err := client.CreateNode(config) if err != nil { t.Fatalf("error creating node: %s", err) } diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go index c38e288552..b370fe2cd2 100644 --- a/p2p/simulations/mocker.go +++ b/p2p/simulations/mocker.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" ) //a map of mocker names to its function @@ -165,7 +166,8 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) { func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error) { ids := make([]discover.NodeID, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := net.NewNode() + conf := adapters.RandomNodeConfig() + node, err := net.NewNodeWithConfig(conf) if err != nil { log.Error("Error creating a node! %s", err) return nil, err diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index caf428ece1..6919da1cd5 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -78,26 +78,12 @@ func (self *Network) Events() *event.Feed { return &self.events } -// NewNode adds a new node to the network with a random ID -func (self *Network) NewNode() (*Node, error) { - conf := adapters.RandomNodeConfig() - conf.Services = []string{self.DefaultService} - return self.NewNodeWithConfig(conf) -} - // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { self.lock.Lock() defer self.lock.Unlock() - // create a random ID and PrivateKey if not set - if conf.ID == (discover.NodeID{}) { - c := adapters.RandomNodeConfig() - conf.ID = c.ID - conf.PrivateKey = c.PrivateKey - } - id := conf.ID if conf.Reachable == nil { conf.Reachable = func(otherID discover.NodeID) bool { _, err := self.InitConn(conf.ID, otherID) @@ -105,14 +91,9 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) } } - // assign a name to the node if not set - if conf.Name == "" { - conf.Name = fmt.Sprintf("node%02d", len(self.Nodes)+1) - } - // check the node doesn't already exist - if node := self.getNode(id); node != nil { - return nil, fmt.Errorf("node with ID %q already exists", id) + if node := self.getNode(conf.ID); node != nil { + return nil, fmt.Errorf("node with ID %q already exists", conf.ID) } if node := self.getNodeByName(conf.Name); node != nil { return nil, fmt.Errorf("node with name %q already exists", conf.Name) @@ -132,8 +113,8 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Node: adapterNode, Config: conf, } - log.Trace(fmt.Sprintf("node %v created", id)) - self.nodeMap[id] = len(self.Nodes) + log.Trace(fmt.Sprintf("node %v created", conf.ID)) + self.nodeMap[conf.ID] = len(self.Nodes) self.Nodes = append(self.Nodes, node) // emit a "control" event diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index 2a062121be..f178bac502 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -41,7 +41,8 @@ func TestNetworkSimulation(t *testing.T) { nodeCount := 20 ids := make([]discover.NodeID, nodeCount) for i := 0; i < nodeCount; i++ { - node, err := network.NewNode() + conf := adapters.RandomNodeConfig() + node, err := network.NewNodeWithConfig(conf) if err != nil { t.Fatalf("error creating node: %s", err) } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index b707ef5c3c..76b2751a58 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -109,10 +109,11 @@ func TestApiPut(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - key, _, err := api.Put(content, exp.MimeType) + key, wait, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } + wait() resp := testGet(t, api, key.Hex(), "") checkResponse(t, resp, exp) }) diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index dbd97182fd..9b9f5c2f96 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -110,7 +110,8 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife //(and return the correct HTTP status code) func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { if code == http.StatusInternalServerError { - log.Error(msg) + //log.Error(msg) + log.Output(msg, log.LvlError, 3) } respond(w, r, &ErrorParams{ Code: code, diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 288ee30622..c8dc02914f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -91,21 +91,21 @@ type Request struct { // body in swarm and returns the resulting storage key as a text/plain response func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { - s.BadRequest(w, r, "raw POST request cannot contain a path") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "raw POST request cannot contain a path"), http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { - s.BadRequest(w, r, "missing Content-Length header in request") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "missing Content-Length header in request"), http.StatusBadRequest) return } key, _, err := s.api.Store(r.Body, r.ContentLength) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } - s.logDebug("content for %s stored", key.Log()) + log.Debug(fmt.Sprintf("content for %s stored", key.Log())) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) @@ -120,7 +120,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { - s.BadRequest(w, r, err.Error()) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, err), http.StatusBadRequest) return } @@ -128,13 +128,13 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { if r.uri.Addr != "" { key, err = s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } } else { key, err = s.api.NewManifest() if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } } @@ -153,7 +153,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } }) if err != nil { - s.Error(w, r, fmt.Errorf("error creating manifest: %s", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error creating manifest: %s", err)), http.StatusInternalServerError) return } @@ -186,12 +186,12 @@ func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error { Size: hdr.Size, ModTime: hdr.ModTime, } - s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + log.Debug(fmt.Sprintf("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)) contentKey, err := mw.AddEntry(tr, entry) if err != nil { return fmt.Errorf("error adding manifest entry from tar stream: %s", err) } - s.logDebug("content for %s stored", contentKey.Log()) + log.Debug(fmt.Sprintf("content for %s stored", contentKey.Log())) } } @@ -243,12 +243,12 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma Size: size, ModTime: time.Now(), } - s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + log.Debug(fmt.Sprintf("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)) contentKey, err := mw.AddEntry(reader, entry) if err != nil { return fmt.Errorf("error adding manifest entry from multipart form: %s", err) } - s.logDebug("content for %s stored", contentKey.Log()) + log.Debug(fmt.Sprintf("content for %s stored", contentKey.Log())) } } @@ -263,7 +263,7 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error if err != nil { return err } - s.logDebug("content for %s stored", key.Log()) + log.Debug(fmt.Sprintf("content for %s stored", key.Log())) return nil } @@ -273,16 +273,16 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { - s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log()) + log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log())) return mw.RemoveEntry(r.uri.Path) }) if err != nil { - s.Error(w, r, fmt.Errorf("error updating manifest: %s", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error updating manifest: %s", err)), http.StatusInternalServerError) return } @@ -296,12 +296,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { - s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, fmt.Sprintf("Cannot parse frequency parameter: %v", err)), http.StatusBadRequest) return } key, err := s.api.ResourceCreate(r.Context(), r.uri.Addr, frequency) if err != nil { - s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Resource creation failed: %v", err)), http.StatusInternalServerError) return } outdata = key.Hex() @@ -309,12 +309,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { data, err := ioutil.ReadAll(r.Body) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } _, _, _, err = s.api.ResourceUpdate(r.Context(), r.uri.Addr, data) if err != nil { - s.Error(w, r, fmt.Errorf("Update resource failed: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Update resource failed: %v", err)), http.StatusInternalServerError) return } @@ -368,11 +368,11 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin } updateKey, data, err = s.api.ResourceLookup(r.Context(), name, uint32(period), uint32(version)) default: - s.BadRequest(w, r, "Invalid mutable resource request") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "Invalid mutable resource request"), http.StatusBadRequest) return } if err != nil { - s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Mutable resource lookup failed: %v", err)), http.StatusInternalServerError) return } log.Debug("Found update", "key", updateKey) @@ -388,7 +388,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } @@ -397,7 +397,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { walker, err := s.api.NewManifestWalker(key, nil) if err != nil { - s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key)) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, fmt.Sprintf("%s is not a manifest", key)), http.StatusBadRequest) return } var entry *api.ManifestEntry @@ -425,7 +425,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - s.NotFound(w, r, errors.New("Manifest entry could not be loaded")) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Manifest entry could not be loaded")), http.StatusNotFound) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) @@ -434,7 +434,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { - s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Root chunk not found %s: %s", key, err)), http.StatusNotFound) return } @@ -460,19 +460,19 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // contained in the manifest func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { - s.BadRequest(w, r, "files request cannot contain a path") + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "files request cannot contain a path"), http.StatusBadRequest) return } key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } walker, err := s.api.NewManifestWalker(key, nil) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } @@ -519,7 +519,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { return nil }) if err != nil { - s.logError("error generating tar stream: %s", err) + log.Error(fmt.Sprintf("error generating tar stream: %s", err)) } } @@ -535,14 +535,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } list, err := s.getManifestList(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } @@ -559,7 +559,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { List: &list, }) if err != nil { - s.logError("error rendering list HTML: %s", err) + log.Error(fmt.Sprintf("error rendering list HTML: %s", err)) } return } @@ -635,7 +635,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError) return } @@ -643,9 +643,9 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { if err != nil { switch status { case http.StatusNotFound: - s.NotFound(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) default: - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) } return } @@ -656,11 +656,11 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) return } - s.logDebug(fmt.Sprintf("Multiple choices! --> %v", list)) + log.Debug(fmt.Sprintf("Multiple choices! --> %v", list)) //show a nice page links to available entries ShowMultipleChoices(w, &r.Request, list) return @@ -668,7 +668,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { - s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("File not found %s: %s", r.uri, err)), http.StatusNotFound) return } @@ -678,16 +678,16 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) + log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) req := &Request{Request: *r, uri: uri} if err != nil { - s.logError("Invalid URI %q: %s", r.URL.Path, err) - s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) + log.Error(fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) + ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Method, uri, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)), http.StatusBadRequest) return } - s.logDebug("%s request received for %s", r.Method, uri) + log.Debug(fmt.Sprintf("%s request received for %s", r.Method, uri)) switch r.Method { case "POST": @@ -763,26 +763,6 @@ func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWri if err != nil { return nil, err } - s.logDebug("generated manifest %s", key) + log.Debug(fmt.Sprintf("generated manifest %s", key)) return key, nil } - -func (s *Server) logDebug(format string, v ...interface{}) { - log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) -} - -func (s *Server) logError(format string, v ...interface{}) { - log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) -} - -func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { - ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest) -} - -func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { - ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) -} - -func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) { - ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) -} diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index d1c5d96c1c..796b4167d0 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -65,7 +65,8 @@ func (a *Api) NewManifest() (storage.Key, error) { if err != nil { return nil, err } - key, _, err := a.Store(bytes.NewReader(data), int64(len(data))) + key, wait, err := a.Store(bytes.NewReader(data), int64(len(data))) + wait() return key, err } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 4679fabad3..a2ba70c7ac 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -16,7 +16,11 @@ package api -import "path" +import ( + "path" + + "github.com/ethereum/go-ethereum/swarm/storage" +) type Response struct { MimeType string @@ -41,12 +45,8 @@ func NewStorage(api *Api) *Storage { // its content type // // DEPRECATED: Use the HTTP API instead -func (self *Storage) Put(content, contentType string) (string, error) { - key, _, err := self.api.Put(content, contentType) - if err != nil { - return "", err - } - return key.Hex(), err +func (self *Storage) Put(content, contentType string) (storage.Key, func(), error) { + return self.api.Put(content, contentType) } // Get retrieves the content from bzzpath and reads the response in full diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go index d260dd61d8..bcbf53ee37 100644 --- a/swarm/api/storage_test.go +++ b/swarm/api/storage_test.go @@ -31,10 +31,12 @@ func TestStoragePutGet(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - bzzhash, err := api.Put(content, exp.MimeType) + bzzkey, wait, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } + wait() + bzzhash := bzzkey.Hex() // to check put against the Api#Get resp0 := testGet(t, api.api, bzzhash, "") checkResponse(t, resp0, exp) diff --git a/swarm/network/bitvector/bitvector.go b/swarm/network/bitvector/bitvector.go index 256c9fd5f3..5f2f64d027 100644 --- a/swarm/network/bitvector/bitvector.go +++ b/swarm/network/bitvector/bitvector.go @@ -30,7 +30,7 @@ func NewFromBytes(b []byte, l int) (bv *BitVector, err error) { func (bv *BitVector) Get(i int) bool { bi := i / 8 - return uint8(bv.b[bi])&(0x1< k.RetryInterval; delta /= div { + for delta := timeAgo; delta > k.RetryInterval; delta /= div { retries++ } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index f7ab37814a..9d9ddbc934 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -283,16 +283,15 @@ func TestSuggestPeerFindPeers(t *testing.T) { func TestSuggestPeerRetries(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 k := newTestKademlia("00000000") - cycle := time.Second - k.RetryInterval = uint(cycle) + k.RetryInterval = int64(time.Second) // cycle k.MaxRetries = 50 k.RetryExponent = 2 sleep := func(n int) { - t := k.RetryInterval + ts := k.RetryInterval for i := 1; i < n; i++ { - t *= k.RetryExponent + ts *= int64(k.RetryExponent) } - time.Sleep(time.Duration(t)) + time.Sleep(time.Duration(ts)) } k.Register("01000000") diff --git a/swarm/network/light/lightnode.go b/swarm/network/light/lightnode.go index 93ebbc8a78..06a0b4efcd 100644 --- a/swarm/network/light/lightnode.go +++ b/swarm/network/light/lightnode.go @@ -88,7 +88,7 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { end = true } copy(b[n:], chunk.SData[:m]) - n += int64(m) + n += m } for { diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index cb01eb1765..0cf682fb98 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -208,8 +208,10 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* // shared among swarm subprotocols func performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error { ctx, cancel := context.WithTimeout(context.Background(), bzzHandshakeTimeout) - defer cancel() - defer close(handshake.done) + defer func() { + close(handshake.done) + cancel() + }() rsh, err := p.Handshake(ctx, handshake, checkHandshake) if err != nil { handshake.err = err diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index a22bc9829a..7f0e8ccdcd 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -70,8 +70,7 @@ func BenchmarkDiscovery_64_4(b *testing.B) { benchmarkDiscovery(b, 64, 4) } func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) } func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) } -func TestDiscoverySimulationDockerAdapter(t *testing.T) { - t.Skip("broken (cannot build image)") +func XTestDiscoverySimulationDockerAdapter(t *testing.T) { testDiscoverySimulationDockerAdapter(t, *nodeCount, *initCount) } @@ -101,8 +100,20 @@ func testDiscoverySimulationExecAdapter(t *testing.T, nodes, conns int) { testDiscoverySimulation(t, nodes, conns, adapters.NewExecAdapter(baseDir)) } +func TestDiscoverySimulationSocketAdapter(t *testing.T) { + testDiscoverySimulationSocketAdapter(t, *nodeCount, *initCount) +} + func TestDiscoverySimulationSimAdapter(t *testing.T) { - testDiscoverySimulation(t, *nodeCount, *initCount, adapters.NewSimAdapter(services)) + testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount) +} + +func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { + testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services)) +} + +func testDiscoverySimulationSocketAdapter(t *testing.T, nodes, conns int) { + testDiscoverySimulation(t, nodes, conns, adapters.NewSocketAdapter(services)) } func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { @@ -154,7 +165,8 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul trigger := make(chan discover.NodeID) ids := make([]discover.NodeID, nodes) for i := 0; i < nodes; i++ { - node, err := net.NewNode() + conf := adapters.RandomNodeConfig() + node, err := net.NewNodeWithConfig(conf) if err != nil { return nil, fmt.Errorf("error starting node: %s", err) } diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 3ef991158e..024d612dcd 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -191,7 +191,7 @@ R: // this should be has locally chunk, err := d.db.Get(req.Key) if !bytes.Equal(chunk.Key, req.Key) { - panic(fmt.Errorf("processReceivedChunks: chunk key %s != req key %s (peer %s)", chunk.Key.Hex(), storage.Key(req.Key).Hex(), req.peer.ID())) + panic(fmt.Errorf("processReceivedChunks: chunk key %s != req key %s (peer %s)", chunk.Key.Hex(), req.Key.Hex(), req.peer.ID())) } if err == nil { continue R diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 183ea2b9e9..b737c071b9 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -57,7 +57,7 @@ func TestStreamerRetrieveRequest(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 5, Msg: &RetrieveRequestMsg{ Key: hash0[:], @@ -97,7 +97,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 5, Msg: &RetrieveRequestMsg{ Key: chunk.Key[:], @@ -106,7 +106,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: nil, @@ -154,7 +154,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 5, Msg: &RetrieveRequestMsg{ Key: hash, @@ -163,7 +163,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: &HandoverProof{ @@ -194,7 +194,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 5, Msg: &RetrieveRequestMsg{ Key: hash, @@ -204,7 +204,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 6, Msg: &ChunkDeliveryMsg{ Key: hash, @@ -256,7 +256,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 4, Msg: &SubscribeMsg{ Stream: "foo", @@ -272,7 +272,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { p2ptest.Exchange{ Label: "ChunkDeliveryRequest message", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 6, Msg: &ChunkDeliveryMsg{ Key: chunkKey, @@ -321,11 +321,12 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck defaultSkipCheck = skipCheck toAddr = network.NewAddrFromNodeID conf := &streamTesting.RunConfig{ - Adapter: *adapter, - NodeCount: nodes, - ConnLevel: conns, - ToAddr: toAddr, - Services: services, + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + EnableMsgEvents: false, } sim, teardown, err := streamTesting.NewSimulation(conf) @@ -495,11 +496,12 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip defer cancel() conf := &streamTesting.RunConfig{ - Adapter: *adapter, - NodeCount: nodes, - ConnLevel: conns, - ToAddr: toAddr, - Services: services, + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + EnableMsgEvents: false, } sim, teardown, err := streamTesting.NewSimulation(conf) defer teardown() diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 22592d288c..63c8783fdf 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -269,9 +269,6 @@ func (m TakeoverProofMsg) String() string { func (p *Peer) handleTakeoverProofMsg(req *TakeoverProofMsg) error { _, err := p.getServer(req.Stream) - if err != nil { - return err - } // store the strongest takeoverproof for the stream in streamer - return nil + return err } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 12810789d9..c1e64bd740 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -36,7 +36,7 @@ var ( errClientNotFound = errors.New("client not found") ) -// Peer is the Peer extention for the streaming protocol +// Peer is the Peer extension for the streaming protocol type Peer struct { *protocols.Peer streamer *Registry diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 5770f679c3..24925edd7e 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -266,7 +266,7 @@ func keyToString(key []byte) string { if l == 0 { return "" } - return fmt.Sprintf("%s-%d", string(key[:l-1]), uint8(key[l-1])) + return fmt.Sprintf("%s-%d", string(key[:l-1]), key[l-1]) } type server struct { diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index 951e008a53..a2aabc7f82 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -113,7 +113,7 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 4, Msg: &SubscribeMsg{ Stream: "foo", @@ -139,7 +139,7 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Unsubscribe message", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 0, Msg: &UnsubscribeMsg{ Stream: "foo", @@ -173,7 +173,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 4, Msg: &SubscribeMsg{ Stream: "foo", @@ -186,7 +186,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &OfferedHashesMsg{ Stream: "foo", @@ -210,7 +210,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "unsubscribe message", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 0, Msg: &UnsubscribeMsg{ Stream: "foo", @@ -244,7 +244,7 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 4, Msg: &SubscribeMsg{ Stream: "bar", @@ -257,7 +257,7 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 7, Msg: &SubscribeErrorMsg{ Error: "stream bar not registered", @@ -295,7 +295,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 4, Msg: &SubscribeMsg{ Stream: "foo", @@ -311,7 +311,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { p2ptest.Exchange{ Label: "WantedHashes message", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: &HandoverProof{ @@ -326,7 +326,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 2, Msg: &WantedHashesMsg{ Stream: "foo", diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index 6d8473afc9..605429693d 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -65,7 +65,7 @@ const maxPO = 32 func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) { streamer.RegisterServerFunc("SYNC", func(p *Peer, t []byte) (Server, error) { - po := uint8(t[0]) + po := t[0] // TODO: make this work for HISTORY too return NewSwarmSyncerServer(false, po, db) }) diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 58d780c36f..480bf61eaa 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -51,11 +51,12 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return addr } conf := &streamTesting.RunConfig{ - Adapter: *adapter, - NodeCount: nodes, - ConnLevel: conns, - ToAddr: toAddr, - Services: services, + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + EnableMsgEvents: false, } // create context for simulation run timeout := 30 * time.Second diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index e788e13dd8..39b2c1df1d 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -117,12 +117,13 @@ func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finish } type RunConfig struct { - Adapter string - Step *simulations.Step - NodeCount int - ConnLevel int - ToAddr func(discover.NodeID) *network.BzzAddr - Services adapters.Services + Adapter string + Step *simulations.Step + NodeCount int + ConnLevel int + ToAddr func(discover.NodeID) *network.BzzAddr + Services adapters.Services + EnableMsgEvents bool } func NewSimulation(conf *RunConfig) (*Simulation, func(), error) { @@ -144,7 +145,9 @@ func NewSimulation(conf *RunConfig) (*Simulation, func(), error) { addrs := make([]network.Addr, nodes) // start nodes for i := 0; i < nodes; i++ { - node, err := net.NewNode() + nodeconf := adapters.RandomNodeConfig() + nodeconf.EnableMsgEvents = conf.EnableMsgEvents + node, err := net.NewNodeWithConfig(nodeconf) if err != nil { return nil, teardown, fmt.Errorf("error creating node: %s", err) } diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 3d07074f0f..ae6b423382 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -180,9 +180,9 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { DefaultService: "bzz", }) for i := 0; i < numnodes; i++ { - nodes[i], err = net.NewNodeWithConfig(&adapters.NodeConfig{ - Services: []string{"bzz", "pss"}, - }) + nodeconf := adapters.RandomNodeConfig() + nodeconf.Services = []string{"bzz", "pss"} + nodes[i], err = net.NewNodeWithConfig(nodeconf) if err != nil { return nil, fmt.Errorf("error creating node 1: %v", err) } @@ -232,11 +232,11 @@ func newServices() adapters.Services { "pss": func(ctx *adapters.ServiceContext) (node.Service, error) { cachedir, err := ioutil.TempDir("", "pss-cache") if err != nil { - return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err) + return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err) } dpa, err := storage.NewLocalDPA(cachedir, make([]byte, 32)) if err != nil { - return nil, fmt.Errorf("local dpa creation failed: %v", err) + return nil, fmt.Errorf("local dpa creation failed: %s", err) } ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index b8ee373cad..d2d02898a6 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -498,7 +498,7 @@ func (self *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessag func (self *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { recvmsg, err := envelope.OpenAsymmetric(self.privateKey) if err != nil { - return nil, "", nil, fmt.Errorf("could not decrypt message: %v", err) + return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err) } // check signature (if signed), strip padding if !recvmsg.Validate() { diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index e92d0236b1..242f653463 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "crypto/ecdsa" + "encoding/binary" "encoding/hex" + "encoding/json" "flag" "fmt" "io/ioutil" @@ -17,6 +19,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -260,8 +263,8 @@ func TestKeys(t *testing.T) { if err != nil { t.Fatalf("create 'our' key fail") } - ctx, cancel = context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() theirkeys, err := wapi.NewKeyPair(ctx) if err != nil { t.Fatalf("create 'their' key fail") @@ -392,6 +395,679 @@ func TestMismatch(t *testing.T) { } +// send symmetrically encrypted message between two directly connected peers +func TestSymSend(t *testing.T) { + t.Run("32", testSymSend) + t.Run("8", testSymSend) + t.Run("0", testSymSend) +} + +func testSymSend(t *testing.T) { + + // address hint size + var addrsize int64 + var err error + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("sym send test", "addrsize", addrsize) + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + + var topic string + err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + + // retrieve public key from pss instance + // set this public key reciprocally + var lpubkeyhex string + err = clients[0].Call(&lpubkeyhex, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkeyhex string + err = clients[1].Call(&rpubkeyhex, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 500) + + // at this point we've verified that symkeys are saved and match on each peer + // now try sending symmetrically encrypted message, both directions + lmsgC := make(chan APIMsg) + lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) + defer lcancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + log.Trace("lsub", "id", lsub) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) + defer rcancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + log.Trace("rsub", "id", rsub) + defer rsub.Unsubscribe() + + lrecvkey := network.RandomAddr().Over() + rrecvkey := network.RandomAddr().Over() + + var lkeyids [2]string + var rkeyids [2]string + + // manually set reciprocal symkeys + err = clients[0].Call(&lkeyids, "psstest_setSymKeys", rpubkeyhex, lrecvkey, rrecvkey, defaultSymKeySendLimit, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(&rkeyids, "psstest_setSymKeys", rpubkeyhex, rrecvkey, lrecvkey, defaultSymKeySendLimit, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // send and verify delivery + lmsg := []byte("plugh") + err = clients[1].Call(nil, "pss_sendSym", rkeyids[1], topic, hexutil.Encode(lmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-lmsgC: + if !bytes.Equal(recvmsg.Msg, lmsg) { + t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg) + } + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + rmsg := []byte("xyzzy") + err = clients[0].Call(nil, "pss_sendSym", lkeyids[1], topic, hexutil.Encode(rmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-rmsgC: + if !bytes.Equal(recvmsg.Msg, rmsg) { + t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) + } + case cerr := <-rctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } +} + +// send asymmetrically encrypted message between two directly connected peers +func TestAsymSend(t *testing.T) { + t.Run("32", testAsymSend) + t.Run("8", testAsymSend) + t.Run("0", testAsymSend) +} + +func testAsymSend(t *testing.T) { + + // address hint size + var addrsize int64 + var err error + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("asym send test", "addrsize", addrsize) + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + + var topic string + err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Millisecond * 250) + + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + + // retrieve public key from pss instance + // set this public key reciprocally + var lpubkey string + err = clients[0].Call(&lpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkey string + err = clients[1].Call(&rpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 500) // replace with hive healthy code + + lmsgC := make(chan APIMsg) + lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10) + defer lcancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + log.Trace("lsub", "id", lsub) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10) + defer rcancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + log.Trace("rsub", "id", rsub) + defer rsub.Unsubscribe() + + // store reciprocal public keys + err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // send and verify delivery + rmsg := []byte("xyzzy") + err = clients[0].Call(nil, "pss_sendAsym", rpubkey, topic, hexutil.Encode(rmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-rmsgC: + if !bytes.Equal(recvmsg.Msg, rmsg) { + t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) + } + case cerr := <-rctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + lmsg := []byte("plugh") + err = clients[1].Call(nil, "pss_sendAsym", lpubkey, topic, hexutil.Encode(lmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-lmsgC: + if !bytes.Equal(recvmsg.Msg, lmsg) { + t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg.Msg) + } + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } +} + +type Job struct { + Msg []byte + SendNode discover.NodeID + RecvNode discover.NodeID +} + +func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubkeys map[discover.NodeID]string, topic string) { + for j := range jobs { + rpcs[j.SendNode].Call(nil, "pss_sendAsym", pubkeys[j.RecvNode], topic, hexutil.Encode(j.Msg)) + } +} + +// params in run name: +// nodes/msgs/addrbytes/adaptertype +// if adaptertype is exec uses execadapter, simadapter otherwise +func XTestNetwork(t *testing.T) { + t.Run("3/2000/4/sock", testNetwork) + t.Run("4/2000/4/sock", testNetwork) + t.Run("8/2000/4/sock", testNetwork) + t.Run("16/2000/4/sock", testNetwork) + t.Run("32/2000/4/sock", testNetwork) + t.Run("64/2000/4/sim", testNetwork) +} + +func testNetwork(t *testing.T) { + type msgnotifyC struct { + id discover.NodeID + msgIdx int + } + + paramstring := strings.Split(t.Name(), "/") + nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0) + msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0) + addrsize, _ := strconv.ParseInt(paramstring[3], 10, 0) + adapter := paramstring[4] + + log.Info("network test", "nodecount", nodecount, "msgcount", msgcount, "addrhintsize", addrsize) + + nodes := make([]discover.NodeID, nodecount) + bzzaddrs := make(map[discover.NodeID]string, nodecount) + rpcs := make(map[discover.NodeID]*rpc.Client, nodecount) + pubkeys := make(map[discover.NodeID]string, nodecount) + + sentmsgs := make([][]byte, msgcount) + recvmsgs := make([]bool, msgcount) + nodemsgcount := make(map[discover.NodeID]int, nodecount) + + trigger := make(chan discover.NodeID) + + var a adapters.NodeAdapter + if adapter == "exec" { + dirname, err := ioutil.TempDir(".", "") + if err != nil { + t.Fatal(err) + } + a = adapters.NewExecAdapter(dirname) + } else if adapter == "sock" { + a = adapters.NewSocketAdapter(services) + } else if adapter == "tcp" { + a = adapters.NewTCPAdapter(services) + } else if adapter == "sim" { + a = adapters.NewSimAdapter(services) + } + net := simulations.NewNetwork(a, &simulations.NetworkConfig{ + ID: "0", + }) + defer net.Shutdown() + + f, err := os.Open(fmt.Sprintf("testdata/snapshot_%d.json", nodecount)) + if err != nil { + t.Fatal(err) + } + jsonbyte, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(jsonbyte, &snap) + if err != nil { + t.Fatal(err) + } + err = net.Load(&snap) + if err != nil { + t.Fatal(err) + } + + triggerChecks := func(trigger chan discover.NodeID, id discover.NodeID, rpcclient *rpc.Client, topic string) error { + msgC := make(chan APIMsg) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic) + if err != nil { + t.Fatal(err) + } + go func() { + defer sub.Unsubscribe() + for { + select { + case recvmsg := <-msgC: + idx, _ := binary.Uvarint(recvmsg.Msg) + if !recvmsgs[idx] { + log.Debug("msg recv", "idx", idx, "id", id) + recvmsgs[idx] = true + trigger <- id + } + case <-sub.Err(): + return + } + } + }() + return nil + } + + var topic string + for i, nod := range net.GetNodes() { + nodes[i] = nod.ID() + rpcs[nodes[i]], err = nod.Client() + if err != nil { + t.Fatal(err) + } + if topic == "" { + err = rpcs[nodes[i]].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + } + var pubkey string + err = rpcs[nodes[i]].Call(&pubkey, "pss_getPublicKey") + if err != nil { + t.Fatal(err) + } + pubkeys[nod.ID()] = pubkey + var addrhex string + err = rpcs[nodes[i]].Call(&addrhex, "pss_baseAddr") + if err != nil { + t.Fatal(err) + } + bzzaddrs[nodes[i]] = addrhex + err = triggerChecks(trigger, nodes[i], rpcs[nodes[i]], topic) + if err != nil { + t.Fatal(err) + } + } + + // setup workers + jobs := make(chan Job, 10) + for w := 1; w <= 10; w++ { + go worker(w, jobs, rpcs, pubkeys, topic) + } + + for i := 0; i < int(msgcount); i++ { + sendnodeidx := rand.Intn(int(nodecount)) + recvnodeidx := rand.Intn(int(nodecount - 1)) + if recvnodeidx >= sendnodeidx { + recvnodeidx++ + } + nodemsgcount[nodes[recvnodeidx]]++ + sentmsgs[i] = make([]byte, 8) + c := binary.PutUvarint(sentmsgs[i], uint64(i)) + if c == 0 { + t.Fatal("0 byte message") + } + if err != nil { + t.Fatal(err) + } + err = rpcs[nodes[sendnodeidx]].Call(nil, "pss_setPeerPublicKey", pubkeys[nodes[recvnodeidx]], topic, bzzaddrs[nodes[recvnodeidx]]) + if err != nil { + t.Fatal(err) + } + + jobs <- Job{ + Msg: sentmsgs[i], + SendNode: nodes[sendnodeidx], + RecvNode: nodes[recvnodeidx], + } + } + + finalmsgcount := 0 + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() +outer: + for i := 0; i < int(msgcount); i++ { + select { + case id := <-trigger: + nodemsgcount[id]-- + finalmsgcount++ + case <-ctx.Done(): + log.Warn("timeout") + break outer + } + } + + for i, msg := range recvmsgs { + if !msg { + log.Debug("missing message", "idx", i) + } + } + t.Logf("%d of %d messages received", finalmsgcount, msgcount) + + if finalmsgcount != int(msgcount) { + t.Fatalf("%d messages were not received", int(msgcount)-finalmsgcount) + } + +} + +// symmetric send performance with varying message sizes +func BenchmarkSymkeySend(b *testing.B) { + b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkSymKeySend) +} + +func benchmarkSymKeySend(b *testing.B) { + msgsizestring := strings.Split(b.Name(), "/") + if len(msgsizestring) != 2 { + b.Fatalf("benchmark called without msgsize param") + } + msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + ps := newTestPss(privkey, nil, nil) + msg := make([]byte, msgsize) + rand.Read(msg) + topic := BytesToTopic([]byte("foo")) + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + symkeyid, err := ps.generateSymmetricKey(topic, &to, true) + if err != nil { + b.Fatalf("could not generate symkey: %v", err) + } + symkey, err := ps.w.GetSymKey(symkeyid) + if err != nil { + b.Fatalf("could not retrieve symkey: %v", err) + } + ps.SetSymmetricKey(symkey, topic, &to, false) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps.SendSym(symkeyid, topic, msg) + } +} + +// asymmetric send performance with varying message sizes +func BenchmarkAsymkeySend(b *testing.B) { + b.Run(fmt.Sprintf("%d", 256), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkAsymKeySend) +} + +func benchmarkAsymKeySend(b *testing.B) { + msgsizestring := strings.Split(b.Name(), "/") + if len(msgsizestring) != 2 { + b.Fatalf("benchmark called without msgsize param") + } + msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + ps := newTestPss(privkey, nil, nil) + msg := make([]byte, msgsize) + rand.Read(msg) + topic := BytesToTopic([]byte("foo")) + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to) + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) + } +} +func BenchmarkSymkeyBruteforceChangeaddr(b *testing.B) { + for i := 100; i < 100000; i = i * 10 { + for j := 32; j < 10000; j = j * 8 { + b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceChangeaddr) + } + //b.Run(fmt.Sprintf("%d", i), benchmarkSymkeyBruteforceChangeaddr) + } +} + +// decrypt performance using symkey cache, worst case +// (decrypt key always last in cache) +func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { + keycountstring := strings.Split(b.Name(), "/") + cachesize := int64(0) + var ps *Pss + if len(keycountstring) < 2 { + b.Fatalf("benchmark called without count param") + } + keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) + } + if len(keycountstring) == 3 { + cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) + } + } + pssmsgs := make([]*PssMsg, 0, keycount) + var keyid string + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + if cachesize > 0 { + ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) + } else { + ps = newTestPss(privkey, nil, nil) + } + topic := BytesToTopic([]byte("foo")) + for i := 0; i < int(keycount); i++ { + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + keyid, err = ps.generateSymmetricKey(topic, &to, true) + if err != nil { + b.Fatalf("cant generate symkey #%d: %v", i, err) + } + symkey, err := ps.w.GetSymKey(keyid) + if err != nil { + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) + } + wparams := &whisper.MessageParams{ + TTL: defaultWhisperTTL, + KeySym: symkey, + Topic: whisper.TopicType(topic), + WorkTime: defaultWhisperWorkTime, + PoW: defaultWhisperPoW, + Payload: []byte("xyzzy"), + Padding: []byte("1234567890abcdef"), + } + woutmsg, err := whisper.NewSentMessage(wparams) + if err != nil { + b.Fatalf("could not create whisper message: %v", err) + } + env, err := woutmsg.Wrap(wparams) + if err != nil { + b.Fatalf("could not generate whisper envelope: %v", err) + } + ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + }) + pssmsgs = append(pssmsgs, &PssMsg{ + To: to, + Payload: env, + }) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]) { + b.Fatalf("pss processing failed: %v", err) + } + } +} + +func BenchmarkSymkeyBruteforceSameaddr(b *testing.B) { + for i := 100; i < 100000; i = i * 10 { + for j := 32; j < 10000; j = j * 8 { + b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceSameaddr) + } + } +} + +// decrypt performance using symkey cache, best case +// (decrypt key always first in cache) +func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { + var keyid string + var ps *Pss + cachesize := int64(0) + keycountstring := strings.Split(b.Name(), "/") + if len(keycountstring) < 2 { + b.Fatalf("benchmark called without count param") + } + keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) + } + if len(keycountstring) == 3 { + cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) + } + } + addr := make([]PssAddress, keycount) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + if cachesize > 0 { + ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) + } else { + ps = newTestPss(privkey, nil, nil) + } + topic := BytesToTopic([]byte("foo")) + for i := 0; i < int(keycount); i++ { + copy(addr[i], network.RandomAddr().Over()) + keyid, err = ps.generateSymmetricKey(topic, &addr[i], true) + if err != nil { + b.Fatalf("cant generate symkey #%d: %v", i, err) + } + + } + symkey, err := ps.w.GetSymKey(keyid) + if err != nil { + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) + } + wparams := &whisper.MessageParams{ + TTL: defaultWhisperTTL, + KeySym: symkey, + Topic: whisper.TopicType(topic), + WorkTime: defaultWhisperWorkTime, + PoW: defaultWhisperPoW, + Payload: []byte("xyzzy"), + Padding: []byte("1234567890abcdef"), + } + woutmsg, err := whisper.NewSentMessage(wparams) + if err != nil { + b.Fatalf("could not create whisper message: %v", err) + } + env, err := woutmsg.Wrap(wparams) + if err != nil { + b.Fatalf("could not generate whisper envelope: %v", err) + } + ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + }) + pssmsg := &PssMsg{ + To: addr[len(addr)-1][:], + Payload: env, + } + for i := 0; i < b.N; i++ { + if !ps.process(pssmsg) { + b.Fatalf("pss processing failed: %v", err) + } + } +} + // setup simulated network and connect nodes in circle func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { nodes := make([]*simulations.Node, numnodes) @@ -405,9 +1081,9 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { DefaultService: "bzz", }) for i := 0; i < numnodes; i++ { - nodes[i], err = net.NewNodeWithConfig(&adapters.NodeConfig{ - Services: []string{"bzz", pssProtocolName}, - }) + nodeconf := adapters.RandomNodeConfig() + nodeconf.Services = []string{"bzz", pssProtocolName} + nodes[i], err = net.NewNodeWithConfig(nodeconf) if err != nil { return nil, fmt.Errorf("error creating node 1: %v", err) } @@ -457,11 +1133,11 @@ func newServices() adapters.Services { pssProtocolName: func(ctx *adapters.ServiceContext) (node.Service, error) { cachedir, err := ioutil.TempDir("", "pss-cache") if err != nil { - return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err) + return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err) } dpa, err := storage.NewLocalDPA(cachedir, network.NewAddrFromNodeID(ctx.Config.ID).Over()) if err != nil { - return nil, fmt.Errorf("local dpa creation failed: %v", err) + return nil, fmt.Errorf("local dpa creation failed: %s", err) } // execadapter does not exec init() diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 9ba6f5c1e0..f643fb502d 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -378,7 +378,7 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { return 0, err } if off+int64(len(b)) >= size { - return int(size - int64(off)), io.EOF + return int(size - off), io.EOF } return len(b), nil } diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 17a7534646..ea1ef46a02 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -98,7 +98,7 @@ type DbStore struct { // TODO: Instead of passing the distance function, just pass the address from which distances are calculated // to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing -// a function diferent from the one that is actually used. +// a function different from the one that is actually used. func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *DbStore, err error) { s = new(DbStore) s.hashfunc = hash @@ -126,7 +126,7 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uin for i := 0; i < 0x100; i++ { k := make([]byte, 2) k[0] = keyDistanceCnt - k[1] = byte(uint8(i)) + k[1] = uint8(i) cnt, _ := s.db.Get(k) s.bucketCnt[i] = BytesToU64(cnt) s.bucketCnt[i]++ @@ -211,7 +211,7 @@ func getOldDataKey(idx uint64) []byte { func getDataKey(idx uint64, po uint8) []byte { key := make([]byte, 10) key[0] = keyData - key[1] = byte(po) + key[1] = po binary.BigEndian.PutUint64(key[2:], idx) return key @@ -483,9 +483,9 @@ func (s *DbStore) ReIndex() { oldCntKey[0] = keyDistanceCnt newCntKey[0] = keyDistanceCnt key[0] = keyData - key[1] = byte(s.po(Key(key[1:]))) + key[1] = s.po(Key(key[1:])) oldCntKey[1] = key[1] - newCntKey[1] = byte(s.po(Key(newKey[1:]))) + newCntKey[1] = s.po(Key(newKey[1:])) copy(newKey[2:], key[1:]) newValue := append(hash, data...) @@ -733,8 +733,7 @@ func (s *DbStore) setCapacity(c uint64) { s.capacity = c if s.entryCnt > c { - var ratio float32 - ratio = float32(1.01) - float32(c)/float32(s.entryCnt) + ratio := float32(1.01) - float32(c)/float32(s.entryCnt) if ratio < gcArrayFreeRatio { ratio = gcArrayFreeRatio } @@ -760,7 +759,7 @@ func (s *DbStore) SyncIterator(since uint64, until uint64, po uint8, f func(Key, for ok := it.Seek(sincekey); ok; ok = it.Next() { dbkey := it.Key() - if dbkey[0] != keyData || dbkey[1] != byte(po) || bytes.Compare(untilkey, dbkey) < 0 { + if dbkey[0] != keyData || dbkey[1] != po || bytes.Compare(untilkey, dbkey) < 0 { break } key := make([]byte, 32) diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 6b86ed518e..65a1bc9669 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -206,7 +206,7 @@ func testIterator(t *testing.T, mock bool) { } for i = 0; i < chunkcount; i++ { - if bytes.Compare(chunkkeys[i], chunkkeys_results[i]) != 0 { + if !bytes.Equal(chunkkeys[i], chunkkeys_results[i]) { t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeys_results[i]) } } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 956b8ddd8b..2e6f6d7d47 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -88,7 +88,7 @@ func Proximity(one, other []byte) (ret int) { m = MaxPO % 8 } for j := 0; j < m; j++ { - if (uint8(oxo)>>uint8(7-j))&0x01 != 0 { + if (oxo>>uint8(7-j))&0x01 != 0 { return i*8 + j } } @@ -156,10 +156,7 @@ func (c KeyCollection) Len() int { } func (c KeyCollection) Less(i, j int) bool { - if bytes.Compare(c[i], c[j]) == -1 { - return true - } - return false + return bytes.Compare(c[i], c[j]) == -1 } func (c KeyCollection) Swap(i, j int) { diff --git a/swarm/swarm.go b/swarm/swarm.go index 3ca97bbecc..0ec85adadd 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -279,16 +279,13 @@ func (self *Swarm) Stop() error { // implements the node.Service interface func (self *Swarm) Protocols() (protos []p2p.Protocol) { - protos = append(protos, self.bzz.Protocols()...) if self.ps != nil { protos = append(protos, self.ps.Protocols()...) } if self.streamer != nil { - for _, p := range self.streamer.Protocols() { - protos = append(protos, p) - } + protos = append(protos, self.streamer.Protocols()...) } return } diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index 2248a1b129..a2b332a2e0 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -162,8 +162,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) - path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) + path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go deleted file mode 100644 index 0edd3782f5..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,go1.10 - -package notify - -const refZero = 0 diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go deleted file mode 100644 index b81c3c1859..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,cgo,!go1.10 - -package notify - -/* -#include -*/ -import "C" - -var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/vendor.json b/vendor/vendor.json index 675744d773..387afbb763 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -322,10 +322,10 @@ "revisionTime": "2016-11-28T21:05:44Z" }, { - "checksumSHA1": "1ESHllhZOIBg7MnlGHUdhz047bI=", + "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", "path": "github.com/rjeczalik/notify", - "revision": "27b537f07230b3f917421af6dcf044038dbe57e2", - "revisionTime": "2018-01-03T13:19:05Z" + "revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9", + "revisionTime": "2018-02-03T14:01:15Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index a3d6d5db5c..ad4d64cbf0 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -93,7 +93,7 @@ var masterBloomFilter []byte var masterPow = 0.00000001 var round = 1 -func TestSimulation(t *testing.T) { +func XTestSimulation(t *testing.T) { // create a chain of whisper nodes, // installs the filters with shared (predefined) parameters initialize(t)