From dcae0d348bb7f5d9052e50a83383a33538ce376a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 11 Oct 2018 20:32:14 +0200 Subject: [PATCH 001/100] p2p/simulations: fix a deadlock and clean up adapters (#17891) This fixes a rare deadlock with the inproc adapter: - A node is stopped, which acquires Network.lock. - The protocol code being simulated (swarm/network in my case) waits for its goroutines to shut down. - One of those goroutines calls into the simulation to add a peer, which waits for Network.lock. The fix for the deadlock is really simple, just release the lock before stopping the simulation node. Other changes in this PR clean up the exec adapter so it reports node startup errors better and remove the docker adapter because it just adds overhead. In the exec adapter, node information is now posted to a one-shot server. This avoids log parsing and allows reporting startup errors to the simulation host. A small change in package node was needed because simulation nodes use port zero. Node.{HTTP,WS}Endpoint now return the live endpoints after startup by checking the TCP listener. --- node/node.go | 12 + p2p/simulations/README.md | 12 - p2p/simulations/adapters/docker.go | 190 --------------- p2p/simulations/adapters/exec.go | 225 ++++++++++-------- p2p/simulations/adapters/ws.go | 51 ---- p2p/simulations/adapters/ws_test.go | 21 -- p2p/simulations/examples/ping-pong.go | 8 - p2p/simulations/network.go | 44 ++-- .../simulations/discovery/discovery_test.go | 19 +- 9 files changed, 164 insertions(+), 418 deletions(-) delete mode 100644 p2p/simulations/adapters/docker.go delete mode 100644 p2p/simulations/adapters/ws.go delete mode 100644 p2p/simulations/adapters/ws_test.go diff --git a/node/node.go b/node/node.go index ada3837217..85299dba74 100644 --- a/node/node.go +++ b/node/node.go @@ -549,11 +549,23 @@ func (n *Node) IPCEndpoint() string { // HTTPEndpoint retrieves the current HTTP endpoint used by the protocol stack. func (n *Node) HTTPEndpoint() string { + n.lock.Lock() + defer n.lock.Unlock() + + if n.httpListener != nil { + return n.httpListener.Addr().String() + } return n.httpEndpoint } // WSEndpoint retrieves the current WS endpoint used by the protocol stack. func (n *Node) WSEndpoint() string { + n.lock.Lock() + defer n.lock.Unlock() + + if n.wsListener != nil { + return n.wsListener.Addr().String() + } return n.wsEndpoint } diff --git a/p2p/simulations/README.md b/p2p/simulations/README.md index d1f8649eae..871d71b2c7 100644 --- a/p2p/simulations/README.md +++ b/p2p/simulations/README.md @@ -63,18 +63,6 @@ using the devp2p node stack rather than executing `main()`. The nodes listen for devp2p connections and WebSocket RPC clients on random localhost ports. -### DockerAdapter - -The `DockerAdapter` is similar to the `ExecAdapter` but executes `docker run` -to run the node in a Docker container using a Docker image containing the -simulation binary at `/bin/p2p-node`. - -The Docker image is built using `docker build` when the adapter is initialised, -meaning no prior setup is necessary other than having a working Docker client. - -Each node listens on the external IP of the container and the default p2p and -RPC ports (`30303` and `8546` respectively). - ## Network A simulation network is created with an ID and default service (which is used diff --git a/p2p/simulations/adapters/docker.go b/p2p/simulations/adapters/docker.go deleted file mode 100644 index 82eab0e9cd..0000000000 --- a/p2p/simulations/adapters/docker.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package adapters - -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - - "github.com/docker/docker/pkg/reexec" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/enode" -) - -var ( - ErrLinuxOnly = errors.New("DockerAdapter can only be used on Linux as it uses the current binary (which must be a Linux binary)") -) - -// DockerAdapter is a NodeAdapter which runs simulation nodes inside Docker -// containers. -// -// A Docker image is built which contains the current binary at /bin/p2p-node -// which when executed runs the underlying service (see the description -// of the execP2PNode function for more details) -type DockerAdapter struct { - ExecAdapter -} - -// NewDockerAdapter builds the p2p-node Docker image containing the current -// binary and returns a DockerAdapter -func NewDockerAdapter() (*DockerAdapter, error) { - // Since Docker containers run on Linux and this adapter runs the - // current binary in the container, it must be compiled for Linux. - // - // It is reasonable to require this because the caller can just - // compile the current binary in a Docker container. - if runtime.GOOS != "linux" { - return nil, ErrLinuxOnly - } - - if err := buildDockerImage(); err != nil { - return nil, err - } - - return &DockerAdapter{ - ExecAdapter{ - nodes: make(map[enode.ID]*ExecNode), - }, - }, nil -} - -// Name returns the name of the adapter for logging purposes -func (d *DockerAdapter) Name() string { - return "docker-adapter" -} - -// NewNode returns a new DockerNode using the given config -func (d *DockerAdapter) NewNode(config *NodeConfig) (Node, error) { - if len(config.Services) == 0 { - return nil, errors.New("node must have at least one service") - } - for _, service := range config.Services { - if _, exists := serviceFuncs[service]; !exists { - return nil, fmt.Errorf("unknown node service %q", service) - } - } - - // generate the config - conf := &execNodeConfig{ - Stack: node.DefaultConfig, - Node: config, - } - conf.Stack.DataDir = "/data" - conf.Stack.WSHost = "0.0.0.0" - conf.Stack.WSOrigins = []string{"*"} - conf.Stack.WSExposeAll = true - conf.Stack.P2P.EnableMsgEvents = false - conf.Stack.P2P.NoDiscovery = true - conf.Stack.P2P.NAT = nil - conf.Stack.NoUSB = true - - // listen on all interfaces on a given port, which we set when we - // initialise NodeConfig (usually a random port) - conf.Stack.P2P.ListenAddr = fmt.Sprintf(":%d", config.Port) - - node := &DockerNode{ - ExecNode: ExecNode{ - ID: config.ID, - Config: conf, - adapter: &d.ExecAdapter, - }, - } - node.newCmd = node.dockerCommand - d.ExecAdapter.nodes[node.ID] = &node.ExecNode - return node, nil -} - -// DockerNode wraps an ExecNode but exec's the current binary in a docker -// container rather than locally -type DockerNode struct { - ExecNode -} - -// dockerCommand returns a command which exec's the binary in a Docker -// container. -// -// It uses a shell so that we can pass the _P2P_NODE_CONFIG environment -// variable to the container using the --env flag. -func (n *DockerNode) dockerCommand() *exec.Cmd { - return exec.Command( - "sh", "-c", - fmt.Sprintf( - `exec docker run --interactive --env _P2P_NODE_CONFIG="${_P2P_NODE_CONFIG}" %s p2p-node %s %s`, - dockerImage, strings.Join(n.Config.Node.Services, ","), n.ID.String(), - ), - ) -} - -// dockerImage is the name of the Docker image which gets built to run the -// simulation node -const dockerImage = "p2p-node" - -// buildDockerImage builds the Docker image which is used to run the simulation -// node in a Docker container. -// -// It adds the current binary as "p2p-node" so that it runs execP2PNode -// when executed. -func buildDockerImage() error { - // create a directory to use as the build context - dir, err := ioutil.TempDir("", "p2p-docker") - if err != nil { - return err - } - defer os.RemoveAll(dir) - - // copy the current binary into the build context - bin, err := os.Open(reexec.Self()) - if err != nil { - return err - } - defer bin.Close() - dst, err := os.OpenFile(filepath.Join(dir, "self.bin"), os.O_WRONLY|os.O_CREATE, 0755) - if err != nil { - return err - } - defer dst.Close() - if _, err := io.Copy(dst, bin); err != nil { - return err - } - - // create the Dockerfile - dockerfile := []byte(` -FROM ubuntu:16.04 -RUN mkdir /data -ADD self.bin /bin/p2p-node - `) - if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dockerfile, 0644); err != nil { - return err - } - - // run 'docker build' - cmd := exec.Command("docker", "build", "-t", dockerImage, dir) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("error building docker image: %s", err) - } - - return nil -} diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index dc7d277cac..abb1967171 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -17,7 +17,7 @@ package adapters import ( - "bufio" + "bytes" "context" "crypto/ecdsa" "encoding/json" @@ -25,6 +25,7 @@ import ( "fmt" "io" "net" + "net/http" "os" "os/exec" "os/signal" @@ -43,12 +44,14 @@ import ( "golang.org/x/net/websocket" ) -// ExecAdapter is a NodeAdapter which runs simulation nodes by executing the -// current binary as a child process. -// -// An init hook is used so that the child process executes the node services -// (rather than whataver the main() function would normally do), see the -// execP2PNode function for more information. +func init() { + // Register a reexec function to start a simulation node when the current binary is + // executed as "p2p-node" (rather than whataver the main() function would normally do). + reexec.Register("p2p-node", execP2PNode) +} + +// ExecAdapter is a NodeAdapter which runs simulation nodes by executing the current binary +// as a child process. type ExecAdapter struct { // BaseDir is the directory under which the data directories for each // simulation node are created. @@ -150,15 +153,13 @@ func (n *ExecNode) Client() (*rpc.Client, error) { } // Start exec's the node passing the ID and service as command line arguments -// and the node config encoded as JSON in the _P2P_NODE_CONFIG environment -// variable +// and the node config encoded as JSON in an environment variable. func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { if n.Cmd != nil { return errors.New("already started") } defer func() { if err != nil { - log.Error("node failed to start", "err", err) n.Stop() } }() @@ -175,59 +176,78 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { return fmt.Errorf("error generating node config: %s", err) } - // use a pipe for stderr so we can both copy the node's stderr to - // os.Stderr and read the WebSocket address from the logs - stderrR, stderrW := io.Pipe() - stderr := io.MultiWriter(os.Stderr, stderrW) + // start the one-shot server that waits for startup information + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + statusURL, statusC := n.waitForStartupJSON(ctx) // start the node cmd := n.newCmd() cmd.Stdout = os.Stdout - cmd.Stderr = stderr - cmd.Env = append(os.Environ(), fmt.Sprintf("_P2P_NODE_CONFIG=%s", confData)) + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), + envStatusURL+"="+statusURL, + envNodeConfig+"="+string(confData), + ) if err := cmd.Start(); err != nil { return fmt.Errorf("error starting node: %s", err) } n.Cmd = cmd // read the WebSocket address from the stderr logs - var wsAddr string - wsAddrC := make(chan string) - go func() { - s := bufio.NewScanner(stderrR) - for s.Scan() { - if strings.Contains(s.Text(), "WebSocket endpoint opened") { - wsAddrC <- wsAddrPattern.FindString(s.Text()) - } - } - }() - select { - case wsAddr = <-wsAddrC: - if wsAddr == "" { - return errors.New("failed to read WebSocket address from stderr") - } - case <-time.After(10 * time.Second): - return errors.New("timed out waiting for WebSocket address on stderr") + status := <-statusC + if status.Err != "" { + return errors.New(status.Err) } - - // create the RPC client and load the node info - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - client, err := rpc.DialWebsocket(ctx, wsAddr, "") + client, err := rpc.DialWebsocket(ctx, status.WSEndpoint, "http://localhost") if err != nil { - return fmt.Errorf("error dialing rpc websocket: %s", err) + return fmt.Errorf("can't connect to RPC server: %v", err) } - var info p2p.NodeInfo - if err := client.CallContext(ctx, &info, "admin_nodeInfo"); err != nil { - return fmt.Errorf("error getting node info: %s", err) - } - n.client = client - n.wsAddr = wsAddr - n.Info = &info + // node ready :) + n.client = client + n.wsAddr = status.WSEndpoint + n.Info = status.NodeInfo return nil } +// waitForStartupJSON runs a one-shot HTTP server to receive a startup report. +func (n *ExecNode) waitForStartupJSON(ctx context.Context) (string, chan nodeStartupJSON) { + var ( + ch = make(chan nodeStartupJSON, 1) + quitOnce sync.Once + srv http.Server + ) + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ch <- nodeStartupJSON{Err: err.Error()} + return "", ch + } + quit := func(status nodeStartupJSON) { + quitOnce.Do(func() { + l.Close() + ch <- status + }) + } + srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var status nodeStartupJSON + if err := json.NewDecoder(r.Body).Decode(&status); err != nil { + status.Err = fmt.Sprintf("can't decode startup report: %v", err) + } + quit(status) + }) + // Run the HTTP server, but don't wait forever and shut it down + // if the context is canceled. + go srv.Serve(l) + go func() { + <-ctx.Done() + quit(nodeStartupJSON{Err: "didn't get startup report"}) + }() + + url := "http://" + l.Addr().String() + return url, ch +} + // execCommand returns a command which runs the node locally by exec'ing // the current binary but setting argv[0] to "p2p-node" so that the child // runs execP2PNode @@ -318,12 +338,6 @@ func (n *ExecNode) Snapshots() (map[string][]byte, error) { return snapshots, n.client.Call(&snapshots, "simulation_snapshot") } -func init() { - // register a reexec function to start a devp2p node when the current - // binary is executed as "p2p-node" - reexec.Register("p2p-node", execP2PNode) -} - // execNodeConfig is used to serialize the node configuration so it can be // passed to the child process as a JSON encoded environment variable type execNodeConfig struct { @@ -333,55 +347,69 @@ type execNodeConfig struct { PeerAddrs map[string]string `json:"peer_addrs,omitempty"` } -// ExternalIP gets an external IP address so that Enode URL is usable -func ExternalIP() net.IP { - addrs, err := net.InterfaceAddrs() - if err != nil { - log.Crit("error getting IP address", "err", err) - } - for _, addr := range addrs { - if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && !ip.IP.IsLinkLocalUnicast() { - return ip.IP - } - } - log.Warn("unable to determine explicit IP address, falling back to loopback") - return net.IP{127, 0, 0, 1} -} - -// execP2PNode starts a devp2p node when the current binary is executed with +// execP2PNode starts a simulation node when the current binary is executed with // argv[0] being "p2p-node", reading the service / ID from argv[1] / argv[2] -// and the node config from the _P2P_NODE_CONFIG environment variable +// and the node config from an environment variable. func execP2PNode() { glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat())) glogger.Verbosity(log.LvlInfo) log.Root().SetHandler(glogger) + statusURL := os.Getenv(envStatusURL) + if statusURL == "" { + log.Crit("missing " + envStatusURL) + } + // Start the node and gather startup report. + var status nodeStartupJSON + stack, stackErr := startExecNodeStack() + if stackErr != nil { + status.Err = stackErr.Error() + } else { + status.WSEndpoint = "ws://" + stack.WSEndpoint() + status.NodeInfo = stack.Server().NodeInfo() + } + + // Send status to the host. + statusJSON, _ := json.Marshal(status) + if _, err := http.Post(statusURL, "application/json", bytes.NewReader(statusJSON)); err != nil { + log.Crit("Can't post startup info", "url", statusURL, "err", err) + } + if stackErr != nil { + os.Exit(1) + } + + // Stop the stack if we get a SIGTERM signal. + go func() { + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGTERM) + defer signal.Stop(sigc) + <-sigc + log.Info("Received SIGTERM, shutting down...") + stack.Stop() + }() + stack.Wait() // Wait for the stack to exit. +} + +func startExecNodeStack() (*node.Node, error) { // read the services from argv serviceNames := strings.Split(os.Args[1], ",") // decode the config - confEnv := os.Getenv("_P2P_NODE_CONFIG") + confEnv := os.Getenv(envNodeConfig) if confEnv == "" { - log.Crit("missing _P2P_NODE_CONFIG") + return nil, fmt.Errorf("missing " + envNodeConfig) } var conf execNodeConfig if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { - log.Crit("error decoding _P2P_NODE_CONFIG", "err", err) + return nil, fmt.Errorf("error decoding %s: %v", envNodeConfig, err) } conf.Stack.P2P.PrivateKey = conf.Node.PrivateKey conf.Stack.Logger = log.New("node.id", conf.Node.ID.String()) - if strings.HasPrefix(conf.Stack.P2P.ListenAddr, ":") { - conf.Stack.P2P.ListenAddr = ExternalIP().String() + conf.Stack.P2P.ListenAddr - } - if conf.Stack.WSHost == "0.0.0.0" { - conf.Stack.WSHost = ExternalIP().String() - } - // initialize the devp2p stack stack, err := node.New(&conf.Stack) if err != nil { - log.Crit("error creating node stack", "err", err) + return nil, fmt.Errorf("error creating node stack: %v", err) } // register the services, collecting them into a map so we can wrap @@ -390,7 +418,7 @@ func execP2PNode() { for _, name := range serviceNames { serviceFunc, exists := serviceFuncs[name] if !exists { - log.Crit("unknown node service", "name", name) + return nil, fmt.Errorf("unknown node service %q", err) } constructor := func(nodeCtx *node.ServiceContext) (node.Service, error) { ctx := &ServiceContext{ @@ -409,34 +437,35 @@ func execP2PNode() { return service, nil } if err := stack.Register(constructor); err != nil { - log.Crit("error starting service", "name", name, "err", err) + return stack, fmt.Errorf("error registering service %q: %v", name, err) } } // register the snapshot service - if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + err = stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { return &snapshotService{services}, nil - }); err != nil { - log.Crit("error starting snapshot service", "err", err) + }) + if err != nil { + return stack, fmt.Errorf("error starting snapshot service: %v", err) } // start the stack - if err := stack.Start(); err != nil { - log.Crit("error stating node stack", "err", err) + if err = stack.Start(); err != nil { + err = fmt.Errorf("error starting stack: %v", err) } + return stack, err +} - // stop the stack if we get a SIGTERM signal - go func() { - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGTERM) - defer signal.Stop(sigc) - <-sigc - log.Info("Received SIGTERM, shutting down...") - stack.Stop() - }() +const ( + envStatusURL = "_P2P_STATUS_URL" + envNodeConfig = "_P2P_NODE_CONFIG" +) - // wait for the stack to exit - stack.Wait() +// nodeStartupJSON is sent to the simulation host after startup. +type nodeStartupJSON struct { + Err string + WSEndpoint string + NodeInfo *p2p.NodeInfo } // snapshotService is a node.Service which wraps a list of services and diff --git a/p2p/simulations/adapters/ws.go b/p2p/simulations/adapters/ws.go deleted file mode 100644 index 979a21709e..0000000000 --- a/p2p/simulations/adapters/ws.go +++ /dev/null @@ -1,51 +0,0 @@ -package adapters - -import ( - "bufio" - "errors" - "io" - "regexp" - "strings" - "time" -) - -// wsAddrPattern is a regex used to read the WebSocket address from the node's -// log -var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) - -func matchWSAddr(str string) (string, bool) { - if !strings.Contains(str, "WebSocket endpoint opened") { - return "", false - } - - return wsAddrPattern.FindString(str), true -} - -// findWSAddr scans through reader r, looking for the log entry with -// WebSocket address information. -func findWSAddr(r io.Reader, timeout time.Duration) (string, error) { - ch := make(chan string) - - go func() { - s := bufio.NewScanner(r) - for s.Scan() { - addr, ok := matchWSAddr(s.Text()) - if ok { - ch <- addr - } - } - close(ch) - }() - - var wsAddr string - select { - case wsAddr = <-ch: - if wsAddr == "" { - return "", errors.New("empty result") - } - case <-time.After(timeout): - return "", errors.New("timed out") - } - - return wsAddr, nil -} diff --git a/p2p/simulations/adapters/ws_test.go b/p2p/simulations/adapters/ws_test.go deleted file mode 100644 index 0bb9ed2b2b..0000000000 --- a/p2p/simulations/adapters/ws_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package adapters - -import ( - "bytes" - "testing" - "time" -) - -func TestFindWSAddr(t *testing.T) { - line := `t=2018-05-02T19:00:45+0200 lvl=info msg="WebSocket endpoint opened" node.id=26c65a606d1125a44695bc08573190d047152b6b9a776ccbbe593e90f91444d9c1ebdadac6a775ad9fdd0923468a1d698ed3a842c1fb89c1bc0f9d4801f8c39c url=ws://127.0.0.1:59975` - buf := bytes.NewBufferString(line) - got, err := findWSAddr(buf, 10*time.Second) - if err != nil { - t.Fatalf("Failed to find addr: %v", err) - } - expected := `ws://127.0.0.1:59975` - - if got != expected { - t.Fatalf("Expected to get '%s', but got '%s'", expected, got) - } -} diff --git a/p2p/simulations/examples/ping-pong.go b/p2p/simulations/examples/ping-pong.go index 597cc950c9..cde2f3a677 100644 --- a/p2p/simulations/examples/ping-pong.go +++ b/p2p/simulations/examples/ping-pong.go @@ -70,14 +70,6 @@ func main() { log.Info("using exec adapter", "tmpdir", tmpdir) adapter = adapters.NewExecAdapter(tmpdir) - case "docker": - log.Info("using docker adapter") - var err error - adapter, err = adapters.NewDockerAdapter() - if err != nil { - log.Crit("error creating docker adapter", "err", err) - } - default: log.Crit(fmt.Sprintf("unknown node adapter %q", *adapterType)) } diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 101ac09f85..200015ff39 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -116,7 +116,7 @@ func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Node: adapterNode, Config: conf, } - log.Trace(fmt.Sprintf("node %v created", conf.ID)) + log.Trace("Node created", "id", conf.ID) net.nodeMap[conf.ID] = len(net.Nodes) net.Nodes = append(net.Nodes, node) @@ -167,6 +167,7 @@ func (net *Network) Start(id enode.ID) error { func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) error { net.lock.Lock() defer net.lock.Unlock() + node := net.getNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) @@ -174,13 +175,13 @@ func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) if node.Up { return fmt.Errorf("node %v already up", id) } - log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, net.nodeAdapter.Name())) + log.Trace("Starting node", "id", id, "adapter", net.nodeAdapter.Name()) if err := node.Start(snapshots); err != nil { - log.Warn(fmt.Sprintf("start up failed: %v", err)) + log.Warn("Node startup failed", "id", id, "err", err) return err } node.Up = true - log.Info(fmt.Sprintf("started node %v: %v", id, node.Up)) + log.Info("Started node", "id", id) net.events.Send(NewEvent(node)) @@ -209,7 +210,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub defer net.lock.Unlock() node := net.getNode(id) if node == nil { - log.Error("Can not find node for id", "id", id) return } node.Up = false @@ -240,7 +240,7 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub case err := <-sub.Err(): if err != nil { - log.Error(fmt.Sprintf("error getting peer events for node %v", id), "err", err) + log.Error("Error in peer event subscription", "id", id, "err", err) } return } @@ -250,7 +250,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub // Stop stops the node with the given ID func (net *Network) Stop(id enode.ID) error { net.lock.Lock() - defer net.lock.Unlock() node := net.getNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) @@ -258,12 +257,17 @@ func (net *Network) Stop(id enode.ID) error { if !node.Up { return fmt.Errorf("node %v already down", id) } - if err := node.Stop(); err != nil { + node.Up = false + net.lock.Unlock() + + err := node.Stop() + if err != nil { + net.lock.Lock() + node.Up = true + net.lock.Unlock() return err } - node.Up = false - log.Info(fmt.Sprintf("stop node %v: %v", id, node.Up)) - + log.Info("Stopped node", "id", id, "err", err) net.events.Send(ControlEvent(node)) return nil } @@ -271,7 +275,7 @@ func (net *Network) Stop(id enode.ID) error { // Connect connects two nodes together by calling the "admin_addPeer" RPC // method on the "one" node so that it connects to the "other" node func (net *Network) Connect(oneID, otherID enode.ID) error { - log.Debug(fmt.Sprintf("connecting %s to %s", oneID, otherID)) + log.Debug("Connecting nodes with addPeer", "id", oneID, "other", otherID) conn, err := net.InitConn(oneID, otherID) if err != nil { return err @@ -481,10 +485,10 @@ func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) { err = conn.nodesUp() if err != nil { - log.Trace(fmt.Sprintf("nodes not up: %v", err)) + log.Trace("Nodes not up", "err", err) return nil, fmt.Errorf("nodes not up: %v", err) } - log.Debug("InitConn - connection initiated") + log.Debug("Connection initiated", "id", oneID, "other", otherID) conn.initiated = time.Now() return conn, nil } @@ -492,9 +496,9 @@ func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) { // Shutdown stops all nodes in the network and closes the quit channel func (net *Network) Shutdown() { for _, node := range net.Nodes { - log.Debug(fmt.Sprintf("stopping node %s", node.ID().TerminalString())) + log.Debug("Stopping node", "id", node.ID()) if err := node.Stop(); err != nil { - log.Warn(fmt.Sprintf("error stopping node %s", node.ID().TerminalString()), "err", err) + log.Warn("Can't stop node", "id", node.ID(), "err", err) } } close(net.quitc) @@ -708,18 +712,18 @@ func (net *Network) Subscribe(events chan *Event) { } func (net *Network) executeControlEvent(event *Event) { - log.Trace("execute control event", "type", event.Type, "event", event) + log.Trace("Executing control event", "type", event.Type, "event", event) switch event.Type { case EventTypeNode: if err := net.executeNodeEvent(event); err != nil { - log.Error("error executing node event", "event", event, "err", err) + log.Error("Error executing node event", "event", event, "err", err) } case EventTypeConn: if err := net.executeConnEvent(event); err != nil { - log.Error("error executing conn event", "event", event, "err", err) + log.Error("Error executing conn event", "event", event, "err", err) } case EventTypeMsg: - log.Warn("ignoring control msg event") + log.Warn("Ignoring control msg event") } } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index d11eabf95f..3c3affe58c 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -125,22 +125,6 @@ func BenchmarkDiscovery_64_4(b *testing.B) { benchmarkDiscovery(b, 64, 4) } func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) } func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) } -func TestDiscoverySimulationDockerAdapter(t *testing.T) { - testDiscoverySimulationDockerAdapter(t, *nodeCount, *initCount) -} - -func testDiscoverySimulationDockerAdapter(t *testing.T, nodes, conns int) { - adapter, err := adapters.NewDockerAdapter() - if err != nil { - if err == adapters.ErrLinuxOnly { - t.Skip(err) - } else { - t.Fatal(err) - } - } - testDiscoverySimulation(t, nodes, conns, adapter) -} - func TestDiscoverySimulationExecAdapter(t *testing.T) { testDiscoverySimulationExecAdapter(t, *nodeCount, *initCount) } @@ -545,8 +529,7 @@ func triggerChecks(trigger chan enode.ID, net *simulations.Network, id enode.ID) } func newService(ctx *adapters.ServiceContext) (node.Service, error) { - node := enode.NewV4(&ctx.Config.PrivateKey.PublicKey, adapters.ExternalIP(), int(ctx.Config.Port), int(ctx.Config.Port)) - addr := network.NewAddr(node) + addr := network.NewAddr(ctx.Config.Node()) kp := network.NewKadParams() kp.MinProxBinSize = testMinProxBinSize From 6f607de5d590ff2fbe8798b04e5924be3b7ca0b4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 12 Oct 2018 11:47:24 +0200 Subject: [PATCH 002/100] p2p, p2p/discover: add signed ENR generation (#17753) This PR adds enode.LocalNode and integrates it into the p2p subsystem. This new object is the keeper of the local node record. For now, a new version of the record is produced every time the client restarts. We'll make it smarter to avoid that in the future. There are a couple of other changes in this commit: discovery now waits for all of its goroutines at shutdown and the p2p server now closes the node database after discovery has shut down. This fixes a leveldb crash in tests. p2p server startup is faster because it doesn't need to wait for the external IP query anymore. --- cmd/bootnode/main.go | 11 +- node/node_test.go | 8 +- p2p/dial.go | 7 +- p2p/dial_test.go | 16 +-- p2p/discover/table.go | 41 +++--- p2p/discover/table_test.go | 10 +- p2p/discover/table_util_test.go | 25 +++- p2p/discover/udp.go | 92 ++++++------ p2p/discover/udp_test.go | 11 +- p2p/discv5/udp.go | 3 +- p2p/enode/localnode.go | 246 ++++++++++++++++++++++++++++++++ p2p/enode/localnode_test.go | 76 ++++++++++ p2p/enode/node.go | 7 + p2p/enode/nodedb.go | 124 +++++++++++----- p2p/enode/nodedb_test.go | 4 +- p2p/enr/enr.go | 2 +- p2p/enr/enr_test.go | 12 ++ p2p/nat/nat.go | 18 +-- p2p/nat/nat_test.go | 2 +- p2p/netutil/iptrack.go | 130 +++++++++++++++++ p2p/netutil/iptrack_test.go | 138 ++++++++++++++++++ p2p/protocol.go | 10 +- p2p/server.go | 238 +++++++++++++++--------------- p2p/server_test.go | 26 ++-- 24 files changed, 979 insertions(+), 278 deletions(-) create mode 100644 p2p/enode/localnode.go create mode 100644 p2p/enode/localnode_test.go create mode 100644 p2p/netutil/iptrack.go create mode 100644 p2p/netutil/iptrack_test.go diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 8459008652..346523ddb6 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -119,16 +119,17 @@ func main() { } if *runv5 { - if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil { + if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil { utils.Fatalf("%v", err) } } else { + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, nodeKey) cfg := discover.Config{ - PrivateKey: nodeKey, - AnnounceAddr: realaddr, - NetRestrict: restrictList, + PrivateKey: nodeKey, + NetRestrict: restrictList, } - if _, err := discover.ListenUDP(conn, cfg); err != nil { + if _, err := discover.ListenUDP(conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } diff --git a/node/node_test.go b/node/node_test.go index e51900bd1e..f833cd6880 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -454,9 +454,9 @@ func TestProtocolGather(t *testing.T) { Count int Maker InstrumentingWrapper }{ - "Zero Protocols": {0, InstrumentedServiceMakerA}, - "Single Protocol": {1, InstrumentedServiceMakerB}, - "Many Protocols": {25, InstrumentedServiceMakerC}, + "zero": {0, InstrumentedServiceMakerA}, + "one": {1, InstrumentedServiceMakerB}, + "many": {10, InstrumentedServiceMakerC}, } for id, config := range services { protocols := make([]p2p.Protocol, config.Count) @@ -480,7 +480,7 @@ func TestProtocolGather(t *testing.T) { defer stack.Stop() protocols := stack.Server().Protocols - if len(protocols) != 26 { + if len(protocols) != 11 { t.Fatalf("mismatching number of protocols launched: have %d, want %d", len(protocols), 26) } for id, config := range services { diff --git a/p2p/dial.go b/p2p/dial.go index 359cdbcbba..d228514fc6 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -71,6 +71,7 @@ type dialstate struct { maxDynDials int ntab discoverTable netrestrict *netutil.Netlist + self enode.ID lookupRunning bool dialing map[enode.ID]connFlag @@ -84,7 +85,6 @@ type dialstate struct { } type discoverTable interface { - Self() *enode.Node Close() Resolve(*enode.Node) *enode.Node LookupRandom() []*enode.Node @@ -126,10 +126,11 @@ type waitExpireTask struct { time.Duration } -func newDialState(static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate { +func newDialState(self enode.ID, static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate { s := &dialstate{ maxDynDials: maxdyn, ntab: ntab, + self: self, netrestrict: netrestrict, static: make(map[enode.ID]*dialTask), dialing: make(map[enode.ID]connFlag), @@ -266,7 +267,7 @@ func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error { return errAlreadyDialing case peers[n.ID()] != nil: return errAlreadyConnected - case s.ntab != nil && n.ID() == s.ntab.Self().ID(): + case n.ID() == s.self: return errSelf case s.netrestrict != nil && !s.netrestrict.Contains(n.IP()): return errNotWhitelisted diff --git a/p2p/dial_test.go b/p2p/dial_test.go index 2de2c59995..f41ab77524 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -89,7 +89,7 @@ func (t fakeTable) ReadRandomNodes(buf []*enode.Node) int { return copy(buf, t) // This test checks that dynamic dials are launched from discovery results. func TestDialStateDynDial(t *testing.T) { runDialTest(t, dialtest{ - init: newDialState(nil, nil, fakeTable{}, 5, nil), + init: newDialState(enode.ID{}, nil, nil, fakeTable{}, 5, nil), rounds: []round{ // A discovery query is launched. { @@ -236,7 +236,7 @@ func TestDialStateDynDialBootnode(t *testing.T) { newNode(uintID(8), nil), } runDialTest(t, dialtest{ - init: newDialState(nil, bootnodes, table, 5, nil), + init: newDialState(enode.ID{}, nil, bootnodes, table, 5, nil), rounds: []round{ // 2 dynamic dials attempted, bootnodes pending fallback interval { @@ -324,7 +324,7 @@ func TestDialStateDynDialFromTable(t *testing.T) { } runDialTest(t, dialtest{ - init: newDialState(nil, nil, table, 10, nil), + init: newDialState(enode.ID{}, nil, nil, table, 10, nil), rounds: []round{ // 5 out of 8 of the nodes returned by ReadRandomNodes are dialed. { @@ -430,7 +430,7 @@ func TestDialStateNetRestrict(t *testing.T) { restrict.Add("127.0.2.0/24") runDialTest(t, dialtest{ - init: newDialState(nil, nil, table, 10, restrict), + init: newDialState(enode.ID{}, nil, nil, table, 10, restrict), rounds: []round{ { new: []task{ @@ -453,7 +453,7 @@ func TestDialStateStaticDial(t *testing.T) { } runDialTest(t, dialtest{ - init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil), rounds: []round{ // Static dials are launched for the nodes that // aren't yet connected. @@ -557,7 +557,7 @@ func TestDialStaticAfterReset(t *testing.T) { }, } dTest := dialtest{ - init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil), rounds: rounds, } runDialTest(t, dTest) @@ -578,7 +578,7 @@ func TestDialStateCache(t *testing.T) { } runDialTest(t, dialtest{ - init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil), rounds: []round{ // Static dials are launched for the nodes that // aren't yet connected. @@ -640,7 +640,7 @@ func TestDialStateCache(t *testing.T) { func TestDialResolve(t *testing.T) { resolved := newNode(uintID(1), net.IP{127, 0, 55, 234}) table := &resolveMock{answer: resolved} - state := newDialState(nil, nil, table, 0, nil) + state := newDialState(enode.ID{}, nil, nil, table, 0, nil) // Check that the task is generated with an incomplete ID. dest := newNode(uintID(1), nil) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 7a3e41de18..afd4c9a272 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -72,21 +72,20 @@ type Table struct { ips netutil.DistinctNetSet db *enode.DB // database of known nodes + net transport refreshReq chan chan struct{} initDone chan struct{} closeReq chan struct{} closed chan struct{} nodeAddedHook func(*node) // for testing - - net transport - self *node // metadata of the local node } // transport is implemented by the UDP transport. // it is an interface so we can test without opening lots of UDP // sockets and without generating a private key. type transport interface { + self() *enode.Node ping(enode.ID, *net.UDPAddr) error findnode(toid enode.ID, addr *net.UDPAddr, target encPubkey) ([]*node, error) close() @@ -100,11 +99,10 @@ type bucket struct { ips netutil.DistinctNetSet } -func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.Node) (*Table, error) { +func newTable(t transport, db *enode.DB, bootnodes []*enode.Node) (*Table, error) { tab := &Table{ net: t, db: db, - self: wrapNode(self), refreshReq: make(chan chan struct{}), initDone: make(chan struct{}), closeReq: make(chan struct{}), @@ -127,6 +125,10 @@ func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.No return tab, nil } +func (tab *Table) self() *enode.Node { + return tab.net.self() +} + func (tab *Table) seedRand() { var b [8]byte crand.Read(b[:]) @@ -136,11 +138,6 @@ func (tab *Table) seedRand() { tab.mutex.Unlock() } -// Self returns the local node. -func (tab *Table) Self() *enode.Node { - return unwrapNode(tab.self) -} - // ReadRandomNodes fills the given slice with random nodes from the table. The results // are guaranteed to be unique for a single invocation, no node will appear twice. func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) { @@ -183,6 +180,10 @@ func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) { // Close terminates the network listener and flushes the node database. func (tab *Table) Close() { + if tab.net != nil { + tab.net.close() + } + select { case <-tab.closed: // already closed. @@ -257,7 +258,7 @@ func (tab *Table) lookup(targetKey encPubkey, refreshIfEmpty bool) []*node { ) // don't query further if we hit ourself. // unlikely to happen often in practice. - asked[tab.self.ID()] = true + asked[tab.self().ID()] = true for { tab.mutex.Lock() @@ -340,8 +341,8 @@ func (tab *Table) loop() { revalidate = time.NewTimer(tab.nextRevalidateTime()) refresh = time.NewTicker(refreshInterval) copyNodes = time.NewTicker(copyNodesInterval) - revalidateDone = make(chan struct{}) refreshDone = make(chan struct{}) // where doRefresh reports completion + revalidateDone chan struct{} // where doRevalidate reports completion waiting = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs ) defer refresh.Stop() @@ -372,9 +373,11 @@ loop: } waiting, refreshDone = nil, nil case <-revalidate.C: + revalidateDone = make(chan struct{}) go tab.doRevalidate(revalidateDone) case <-revalidateDone: revalidate.Reset(tab.nextRevalidateTime()) + revalidateDone = nil case <-copyNodes.C: go tab.copyLiveNodes() case <-tab.closeReq: @@ -382,15 +385,15 @@ loop: } } - if tab.net != nil { - tab.net.close() - } if refreshDone != nil { <-refreshDone } for _, ch := range waiting { close(ch) } + if revalidateDone != nil { + <-revalidateDone + } close(tab.closed) } @@ -408,7 +411,7 @@ func (tab *Table) doRefresh(done chan struct{}) { // Run self lookup to discover new neighbor nodes. // We can only do this if we have a secp256k1 identity. var key ecdsa.PublicKey - if err := tab.self.Load((*enode.Secp256k1)(&key)); err == nil { + if err := tab.self().Load((*enode.Secp256k1)(&key)); err == nil { tab.lookup(encodePubkey(&key), false) } @@ -530,7 +533,7 @@ func (tab *Table) len() (n int) { // bucket returns the bucket for the given node ID hash. func (tab *Table) bucket(id enode.ID) *bucket { - d := enode.LogDist(tab.self.ID(), id) + d := enode.LogDist(tab.self().ID(), id) if d <= bucketMinDistance { return tab.buckets[0] } @@ -543,7 +546,7 @@ func (tab *Table) bucket(id enode.ID) *bucket { // // The caller must not hold tab.mutex. func (tab *Table) add(n *node) { - if n.ID() == tab.self.ID() { + if n.ID() == tab.self().ID() { return } @@ -576,7 +579,7 @@ func (tab *Table) stuff(nodes []*node) { defer tab.mutex.Unlock() for _, n := range nodes { - if n.ID() == tab.self.ID() { + if n.ID() == tab.self().ID() { continue // don't add self } b := tab.bucket(n.ID()) diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index e8631024b6..6b4cd2d186 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -141,7 +141,7 @@ func TestTable_IPLimit(t *testing.T) { defer db.Close() for i := 0; i < tableIPLimit+1; i++ { - n := nodeAtDistance(tab.self.ID(), i, net.IP{172, 0, 1, byte(i)}) + n := nodeAtDistance(tab.self().ID(), i, net.IP{172, 0, 1, byte(i)}) tab.add(n) } if tab.len() > tableIPLimit { @@ -158,7 +158,7 @@ func TestTable_BucketIPLimit(t *testing.T) { d := 3 for i := 0; i < bucketIPLimit+1; i++ { - n := nodeAtDistance(tab.self.ID(), d, net.IP{172, 0, 1, byte(i)}) + n := nodeAtDistance(tab.self().ID(), d, net.IP{172, 0, 1, byte(i)}) tab.add(n) } if tab.len() > bucketIPLimit { @@ -240,7 +240,7 @@ func TestTable_ReadRandomNodesGetAll(t *testing.T) { for i := 0; i < len(buf); i++ { ld := cfg.Rand.Intn(len(tab.buckets)) - tab.stuff([]*node{nodeAtDistance(tab.self.ID(), ld, intIP(ld))}) + tab.stuff([]*node{nodeAtDistance(tab.self().ID(), ld, intIP(ld))}) } gotN := tab.ReadRandomNodes(buf) if gotN != tab.len() { @@ -510,6 +510,10 @@ type preminedTestnet struct { dists [hashBits + 1][]encPubkey } +func (tn *preminedTestnet) self() *enode.Node { + return nullNode +} + func (tn *preminedTestnet) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) { // current log distance is encoded in port number // fmt.Println("findnode query at dist", toaddr.Port) diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 05ae0b6c07..d415194521 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -28,12 +28,17 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" ) -func newTestTable(t transport) (*Table, *enode.DB) { +var nullNode *enode.Node + +func init() { var r enr.Record r.Set(enr.IP{0, 0, 0, 0}) - n := enode.SignNull(&r, enode.ID{}) + nullNode = enode.SignNull(&r, enode.ID{}) +} + +func newTestTable(t transport) (*Table, *enode.DB) { db, _ := enode.OpenDB("") - tab, _ := newTable(t, n, db, nil) + tab, _ := newTable(t, db, nil) return tab, db } @@ -70,10 +75,10 @@ func intIP(i int) net.IP { // fillBucket inserts nodes into the given bucket until it is full. func fillBucket(tab *Table, n *node) (last *node) { - ld := enode.LogDist(tab.self.ID(), n.ID()) + ld := enode.LogDist(tab.self().ID(), n.ID()) b := tab.bucket(n.ID()) for len(b.entries) < bucketSize { - b.entries = append(b.entries, nodeAtDistance(tab.self.ID(), ld, intIP(ld))) + b.entries = append(b.entries, nodeAtDistance(tab.self().ID(), ld, intIP(ld))) } return b.entries[bucketSize-1] } @@ -81,15 +86,25 @@ func fillBucket(tab *Table, n *node) (last *node) { type pingRecorder struct { mu sync.Mutex dead, pinged map[enode.ID]bool + n *enode.Node } func newPingRecorder() *pingRecorder { + var r enr.Record + r.Set(enr.IP{0, 0, 0, 0}) + n := enode.SignNull(&r, enode.ID{}) + return &pingRecorder{ dead: make(map[enode.ID]bool), pinged: make(map[enode.ID]bool), + n: n, } } +func (t *pingRecorder) self() *enode.Node { + return nullNode +} + func (t *pingRecorder) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) { return nil, nil } diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index 45fcce282c..37a0449021 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -23,12 +23,12 @@ import ( "errors" "fmt" "net" + "sync" "time" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/rlp" ) @@ -118,9 +118,11 @@ type ( ) func makeEndpoint(addr *net.UDPAddr, tcpPort uint16) rpcEndpoint { - ip := addr.IP.To4() - if ip == nil { - ip = addr.IP.To16() + ip := net.IP{} + if ip4 := addr.IP.To4(); ip4 != nil { + ip = ip4 + } else if ip6 := addr.IP.To16(); ip6 != nil { + ip = ip6 } return rpcEndpoint{IP: ip, UDP: uint16(addr.Port), TCP: tcpPort} } @@ -165,20 +167,19 @@ type conn interface { LocalAddr() net.Addr } -// udp implements the RPC protocol. +// udp implements the discovery v4 UDP wire protocol. type udp struct { conn conn netrestrict *netutil.Netlist priv *ecdsa.PrivateKey - ourEndpoint rpcEndpoint + localNode *enode.LocalNode + db *enode.DB + tab *Table + wg sync.WaitGroup addpending chan *pending gotreply chan reply - - closing chan struct{} - nat nat.Interface - - *Table + closing chan struct{} } // pending represents a pending reply. @@ -230,60 +231,57 @@ type Config struct { PrivateKey *ecdsa.PrivateKey // These settings are optional: - AnnounceAddr *net.UDPAddr // local address announced in the DHT - NodeDBPath string // if set, the node database is stored at this filesystem location - NetRestrict *netutil.Netlist // network whitelist - Bootnodes []*enode.Node // list of bootstrap nodes - Unhandled chan<- ReadPacket // unhandled packets are sent on this channel + NetRestrict *netutil.Netlist // network whitelist + Bootnodes []*enode.Node // list of bootstrap nodes + Unhandled chan<- ReadPacket // unhandled packets are sent on this channel } // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(c conn, cfg Config) (*Table, error) { - tab, _, err := newUDP(c, cfg) +func ListenUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, error) { + tab, _, err := newUDP(c, ln, cfg) if err != nil { return nil, err } - log.Info("UDP listener up", "self", tab.self) return tab, nil } -func newUDP(c conn, cfg Config) (*Table, *udp, error) { - realaddr := c.LocalAddr().(*net.UDPAddr) - if cfg.AnnounceAddr != nil { - realaddr = cfg.AnnounceAddr - } - self := enode.NewV4(&cfg.PrivateKey.PublicKey, realaddr.IP, realaddr.Port, realaddr.Port) - db, err := enode.OpenDB(cfg.NodeDBPath) - if err != nil { - return nil, nil, err - } - +func newUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, *udp, error) { udp := &udp{ conn: c, priv: cfg.PrivateKey, netrestrict: cfg.NetRestrict, + localNode: ln, + db: ln.Database(), closing: make(chan struct{}), gotreply: make(chan reply), addpending: make(chan *pending), } - // TODO: separate TCP port - udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port)) - tab, err := newTable(udp, self, db, cfg.Bootnodes) + tab, err := newTable(udp, ln.Database(), cfg.Bootnodes) if err != nil { return nil, nil, err } - udp.Table = tab + udp.tab = tab + udp.wg.Add(2) go udp.loop() go udp.readLoop(cfg.Unhandled) - return udp.Table, udp, nil + return udp.tab, udp, nil +} + +func (t *udp) self() *enode.Node { + return t.localNode.Node() } func (t *udp) close() { close(t.closing) t.conn.Close() - t.db.Close() - // TODO: wait for the loops to end. + t.wg.Wait() +} + +func (t *udp) ourEndpoint() rpcEndpoint { + n := t.self() + a := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} + return makeEndpoint(a, uint16(n.TCP())) } // ping sends a ping message to the given node and waits for a reply. @@ -296,7 +294,7 @@ func (t *udp) ping(toid enode.ID, toaddr *net.UDPAddr) error { func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-chan error { req := &ping{ Version: 4, - From: t.ourEndpoint, + From: t.ourEndpoint(), To: makeEndpoint(toaddr, 0), // TODO: maybe use known TCP port from DB Expiration: uint64(time.Now().Add(expiration).Unix()), } @@ -313,6 +311,7 @@ func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-ch } return ok }) + t.localNode.UDPContact(toaddr) t.write(toaddr, req.name(), packet) return errc } @@ -381,6 +380,8 @@ func (t *udp) handleReply(from enode.ID, ptype byte, req packet) bool { // loop runs in its own goroutine. it keeps track of // the refresh timer and the pending reply queue. func (t *udp) loop() { + defer t.wg.Done() + var ( plist = list.New() timeout = time.NewTimer(0) @@ -542,10 +543,11 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (packet, // readLoop runs in its own goroutine. it handles incoming UDP packets. func (t *udp) readLoop(unhandled chan<- ReadPacket) { - defer t.conn.Close() + defer t.wg.Done() if unhandled != nil { defer close(unhandled) } + // Discovery packets are defined to be no larger than 1280 bytes. // Packets larger than this size will be cut at the end and treated // as invalid because their hash won't match. @@ -629,10 +631,11 @@ func (req *ping) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte n := wrapNode(enode.NewV4(key, from.IP, int(req.From.TCP), from.Port)) t.handleReply(n.ID(), pingPacket, req) if time.Since(t.db.LastPongReceived(n.ID())) > bondExpiration { - t.sendPing(n.ID(), from, func() { t.addThroughPing(n) }) + t.sendPing(n.ID(), from, func() { t.tab.addThroughPing(n) }) } else { - t.addThroughPing(n) + t.tab.addThroughPing(n) } + t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) t.db.UpdateLastPingReceived(n.ID(), time.Now()) return nil } @@ -647,6 +650,7 @@ func (req *pong) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte if !t.handleReply(fromID, pongPacket, req) { return errUnsolicitedReply } + t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) t.db.UpdateLastPongReceived(fromID, time.Now()) return nil } @@ -668,9 +672,9 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac [] return errUnknownNode } target := enode.ID(crypto.Keccak256Hash(req.Target[:])) - t.mutex.Lock() - closest := t.closest(target, bucketSize).entries - t.mutex.Unlock() + t.tab.mutex.Lock() + closest := t.tab.closest(target, bucketSize).entries + t.tab.mutex.Unlock() p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())} var sent bool diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go index da95c4f5c4..a4ddaf750d 100644 --- a/p2p/discover/udp_test.go +++ b/p2p/discover/udp_test.go @@ -71,7 +71,9 @@ func newUDPTest(t *testing.T) *udpTest { remotekey: newkey(), remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, } - test.table, test.udp, _ = newUDP(test.pipe, Config{PrivateKey: test.localkey}) + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, test.localkey) + test.table, test.udp, _ = newUDP(test.pipe, ln, Config{PrivateKey: test.localkey}) // Wait for initial refresh so the table doesn't send unexpected findnode. <-test.table.initDone return test @@ -355,12 +357,13 @@ func TestUDP_successfulPing(t *testing.T) { // remote is unknown, the table pings back. hash, _ := test.waitPacketOut(func(p *ping) error { - if !reflect.DeepEqual(p.From, test.udp.ourEndpoint) { - t.Errorf("got ping.From %v, want %v", p.From, test.udp.ourEndpoint) + if !reflect.DeepEqual(p.From, test.udp.ourEndpoint()) { + t.Errorf("got ping.From %#v, want %#v", p.From, test.udp.ourEndpoint()) } wantTo := rpcEndpoint{ // The mirrored UDP address is the UDP packet sender. - IP: test.remoteaddr.IP, UDP: uint16(test.remoteaddr.Port), + IP: test.remoteaddr.IP, + UDP: uint16(test.remoteaddr.Port), TCP: 0, } if !reflect.DeepEqual(p.To, wantTo) { diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index 49e1cb811a..ff5ed983ba 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -230,7 +230,8 @@ type udp struct { } // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { +func ListenUDP(priv *ecdsa.PrivateKey, conn conn, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { + realaddr := conn.LocalAddr().(*net.UDPAddr) transport, err := listenUDP(priv, conn, realaddr) if err != nil { return nil, err diff --git a/p2p/enode/localnode.go b/p2p/enode/localnode.go new file mode 100644 index 0000000000..623f8eae18 --- /dev/null +++ b/p2p/enode/localnode.go @@ -0,0 +1,246 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enode + +import ( + "crypto/ecdsa" + "fmt" + "net" + "reflect" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/netutil" +) + +const ( + // IP tracker configuration + iptrackMinStatements = 10 + iptrackWindow = 5 * time.Minute + iptrackContactWindow = 10 * time.Minute +) + +// LocalNode produces the signed node record of a local node, i.e. a node run in the +// current process. Setting ENR entries via the Set method updates the record. A new version +// of the record is signed on demand when the Node method is called. +type LocalNode struct { + cur atomic.Value // holds a non-nil node pointer while the record is up-to-date. + id ID + key *ecdsa.PrivateKey + db *DB + + // everything below is protected by a lock + mu sync.Mutex + seq uint64 + entries map[string]enr.Entry + udpTrack *netutil.IPTracker // predicts external UDP endpoint + staticIP net.IP + fallbackIP net.IP + fallbackUDP int +} + +// NewLocalNode creates a local node. +func NewLocalNode(db *DB, key *ecdsa.PrivateKey) *LocalNode { + ln := &LocalNode{ + id: PubkeyToIDV4(&key.PublicKey), + db: db, + key: key, + udpTrack: netutil.NewIPTracker(iptrackWindow, iptrackContactWindow, iptrackMinStatements), + entries: make(map[string]enr.Entry), + } + ln.seq = db.localSeq(ln.id) + ln.invalidate() + return ln +} + +// Database returns the node database associated with the local node. +func (ln *LocalNode) Database() *DB { + return ln.db +} + +// Node returns the current version of the local node record. +func (ln *LocalNode) Node() *Node { + n := ln.cur.Load().(*Node) + if n != nil { + return n + } + // Record was invalidated, sign a new copy. + ln.mu.Lock() + defer ln.mu.Unlock() + ln.sign() + return ln.cur.Load().(*Node) +} + +// ID returns the local node ID. +func (ln *LocalNode) ID() ID { + return ln.id +} + +// Set puts the given entry into the local record, overwriting +// any existing value. +func (ln *LocalNode) Set(e enr.Entry) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.set(e) +} + +func (ln *LocalNode) set(e enr.Entry) { + val, exists := ln.entries[e.ENRKey()] + if !exists || !reflect.DeepEqual(val, e) { + ln.entries[e.ENRKey()] = e + ln.invalidate() + } +} + +// Delete removes the given entry from the local record. +func (ln *LocalNode) Delete(e enr.Entry) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.delete(e) +} + +func (ln *LocalNode) delete(e enr.Entry) { + _, exists := ln.entries[e.ENRKey()] + if exists { + delete(ln.entries, e.ENRKey()) + ln.invalidate() + } +} + +// SetStaticIP sets the local IP to the given one unconditionally. +// This disables endpoint prediction. +func (ln *LocalNode) SetStaticIP(ip net.IP) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.staticIP = ip + ln.updateEndpoints() +} + +// SetFallbackIP sets the last-resort IP address. This address is used +// if no endpoint prediction can be made and no static IP is set. +func (ln *LocalNode) SetFallbackIP(ip net.IP) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.fallbackIP = ip + ln.updateEndpoints() +} + +// SetFallbackUDP sets the last-resort UDP port. This port is used +// if no endpoint prediction can be made. +func (ln *LocalNode) SetFallbackUDP(port int) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.fallbackUDP = port + ln.updateEndpoints() +} + +// UDPEndpointStatement should be called whenever a statement about the local node's +// UDP endpoint is received. It feeds the local endpoint predictor. +func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint *net.UDPAddr) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.udpTrack.AddStatement(fromaddr.String(), endpoint.String()) + ln.updateEndpoints() +} + +// UDPContact should be called whenever the local node has announced itself to another node +// via UDP. It feeds the local endpoint predictor. +func (ln *LocalNode) UDPContact(toaddr *net.UDPAddr) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.udpTrack.AddContact(toaddr.String()) + ln.updateEndpoints() +} + +func (ln *LocalNode) updateEndpoints() { + // Determine the endpoints. + newIP := ln.fallbackIP + newUDP := ln.fallbackUDP + if ln.staticIP != nil { + newIP = ln.staticIP + } else if ip, port := predictAddr(ln.udpTrack); ip != nil { + newIP = ip + newUDP = port + } + + // Update the record. + if newIP != nil && !newIP.IsUnspecified() { + ln.set(enr.IP(newIP)) + if newUDP != 0 { + ln.set(enr.UDP(newUDP)) + } else { + ln.delete(enr.UDP(0)) + } + } else { + ln.delete(enr.IP{}) + } +} + +// predictAddr wraps IPTracker.PredictEndpoint, converting from its string-based +// endpoint representation to IP and port types. +func predictAddr(t *netutil.IPTracker) (net.IP, int) { + ep := t.PredictEndpoint() + if ep == "" { + return nil, 0 + } + ipString, portString, _ := net.SplitHostPort(ep) + ip := net.ParseIP(ipString) + port, _ := strconv.Atoi(portString) + return ip, port +} + +func (ln *LocalNode) invalidate() { + ln.cur.Store((*Node)(nil)) +} + +func (ln *LocalNode) sign() { + if n := ln.cur.Load().(*Node); n != nil { + return // no changes + } + + var r enr.Record + for _, e := range ln.entries { + r.Set(e) + } + ln.bumpSeq() + r.SetSeq(ln.seq) + if err := SignV4(&r, ln.key); err != nil { + panic(fmt.Errorf("enode: can't sign record: %v", err)) + } + n, err := New(ValidSchemes, &r) + if err != nil { + panic(fmt.Errorf("enode: can't verify local record: %v", err)) + } + ln.cur.Store(n) + log.Info("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IP(), "udp", n.UDP(), "tcp", n.TCP()) +} + +func (ln *LocalNode) bumpSeq() { + ln.seq++ + ln.db.storeLocalSeq(ln.id, ln.seq) +} diff --git a/p2p/enode/localnode_test.go b/p2p/enode/localnode_test.go new file mode 100644 index 0000000000..f5e3496d63 --- /dev/null +++ b/p2p/enode/localnode_test.go @@ -0,0 +1,76 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enode + +import ( + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/enr" +) + +func newLocalNodeForTesting() (*LocalNode, *DB) { + db, _ := OpenDB("") + key, _ := crypto.GenerateKey() + return NewLocalNode(db, key), db +} + +func TestLocalNode(t *testing.T) { + ln, db := newLocalNodeForTesting() + defer db.Close() + + if ln.Node().ID() != ln.ID() { + t.Fatal("inconsistent ID") + } + + ln.Set(enr.WithEntry("x", uint(3))) + var x uint + if err := ln.Node().Load(enr.WithEntry("x", &x)); err != nil { + t.Fatal("can't load entry 'x':", err) + } else if x != 3 { + t.Fatal("wrong value for entry 'x':", x) + } +} + +func TestLocalNodeSeqPersist(t *testing.T) { + ln, db := newLocalNodeForTesting() + defer db.Close() + + if s := ln.Node().Seq(); s != 1 { + t.Fatalf("wrong initial seq %d, want 1", s) + } + ln.Set(enr.WithEntry("x", uint(1))) + if s := ln.Node().Seq(); s != 2 { + t.Fatalf("wrong seq %d after set, want 2", s) + } + + // Create a new instance, it should reload the sequence number. + // The number increases just after that because a new record is + // created without the "x" entry. + ln2 := NewLocalNode(db, ln.key) + if s := ln2.Node().Seq(); s != 3 { + t.Fatalf("wrong seq %d on new instance, want 3", s) + } + + // Create a new instance with a different node key on the same database. + // This should reset the sequence number. + key, _ := crypto.GenerateKey() + ln3 := NewLocalNode(db, key) + if s := ln3.Node().Seq(); s != 1 { + t.Fatalf("wrong seq %d on instance with changed key, want 1", s) + } +} diff --git a/p2p/enode/node.go b/p2p/enode/node.go index 84088fcd26..b454ab2554 100644 --- a/p2p/enode/node.go +++ b/p2p/enode/node.go @@ -98,6 +98,13 @@ func (n *Node) Pubkey() *ecdsa.PublicKey { return &key } +// Record returns the node's record. The return value is a copy and may +// be modified by the caller. +func (n *Node) Record() *enr.Record { + cpy := n.r + return &cpy +} + // checks whether n is a valid complete node. func (n *Node) ValidateComplete() error { if n.Incomplete() { diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index a929b75d7b..7ee0c09a93 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -35,11 +35,24 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) +// Keys in the node database. +const ( + dbVersionKey = "version" // Version of the database to flush if changes + dbItemPrefix = "n:" // Identifier to prefix node entries with + + dbDiscoverRoot = ":discover" + dbDiscoverSeq = dbDiscoverRoot + ":seq" + dbDiscoverPing = dbDiscoverRoot + ":lastping" + dbDiscoverPong = dbDiscoverRoot + ":lastpong" + dbDiscoverFindFails = dbDiscoverRoot + ":findfail" + dbLocalRoot = ":local" + dbLocalSeq = dbLocalRoot + ":seq" +) + var ( - nodeDBNilID = ID{} // Special node ID to use as a nil element. - nodeDBNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped. - nodeDBCleanupCycle = time.Hour // Time period for running the expiration task. - nodeDBVersion = 6 + dbNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped. + dbCleanupCycle = time.Hour // Time period for running the expiration task. + dbVersion = 7 ) // DB is the node database, storing previously seen nodes and any collected metadata about @@ -50,17 +63,6 @@ type DB struct { quit chan struct{} // Channel to signal the expiring thread to stop } -// Schema layout for the node database -var ( - nodeDBVersionKey = []byte("version") // Version of the database to flush if changes - nodeDBItemPrefix = []byte("n:") // Identifier to prefix node entries with - - nodeDBDiscoverRoot = ":discover" - nodeDBDiscoverPing = nodeDBDiscoverRoot + ":lastping" - nodeDBDiscoverPong = nodeDBDiscoverRoot + ":lastpong" - nodeDBDiscoverFindFails = nodeDBDiscoverRoot + ":findfail" -) - // OpenDB opens a node database for storing and retrieving infos about known peers in the // network. If no path is given an in-memory, temporary database is constructed. func OpenDB(path string) (*DB, error) { @@ -93,13 +95,13 @@ func newPersistentDB(path string) (*DB, error) { // The nodes contained in the cache correspond to a certain protocol version. // Flush all nodes if the version doesn't match. currentVer := make([]byte, binary.MaxVarintLen64) - currentVer = currentVer[:binary.PutVarint(currentVer, int64(nodeDBVersion))] + currentVer = currentVer[:binary.PutVarint(currentVer, int64(dbVersion))] - blob, err := db.Get(nodeDBVersionKey, nil) + blob, err := db.Get([]byte(dbVersionKey), nil) switch err { case leveldb.ErrNotFound: // Version not found (i.e. empty cache), insert it - if err := db.Put(nodeDBVersionKey, currentVer, nil); err != nil { + if err := db.Put([]byte(dbVersionKey), currentVer, nil); err != nil { db.Close() return nil, err } @@ -120,28 +122,27 @@ func newPersistentDB(path string) (*DB, error) { // makeKey generates the leveldb key-blob from a node id and its particular // field of interest. func makeKey(id ID, field string) []byte { - if bytes.Equal(id[:], nodeDBNilID[:]) { + if (id == ID{}) { return []byte(field) } - return append(nodeDBItemPrefix, append(id[:], field...)...) + return append([]byte(dbItemPrefix), append(id[:], field...)...) } // splitKey tries to split a database key into a node id and a field part. func splitKey(key []byte) (id ID, field string) { // If the key is not of a node, return it plainly - if !bytes.HasPrefix(key, nodeDBItemPrefix) { + if !bytes.HasPrefix(key, []byte(dbItemPrefix)) { return ID{}, string(key) } // Otherwise split the id and field - item := key[len(nodeDBItemPrefix):] + item := key[len(dbItemPrefix):] copy(id[:], item[:len(id)]) field = string(item[len(id):]) return id, field } -// fetchInt64 retrieves an integer instance associated with a particular -// database key. +// fetchInt64 retrieves an integer associated with a particular key. func (db *DB) fetchInt64(key []byte) int64 { blob, err := db.lvl.Get(key, nil) if err != nil { @@ -154,18 +155,33 @@ func (db *DB) fetchInt64(key []byte) int64 { return val } -// storeInt64 update a specific database entry to the current time instance as a -// unix timestamp. +// storeInt64 stores an integer in the given key. func (db *DB) storeInt64(key []byte, n int64) error { blob := make([]byte, binary.MaxVarintLen64) blob = blob[:binary.PutVarint(blob, n)] + return db.lvl.Put(key, blob, nil) +} +// fetchUint64 retrieves an integer associated with a particular key. +func (db *DB) fetchUint64(key []byte) uint64 { + blob, err := db.lvl.Get(key, nil) + if err != nil { + return 0 + } + val, _ := binary.Uvarint(blob) + return val +} + +// storeUint64 stores an integer in the given key. +func (db *DB) storeUint64(key []byte, n uint64) error { + blob := make([]byte, binary.MaxVarintLen64) + blob = blob[:binary.PutUvarint(blob, n)] return db.lvl.Put(key, blob, nil) } // Node retrieves a node with a given id from the database. func (db *DB) Node(id ID) *Node { - blob, err := db.lvl.Get(makeKey(id, nodeDBDiscoverRoot), nil) + blob, err := db.lvl.Get(makeKey(id, dbDiscoverRoot), nil) if err != nil { return nil } @@ -184,11 +200,31 @@ func mustDecodeNode(id, data []byte) *Node { // UpdateNode inserts - potentially overwriting - a node into the peer database. func (db *DB) UpdateNode(node *Node) error { + if node.Seq() < db.NodeSeq(node.ID()) { + return nil + } blob, err := rlp.EncodeToBytes(&node.r) if err != nil { return err } - return db.lvl.Put(makeKey(node.ID(), nodeDBDiscoverRoot), blob, nil) + if err := db.lvl.Put(makeKey(node.ID(), dbDiscoverRoot), blob, nil); err != nil { + return err + } + return db.storeUint64(makeKey(node.ID(), dbDiscoverSeq), node.Seq()) +} + +// NodeSeq returns the stored record sequence number of the given node. +func (db *DB) NodeSeq(id ID) uint64 { + return db.fetchUint64(makeKey(id, dbDiscoverSeq)) +} + +// Resolve returns the stored record of the node if it has a larger sequence +// number than n. +func (db *DB) Resolve(n *Node) *Node { + if n.Seq() > db.NodeSeq(n.ID()) { + return n + } + return db.Node(n.ID()) } // DeleteNode deletes all information/keys associated with a node. @@ -218,7 +254,7 @@ func (db *DB) ensureExpirer() { // expirer should be started in a go routine, and is responsible for looping ad // infinitum and dropping stale data from the database. func (db *DB) expirer() { - tick := time.NewTicker(nodeDBCleanupCycle) + tick := time.NewTicker(dbCleanupCycle) defer tick.Stop() for { select { @@ -235,7 +271,7 @@ func (db *DB) expirer() { // expireNodes iterates over the database and deletes all nodes that have not // been seen (i.e. received a pong from) for some allotted time. func (db *DB) expireNodes() error { - threshold := time.Now().Add(-nodeDBNodeExpiration) + threshold := time.Now().Add(-dbNodeExpiration) // Find discovered nodes that are older than the allowance it := db.lvl.NewIterator(nil, nil) @@ -244,7 +280,7 @@ func (db *DB) expireNodes() error { for it.Next() { // Skip the item if not a discovery node id, field := splitKey(it.Key()) - if field != nodeDBDiscoverRoot { + if field != dbDiscoverRoot { continue } // Skip the node if not expired yet (and not self) @@ -260,34 +296,44 @@ func (db *DB) expireNodes() error { // LastPingReceived retrieves the time of the last ping packet received from // a remote node. func (db *DB) LastPingReceived(id ID) time.Time { - return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPing)), 0) + return time.Unix(db.fetchInt64(makeKey(id, dbDiscoverPing)), 0) } // UpdateLastPingReceived updates the last time we tried contacting a remote node. func (db *DB) UpdateLastPingReceived(id ID, instance time.Time) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix()) + return db.storeInt64(makeKey(id, dbDiscoverPing), instance.Unix()) } // LastPongReceived retrieves the time of the last successful pong from remote node. func (db *DB) LastPongReceived(id ID) time.Time { // Launch expirer db.ensureExpirer() - return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0) + return time.Unix(db.fetchInt64(makeKey(id, dbDiscoverPong)), 0) } // UpdateLastPongReceived updates the last pong time of a node. func (db *DB) UpdateLastPongReceived(id ID, instance time.Time) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix()) + return db.storeInt64(makeKey(id, dbDiscoverPong), instance.Unix()) } // FindFails retrieves the number of findnode failures since bonding. func (db *DB) FindFails(id ID) int { - return int(db.fetchInt64(makeKey(id, nodeDBDiscoverFindFails))) + return int(db.fetchInt64(makeKey(id, dbDiscoverFindFails))) } // UpdateFindFails updates the number of findnode failures since bonding. func (db *DB) UpdateFindFails(id ID, fails int) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverFindFails), int64(fails)) + return db.storeInt64(makeKey(id, dbDiscoverFindFails), int64(fails)) +} + +// LocalSeq retrieves the local record sequence counter. +func (db *DB) localSeq(id ID) uint64 { + return db.fetchUint64(makeKey(id, dbLocalSeq)) +} + +// storeLocalSeq stores the local record sequence counter. +func (db *DB) storeLocalSeq(id ID, n uint64) { + db.storeUint64(makeKey(id, dbLocalSeq), n) } // QuerySeeds retrieves random nodes to be used as potential seed nodes @@ -309,7 +355,7 @@ seek: ctr := id[0] rand.Read(id[:]) id[0] = ctr + id[0]%16 - it.Seek(makeKey(id, nodeDBDiscoverRoot)) + it.Seek(makeKey(id, dbDiscoverRoot)) n := nextNode(it) if n == nil { @@ -334,7 +380,7 @@ seek: func nextNode(it iterator.Iterator) *Node { for end := false; !end; end = !it.Next() { id, field := splitKey(it.Key()) - if field != nodeDBDiscoverRoot { + if field != dbDiscoverRoot { continue } return mustDecodeNode(id[:], it.Value()) diff --git a/p2p/enode/nodedb_test.go b/p2p/enode/nodedb_test.go index b476a3439d..96794827cf 100644 --- a/p2p/enode/nodedb_test.go +++ b/p2p/enode/nodedb_test.go @@ -332,7 +332,7 @@ var nodeDBExpirationNodes = []struct { 30303, 30303, ), - pong: time.Now().Add(-nodeDBNodeExpiration + time.Minute), + pong: time.Now().Add(-dbNodeExpiration + time.Minute), exp: false, }, { node: NewV4( @@ -341,7 +341,7 @@ var nodeDBExpirationNodes = []struct { 30303, 30303, ), - pong: time.Now().Add(-nodeDBNodeExpiration - time.Minute), + pong: time.Now().Add(-dbNodeExpiration - time.Minute), exp: true, }, } diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index 251caf4585..444820c158 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -156,7 +156,7 @@ func (r *Record) Set(e Entry) { } func (r *Record) invalidate() { - if r.signature == nil { + if r.signature != nil { r.seq++ } r.signature = nil diff --git a/p2p/enr/enr_test.go b/p2p/enr/enr_test.go index 9bf22478dd..449c898a84 100644 --- a/p2p/enr/enr_test.go +++ b/p2p/enr/enr_test.go @@ -169,6 +169,18 @@ func TestDirty(t *testing.T) { } } +func TestSeq(t *testing.T) { + var r Record + + assert.Equal(t, uint64(0), r.Seq()) + r.Set(UDP(1)) + assert.Equal(t, uint64(0), r.Seq()) + signTest([]byte{5}, &r) + assert.Equal(t, uint64(0), r.Seq()) + r.Set(UDP(2)) + assert.Equal(t, uint64(1), r.Seq()) +} + // TestGetSetOverwrite tests value overwrite when setting a new value with an existing key in record. func TestGetSetOverwrite(t *testing.T) { var r Record diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index a254648c66..8fad921c48 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -129,21 +129,15 @@ func Map(m Interface, c chan struct{}, protocol string, extport, intport int, na // ExtIP assumes that the local machine is reachable on the given // external IP address, and that any required ports were mapped manually. // Mapping operations will not return an error but won't actually do anything. -func ExtIP(ip net.IP) Interface { - if ip == nil { - panic("IP must not be nil") - } - return extIP(ip) -} +type ExtIP net.IP -type extIP net.IP - -func (n extIP) ExternalIP() (net.IP, error) { return net.IP(n), nil } -func (n extIP) String() string { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) } +func (n ExtIP) ExternalIP() (net.IP, error) { return net.IP(n), nil } +func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) } // These do nothing. -func (extIP) AddMapping(string, int, int, string, time.Duration) error { return nil } -func (extIP) DeleteMapping(string, int, int) error { return nil } + +func (ExtIP) AddMapping(string, int, int, string, time.Duration) error { return nil } +func (ExtIP) DeleteMapping(string, int, int) error { return nil } // Any returns a port mapper that tries to discover any supported // mechanism on the local network. diff --git a/p2p/nat/nat_test.go b/p2p/nat/nat_test.go index 469101e997..814e6d9e14 100644 --- a/p2p/nat/nat_test.go +++ b/p2p/nat/nat_test.go @@ -28,7 +28,7 @@ import ( func TestAutoDiscRace(t *testing.T) { ad := startautodisc("thing", func() Interface { time.Sleep(500 * time.Millisecond) - return extIP{33, 44, 55, 66} + return ExtIP{33, 44, 55, 66} }) // Spawn a few concurrent calls to ad.ExternalIP. diff --git a/p2p/netutil/iptrack.go b/p2p/netutil/iptrack.go new file mode 100644 index 0000000000..b9cbd5e1ca --- /dev/null +++ b/p2p/netutil/iptrack.go @@ -0,0 +1,130 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package netutil + +import ( + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +// IPTracker predicts the external endpoint, i.e. IP address and port, of the local host +// based on statements made by other hosts. +type IPTracker struct { + window time.Duration + contactWindow time.Duration + minStatements int + clock mclock.Clock + statements map[string]ipStatement + contact map[string]mclock.AbsTime + lastStatementGC mclock.AbsTime + lastContactGC mclock.AbsTime +} + +type ipStatement struct { + endpoint string + time mclock.AbsTime +} + +// NewIPTracker creates an IP tracker. +// +// The window parameters configure the amount of past network events which are kept. The +// minStatements parameter enforces a minimum number of statements which must be recorded +// before any prediction is made. Higher values for these parameters decrease 'flapping' of +// predictions as network conditions change. Window duration values should typically be in +// the range of minutes. +func NewIPTracker(window, contactWindow time.Duration, minStatements int) *IPTracker { + return &IPTracker{ + window: window, + contactWindow: contactWindow, + statements: make(map[string]ipStatement), + minStatements: minStatements, + contact: make(map[string]mclock.AbsTime), + clock: mclock.System{}, + } +} + +// PredictFullConeNAT checks whether the local host is behind full cone NAT. It predicts by +// checking whether any statement has been received from a node we didn't contact before +// the statement was made. +func (it *IPTracker) PredictFullConeNAT() bool { + now := it.clock.Now() + it.gcContact(now) + it.gcStatements(now) + for host, st := range it.statements { + if c, ok := it.contact[host]; !ok || c > st.time { + return true + } + } + return false +} + +// PredictEndpoint returns the current prediction of the external endpoint. +func (it *IPTracker) PredictEndpoint() string { + it.gcStatements(it.clock.Now()) + + // The current strategy is simple: find the endpoint with most statements. + counts := make(map[string]int) + maxcount, max := 0, "" + for _, s := range it.statements { + c := counts[s.endpoint] + 1 + counts[s.endpoint] = c + if c > maxcount && c >= it.minStatements { + maxcount, max = c, s.endpoint + } + } + return max +} + +// AddStatement records that a certain host thinks our external endpoint is the one given. +func (it *IPTracker) AddStatement(host, endpoint string) { + now := it.clock.Now() + it.statements[host] = ipStatement{endpoint, now} + if time.Duration(now-it.lastStatementGC) >= it.window { + it.gcStatements(now) + } +} + +// AddContact records that a packet containing our endpoint information has been sent to a +// certain host. +func (it *IPTracker) AddContact(host string) { + now := it.clock.Now() + it.contact[host] = now + if time.Duration(now-it.lastContactGC) >= it.contactWindow { + it.gcContact(now) + } +} + +func (it *IPTracker) gcStatements(now mclock.AbsTime) { + it.lastStatementGC = now + cutoff := now.Add(-it.window) + for host, s := range it.statements { + if s.time < cutoff { + delete(it.statements, host) + } + } +} + +func (it *IPTracker) gcContact(now mclock.AbsTime) { + it.lastContactGC = now + cutoff := now.Add(-it.contactWindow) + for host, ct := range it.contact { + if ct < cutoff { + delete(it.contact, host) + } + } +} diff --git a/p2p/netutil/iptrack_test.go b/p2p/netutil/iptrack_test.go new file mode 100644 index 0000000000..a9a2998a65 --- /dev/null +++ b/p2p/netutil/iptrack_test.go @@ -0,0 +1,138 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package netutil + +import ( + "fmt" + mrand "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +const ( + opStatement = iota + opContact + opPredict + opCheckFullCone +) + +type iptrackTestEvent struct { + op int + time int // absolute, in milliseconds + ip, from string +} + +func TestIPTracker(t *testing.T) { + tests := map[string][]iptrackTestEvent{ + "minStatements": { + {opPredict, 0, "", ""}, + {opStatement, 0, "127.0.0.1", "127.0.0.2"}, + {opPredict, 1000, "", ""}, + {opStatement, 1000, "127.0.0.1", "127.0.0.3"}, + {opPredict, 1000, "", ""}, + {opStatement, 1000, "127.0.0.1", "127.0.0.4"}, + {opPredict, 1000, "127.0.0.1", ""}, + }, + "window": { + {opStatement, 0, "127.0.0.1", "127.0.0.2"}, + {opStatement, 2000, "127.0.0.1", "127.0.0.3"}, + {opStatement, 3000, "127.0.0.1", "127.0.0.4"}, + {opPredict, 10000, "127.0.0.1", ""}, + {opPredict, 10001, "", ""}, // first statement expired + {opStatement, 10100, "127.0.0.1", "127.0.0.2"}, + {opPredict, 10200, "127.0.0.1", ""}, + }, + "fullcone": { + {opContact, 0, "", "127.0.0.2"}, + {opStatement, 10, "127.0.0.1", "127.0.0.2"}, + {opContact, 2000, "", "127.0.0.3"}, + {opStatement, 2010, "127.0.0.1", "127.0.0.3"}, + {opContact, 3000, "", "127.0.0.4"}, + {opStatement, 3010, "127.0.0.1", "127.0.0.4"}, + {opCheckFullCone, 3500, "false", ""}, + }, + "fullcone_2": { + {opContact, 0, "", "127.0.0.2"}, + {opStatement, 10, "127.0.0.1", "127.0.0.2"}, + {opContact, 2000, "", "127.0.0.3"}, + {opStatement, 2010, "127.0.0.1", "127.0.0.3"}, + {opStatement, 3000, "127.0.0.1", "127.0.0.4"}, + {opContact, 3010, "", "127.0.0.4"}, + {opCheckFullCone, 3500, "true", ""}, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { runIPTrackerTest(t, test) }) + } +} + +func runIPTrackerTest(t *testing.T, evs []iptrackTestEvent) { + var ( + clock mclock.Simulated + it = NewIPTracker(10*time.Second, 10*time.Second, 3) + ) + it.clock = &clock + for i, ev := range evs { + evtime := time.Duration(ev.time) * time.Millisecond + clock.Run(evtime - time.Duration(clock.Now())) + switch ev.op { + case opStatement: + it.AddStatement(ev.from, ev.ip) + case opContact: + it.AddContact(ev.from) + case opPredict: + if pred := it.PredictEndpoint(); pred != ev.ip { + t.Errorf("op %d: wrong prediction %q, want %q", i, pred, ev.ip) + } + case opCheckFullCone: + pred := fmt.Sprintf("%t", it.PredictFullConeNAT()) + if pred != ev.ip { + t.Errorf("op %d: wrong prediction %s, want %s", i, pred, ev.ip) + } + } + } +} + +// This checks that old statements and contacts are GCed even if Predict* isn't called. +func TestIPTrackerForceGC(t *testing.T) { + var ( + clock mclock.Simulated + window = 10 * time.Second + rate = 50 * time.Millisecond + max = int(window/rate) + 1 + it = NewIPTracker(window, window, 3) + ) + it.clock = &clock + + for i := 0; i < 5*max; i++ { + e1 := make([]byte, 4) + e2 := make([]byte, 4) + mrand.Read(e1) + mrand.Read(e2) + it.AddStatement(string(e1), string(e2)) + it.AddContact(string(e1)) + clock.Run(rate) + } + if len(it.contact) > 2*max { + t.Errorf("contacts not GCed, have %d", len(it.contact)) + } + if len(it.statements) > 2*max { + t.Errorf("statements not GCed, have %d", len(it.statements)) + } +} diff --git a/p2p/protocol.go b/p2p/protocol.go index 4b90a2a708..9438ab8e47 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" ) // Protocol represents a P2P subprotocol implementation. @@ -52,6 +53,9 @@ type Protocol struct { // about a certain peer in the network. If an info retrieval function is set, // but returns nil, it is assumed that the protocol handshake is still running. PeerInfo func(id enode.ID) interface{} + + // Attributes contains protocol specific information for the node record. + Attributes []enr.Entry } func (p Protocol) cap() Cap { @@ -64,10 +68,6 @@ type Cap struct { Version uint } -func (cap Cap) RlpData() interface{} { - return []interface{}{cap.Name, cap.Version} -} - func (cap Cap) String() string { return fmt.Sprintf("%s/%d", cap.Name, cap.Version) } @@ -79,3 +79,5 @@ func (cs capsByNameAndVersion) Swap(i, j int) { cs[i], cs[j] = cs[j], cs[i] } func (cs capsByNameAndVersion) Less(i, j int) bool { return cs[i].Name < cs[j].Name || (cs[i].Name == cs[j].Name && cs[i].Version < cs[j].Version) } + +func (capsByNameAndVersion) ENRKey() string { return "cap" } diff --git a/p2p/server.go b/p2p/server.go index 40db758e2d..6482c04019 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -20,9 +20,11 @@ package p2p import ( "bytes" "crypto/ecdsa" + "encoding/hex" "errors" "fmt" "net" + "sort" "sync" "sync/atomic" "time" @@ -35,8 +37,10 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" + "github.com/ethereum/go-ethereum/rlp" ) const ( @@ -160,6 +164,8 @@ type Server struct { lock sync.Mutex // protects running running bool + nodedb *enode.DB + localnode *enode.LocalNode ntab discoverTable listener net.Listener ourHandshake *protoHandshake @@ -347,43 +353,13 @@ func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription { // Self returns the local node's endpoint information. func (srv *Server) Self() *enode.Node { srv.lock.Lock() - running, listener, ntab := srv.running, srv.listener, srv.ntab + ln := srv.localnode srv.lock.Unlock() - if !running { + if ln == nil { return enode.NewV4(&srv.PrivateKey.PublicKey, net.ParseIP("0.0.0.0"), 0, 0) } - return srv.makeSelf(listener, ntab) -} - -func (srv *Server) makeSelf(listener net.Listener, ntab discoverTable) *enode.Node { - // If the node is running but discovery is off, manually assemble the node infos. - if ntab == nil { - addr := srv.tcpAddr(listener) - return enode.NewV4(&srv.PrivateKey.PublicKey, addr.IP, addr.Port, 0) - } - // Otherwise return the discovery node. - return ntab.Self() -} - -func (srv *Server) tcpAddr(listener net.Listener) net.TCPAddr { - addr := net.TCPAddr{IP: net.IP{0, 0, 0, 0}} - if listener == nil { - return addr // Inbound connections disabled, use zero address. - } - // Otherwise inject the listener address too. - if a, ok := listener.Addr().(*net.TCPAddr); ok { - addr = *a - } - if srv.NAT != nil { - if ip, err := srv.NAT.ExternalIP(); err == nil { - addr.IP = ip - } - } - if addr.IP.IsUnspecified() { - addr.IP = net.IP{127, 0, 0, 1} - } - return addr + return ln.Node() } // Stop terminates the server and all active peer connections. @@ -443,7 +419,9 @@ func (srv *Server) Start() (err error) { if srv.log == nil { srv.log = log.New() } - srv.log.Info("Starting P2P networking") + if srv.NoDial && srv.ListenAddr == "" { + srv.log.Warn("P2P server will be useless, neither dialing nor listening") + } // static fields if srv.PrivateKey == nil { @@ -466,65 +444,120 @@ func (srv *Server) Start() (err error) { srv.peerOp = make(chan peerOpFunc) srv.peerOpDone = make(chan struct{}) - var ( - conn *net.UDPConn - sconn *sharedUDPConn - realaddr *net.UDPAddr - unhandled chan discover.ReadPacket - ) - - if !srv.NoDiscovery || srv.DiscoveryV5 { - addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) - if err != nil { + if err := srv.setupLocalNode(); err != nil { + return err + } + if srv.ListenAddr != "" { + if err := srv.setupListening(); err != nil { return err } - conn, err = net.ListenUDP("udp", addr) - if err != nil { - return err - } - realaddr = conn.LocalAddr().(*net.UDPAddr) - if srv.NAT != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - // TODO: react to external IP changes over time. - if ext, err := srv.NAT.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} - } - } + } + if err := srv.setupDiscovery(); err != nil { + return err } - if !srv.NoDiscovery && srv.DiscoveryV5 { - unhandled = make(chan discover.ReadPacket, 100) - sconn = &sharedUDPConn{conn, unhandled} + dynPeers := srv.maxDialedConns() + dialer := newDialState(srv.localnode.ID(), srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) + srv.loopWG.Add(1) + go srv.run(dialer) + return nil +} + +func (srv *Server) setupLocalNode() error { + // Create the devp2p handshake. + pubkey := crypto.FromECDSAPub(&srv.PrivateKey.PublicKey) + srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: pubkey[1:]} + for _, p := range srv.Protocols { + srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) + } + sort.Sort(capsByNameAndVersion(srv.ourHandshake.Caps)) + + // Create the local node. + db, err := enode.OpenDB(srv.Config.NodeDatabase) + if err != nil { + return err + } + srv.nodedb = db + srv.localnode = enode.NewLocalNode(db, srv.PrivateKey) + srv.localnode.SetFallbackIP(net.IP{127, 0, 0, 1}) + srv.localnode.Set(capsByNameAndVersion(srv.ourHandshake.Caps)) + // TODO: check conflicts + for _, p := range srv.Protocols { + for _, e := range p.Attributes { + srv.localnode.Set(e) + } + } + switch srv.NAT.(type) { + case nil: + // No NAT interface, do nothing. + case nat.ExtIP: + // ExtIP doesn't block, set the IP right away. + ip, _ := srv.NAT.ExternalIP() + srv.localnode.SetStaticIP(ip) + default: + // Ask the router about the IP. This takes a while and blocks startup, + // do it in the background. + srv.loopWG.Add(1) + go func() { + defer srv.loopWG.Done() + if ip, err := srv.NAT.ExternalIP(); err == nil { + srv.localnode.SetStaticIP(ip) + } + }() + } + return nil +} + +func (srv *Server) setupDiscovery() error { + if srv.NoDiscovery && !srv.DiscoveryV5 { + return nil } - // node table + addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) + if err != nil { + return err + } + conn, err := net.ListenUDP("udp", addr) + if err != nil { + return err + } + realaddr := conn.LocalAddr().(*net.UDPAddr) + srv.log.Debug("UDP listener up", "addr", realaddr) + if srv.NAT != nil { + if !realaddr.IP.IsLoopback() { + go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + } + } + srv.localnode.SetFallbackUDP(realaddr.Port) + + // Discovery V4 + var unhandled chan discover.ReadPacket + var sconn *sharedUDPConn if !srv.NoDiscovery { - cfg := discover.Config{ - PrivateKey: srv.PrivateKey, - AnnounceAddr: realaddr, - NodeDBPath: srv.NodeDatabase, - NetRestrict: srv.NetRestrict, - Bootnodes: srv.BootstrapNodes, - Unhandled: unhandled, + if srv.DiscoveryV5 { + unhandled = make(chan discover.ReadPacket, 100) + sconn = &sharedUDPConn{conn, unhandled} } - ntab, err := discover.ListenUDP(conn, cfg) + cfg := discover.Config{ + PrivateKey: srv.PrivateKey, + NetRestrict: srv.NetRestrict, + Bootnodes: srv.BootstrapNodes, + Unhandled: unhandled, + } + ntab, err := discover.ListenUDP(conn, srv.localnode, cfg) if err != nil { return err } srv.ntab = ntab } - + // Discovery V5 if srv.DiscoveryV5 { - var ( - ntab *discv5.Network - err error - ) + var ntab *discv5.Network + var err error if sconn != nil { - ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, "", srv.NetRestrict) } else { - ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, "", srv.NetRestrict) } if err != nil { return err @@ -534,32 +567,10 @@ func (srv *Server) Start() (err error) { } srv.DiscV5 = ntab } - - dynPeers := srv.maxDialedConns() - dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) - - // handshake - pubkey := crypto.FromECDSAPub(&srv.PrivateKey.PublicKey) - srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: pubkey[1:]} - for _, p := range srv.Protocols { - srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) - } - // listen/dial - if srv.ListenAddr != "" { - if err := srv.startListening(); err != nil { - return err - } - } - if srv.NoDial && srv.ListenAddr == "" { - srv.log.Warn("P2P server will be useless, neither dialing nor listening") - } - - srv.loopWG.Add(1) - go srv.run(dialer) return nil } -func (srv *Server) startListening() error { +func (srv *Server) setupListening() error { // Launch the TCP listener. listener, err := net.Listen("tcp", srv.ListenAddr) if err != nil { @@ -568,8 +579,11 @@ func (srv *Server) startListening() error { laddr := listener.Addr().(*net.TCPAddr) srv.ListenAddr = laddr.String() srv.listener = listener + srv.localnode.Set(enr.TCP(laddr.Port)) + srv.loopWG.Add(1) go srv.listenLoop() + // Map the TCP listening port if NAT is configured. if !laddr.IP.IsLoopback() && srv.NAT != nil { srv.loopWG.Add(1) @@ -589,7 +603,10 @@ type dialer interface { } func (srv *Server) run(dialstate dialer) { + srv.log.Info("Started P2P networking", "self", srv.localnode.Node()) defer srv.loopWG.Done() + defer srv.nodedb.Close() + var ( peers = make(map[enode.ID]*Peer) inboundCount = 0 @@ -781,7 +798,7 @@ func (srv *Server) encHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int return DiscTooManyPeers case peers[c.node.ID()] != nil: return DiscAlreadyConnected - case c.node.ID() == srv.Self().ID(): + case c.node.ID() == srv.localnode.ID(): return DiscSelf default: return nil @@ -802,15 +819,11 @@ func (srv *Server) maxDialedConns() int { return srv.MaxPeers / r } -type tempError interface { - Temporary() bool -} - // listenLoop runs in its own goroutine and accepts // inbound connections. func (srv *Server) listenLoop() { defer srv.loopWG.Done() - srv.log.Info("RLPx listener up", "self", srv.Self()) + srv.log.Debug("TCP listener up", "addr", srv.listener.Addr()) tokens := defaultMaxPendingPeers if srv.MaxPendingPeers > 0 { @@ -831,7 +844,7 @@ func (srv *Server) listenLoop() { ) for { fd, err = srv.listener.Accept() - if tempErr, ok := err.(tempError); ok && tempErr.Temporary() { + if netutil.IsTemporaryError(err) { srv.log.Debug("Temporary read error", "err", err) continue } else if err != nil { @@ -864,10 +877,6 @@ func (srv *Server) listenLoop() { // as a peer. It returns when the connection has been added as a peer // or the handshakes have failed. func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { - self := srv.Self() - if self == nil { - return errors.New("shutdown") - } c := &conn{fd: fd, transport: srv.newTransport(fd), flags: flags, cont: make(chan error)} err := srv.setupConn(c, flags, dialDest) if err != nil { @@ -1003,6 +1012,7 @@ type NodeInfo struct { ID string `json:"id"` // Unique node identifier (also the encryption key) Name string `json:"name"` // Name of the node, including client type, version, OS, custom data Enode string `json:"enode"` // Enode URL for adding this peer from remote peers + ENR string `json:"enr"` // Ethereum Node Record IP string `json:"ip"` // IP address of the node Ports struct { Discovery int `json:"discovery"` // UDP listening port for discovery protocol @@ -1014,9 +1024,8 @@ type NodeInfo struct { // NodeInfo gathers and returns a collection of metadata known about the host. func (srv *Server) NodeInfo() *NodeInfo { - node := srv.Self() - // Gather and assemble the generic node infos + node := srv.Self() info := &NodeInfo{ Name: srv.Name, Enode: node.String(), @@ -1027,6 +1036,9 @@ func (srv *Server) NodeInfo() *NodeInfo { } info.Ports.Discovery = node.UDP() info.Ports.Listener = node.TCP() + if enc, err := rlp.EncodeToBytes(node.Record()); err == nil { + info.ENR = "0x" + hex.EncodeToString(enc) + } // Gather all the running protocol infos (only once per protocol type) for _, proto := range srv.Protocols { diff --git a/p2p/server_test.go b/p2p/server_test.go index e0b1fc122e..7e11577d6c 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -225,12 +225,15 @@ func TestServerTaskScheduling(t *testing.T) { // The Server in this test isn't actually running // because we're only interested in what run does. + db, _ := enode.OpenDB("") srv := &Server{ - Config: Config{MaxPeers: 10}, - quit: make(chan struct{}), - ntab: fakeTable{}, - running: true, - log: log.New(), + Config: Config{MaxPeers: 10}, + localnode: enode.NewLocalNode(db, newkey()), + nodedb: db, + quit: make(chan struct{}), + ntab: fakeTable{}, + running: true, + log: log.New(), } srv.loopWG.Add(1) go func() { @@ -271,11 +274,14 @@ func TestServerManyTasks(t *testing.T) { } var ( - srv = &Server{ - quit: make(chan struct{}), - ntab: fakeTable{}, - running: true, - log: log.New(), + db, _ = enode.OpenDB("") + srv = &Server{ + quit: make(chan struct{}), + localnode: enode.NewLocalNode(db, newkey()), + nodedb: db, + ntab: fakeTable{}, + running: true, + log: log.New(), } done = make(chan *testTask) start, end = 0, 0 From 4868964bb99facd8cc6149626023e64db14a6742 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 12 Oct 2018 14:51:38 +0200 Subject: [PATCH 003/100] cmd/swarm: split flags and cli command declarations to the relevant files (#17896) --- cmd/swarm/access.go | 60 +++++- cmd/swarm/db.go | 42 ++++ cmd/swarm/download.go | 9 + cmd/swarm/feeds.go | 62 ++++++ cmd/swarm/flags.go | 179 +++++++++++++++++ cmd/swarm/fs.go | 37 ++++ cmd/swarm/hash.go | 9 + cmd/swarm/list.go | 9 + cmd/swarm/main.go | 438 ++---------------------------------------- cmd/swarm/manifest.go | 34 ++++ cmd/swarm/upload.go | 11 +- 11 files changed, 468 insertions(+), 422 deletions(-) create mode 100644 cmd/swarm/flags.go diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index dd2d513c24..629781edd8 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -29,7 +29,65 @@ import ( "gopkg.in/urfave/cli.v1" ) -var salt = make([]byte, 32) +var ( + salt = make([]byte, 32) + accessCommand = cli.Command{ + CustomHelpTemplate: helpTemplate, + Name: "access", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root manifest", + Subcommands: []cli.Command{ + { + CustomHelpTemplate: helpTemplate, + Name: "new", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + Subcommands: []cli.Command{ + { + Action: accessNewPass, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + }, + Name: "pass", + Usage: "encrypts a reference with a password and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewPK, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + SwarmAccessGrantKeyFlag, + }, + Name: "pk", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewACT, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + SwarmAccessGrantKeysFlag, + SwarmDryRunFlag, + utils.PasswordFileFlag, + }, + Name: "act", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + }, + }, + }, + } +) func init() { if _, err := io.ReadFull(rand.Reader, salt); err != nil { diff --git a/cmd/swarm/db.go b/cmd/swarm/db.go index 107fbf100c..7916beffce 100644 --- a/cmd/swarm/db.go +++ b/cmd/swarm/db.go @@ -29,6 +29,48 @@ import ( "gopkg.in/urfave/cli.v1" ) +var dbCommand = cli.Command{ + Name: "db", + CustomHelpTemplate: helpTemplate, + Usage: "manage the local chunk database", + ArgsUsage: "db COMMAND", + Description: "Manage the local chunk database", + Subcommands: []cli.Command{ + { + Action: dbExport, + CustomHelpTemplate: helpTemplate, + Name: "export", + Usage: "export a local chunk database as a tar archive (use - to send to stdout)", + ArgsUsage: " ", + Description: ` +Export a local chunk database as a tar archive (use - to send to stdout). + + swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar + +The export may be quite large, consider piping the output through the Unix +pv(1) tool to get a progress bar: + + swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar +`, + }, + { + Action: dbImport, + CustomHelpTemplate: helpTemplate, + Name: "import", + Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)", + ArgsUsage: " ", + Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin). + + swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar + +The import may be quite large, consider piping the input through the Unix +pv(1) tool to get a progress bar: + + pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, + }, + }, +} + func dbExport(ctx *cli.Context) { args := ctx.Args() if len(args) != 3 { diff --git a/cmd/swarm/download.go b/cmd/swarm/download.go index 91bc2c93ab..fcbefa0206 100644 --- a/cmd/swarm/download.go +++ b/cmd/swarm/download.go @@ -28,6 +28,15 @@ import ( "gopkg.in/urfave/cli.v1" ) +var downloadCommand = cli.Command{ + Action: download, + Name: "down", + Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag}, + Usage: "downloads a swarm manifest or a file inside a manifest", + ArgsUsage: " []", + Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`, +} + func download(ctx *cli.Context) { log.Debug("downloading content using swarm down") args := ctx.Args() diff --git a/cmd/swarm/feeds.go b/cmd/swarm/feeds.go index 6806c6cf47..f26a8cc7dc 100644 --- a/cmd/swarm/feeds.go +++ b/cmd/swarm/feeds.go @@ -31,6 +31,68 @@ import ( "gopkg.in/urfave/cli.v1" ) +var feedCommand = cli.Command{ + CustomHelpTemplate: helpTemplate, + Name: "feed", + Usage: "(Advanced) Create and update Swarm Feeds", + ArgsUsage: "", + Description: "Works with Swarm Feeds", + Subcommands: []cli.Command{ + { + Action: feedCreateManifest, + CustomHelpTemplate: helpTemplate, + Name: "create", + Usage: "creates and publishes a new feed manifest", + Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic. + The feed topic can be built in the following ways: + * use --topic to set the topic to an arbitrary binary hex string. + * use --name to set the topic to a human-readable name. + For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. + * use both --topic and --name to create named subtopics. + For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning + this feed tracks a discussion about that contract. + The --user flag allows to have this manifest refer to a user other than yourself. If not specified, + it will then default to your local account (--bzzaccount)`, + Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, + }, + { + Action: feedUpdate, + CustomHelpTemplate: helpTemplate, + Name: "update", + Usage: "updates the content of an existing Swarm Feed", + ArgsUsage: "<0x Hex data>", + Description: `publishes a new update on the specified topic + The feed topic can be built in the following ways: + * use --topic to set the topic to an arbitrary binary hex string. + * use --name to set the topic to a human-readable name. + For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. + * use both --topic and --name to create named subtopics. + For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning + this feed tracks a discussion about that contract. + + If you have a manifest, you can specify it with --manifest to refer to the feed, + instead of using --topic / --name + `, + Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag}, + }, + { + Action: feedInfo, + CustomHelpTemplate: helpTemplate, + Name: "info", + Usage: "obtains information about an existing Swarm feed", + Description: `obtains information about an existing Swarm feed + The topic can be specified directly with the --topic flag as an hex string + If no topic is specified, the default topic (zero) will be used + The --name flag can be used to specify subtopics with a specific name. + The --user flag allows to refer to a user other than yourself. If not specified, + it will then default to your local account (--bzzaccount) + If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user + to refer to the feed`, + Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, + }, + }, +} + func NewGenericSigner(ctx *cli.Context) feed.Signer { return feed.NewGenericSigner(getPrivKey(ctx)) } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go new file mode 100644 index 0000000000..7e891f3022 --- /dev/null +++ b/cmd/swarm/flags.go @@ -0,0 +1,179 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Command feed allows the user to create and update signed Swarm feeds +package main + +import cli "gopkg.in/urfave/cli.v1" + +var ( + ChequebookAddrFlag = cli.StringFlag{ + Name: "chequebook", + Usage: "chequebook contract address", + EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR, + } + SwarmAccountFlag = cli.StringFlag{ + Name: "bzzaccount", + Usage: "Swarm account key file", + EnvVar: SWARM_ENV_ACCOUNT, + } + SwarmListenAddrFlag = cli.StringFlag{ + Name: "httpaddr", + Usage: "Swarm HTTP API listening interface", + EnvVar: SWARM_ENV_LISTEN_ADDR, + } + SwarmPortFlag = cli.StringFlag{ + Name: "bzzport", + Usage: "Swarm local http api port", + EnvVar: SWARM_ENV_PORT, + } + SwarmNetworkIdFlag = cli.IntFlag{ + Name: "bzznetworkid", + Usage: "Network identifier (integer, default 3=swarm testnet)", + EnvVar: SWARM_ENV_NETWORK_ID, + } + SwarmSwapEnabledFlag = cli.BoolFlag{ + Name: "swap", + Usage: "Swarm SWAP enabled (default false)", + EnvVar: SWARM_ENV_SWAP_ENABLE, + } + SwarmSwapAPIFlag = cli.StringFlag{ + Name: "swap-api", + Usage: "URL of the Ethereum API provider to use to settle SWAP payments", + EnvVar: SWARM_ENV_SWAP_API, + } + SwarmSyncDisabledFlag = cli.BoolTFlag{ + Name: "nosync", + Usage: "Disable swarm syncing", + EnvVar: SWARM_ENV_SYNC_DISABLE, + } + SwarmSyncUpdateDelay = cli.DurationFlag{ + Name: "sync-update-delay", + Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", + EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, + } + SwarmMaxStreamPeerServersFlag = cli.IntFlag{ + Name: "max-stream-peer-servers", + Usage: "Limit of Stream peer servers, 0 denotes unlimited", + EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS, + Value: 10000, // A very large default value is possible as stream servers have very small memory footprint + } + SwarmLightNodeEnabled = cli.BoolFlag{ + Name: "lightnode", + Usage: "Enable Swarm LightNode (default false)", + EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE, + } + SwarmDeliverySkipCheckFlag = cli.BoolFlag{ + Name: "delivery-skip-check", + Usage: "Skip chunk delivery check (default false)", + EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK, + } + EnsAPIFlag = cli.StringSliceFlag{ + Name: "ens-api", + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", + EnvVar: SWARM_ENV_ENS_API, + } + SwarmApiFlag = cli.StringFlag{ + Name: "bzzapi", + Usage: "Swarm HTTP endpoint", + Value: "http://127.0.0.1:8500", + } + SwarmRecursiveFlag = cli.BoolFlag{ + Name: "recursive", + Usage: "Upload directories recursively", + } + SwarmWantManifestFlag = cli.BoolTFlag{ + Name: "manifest", + Usage: "Automatic manifest upload (default true)", + } + SwarmUploadDefaultPath = cli.StringFlag{ + Name: "defaultpath", + Usage: "path to file served for empty url path (none)", + } + SwarmAccessGrantKeyFlag = cli.StringFlag{ + Name: "grant-key", + Usage: "grants a given public key access to an ACT", + } + SwarmAccessGrantKeysFlag = cli.StringFlag{ + Name: "grant-keys", + Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT", + } + SwarmUpFromStdinFlag = cli.BoolFlag{ + Name: "stdin", + Usage: "reads data to be uploaded from stdin", + } + SwarmUploadMimeType = cli.StringFlag{ + Name: "mime", + Usage: "Manually specify MIME type", + } + SwarmEncryptedFlag = cli.BoolFlag{ + Name: "encrypt", + Usage: "use encrypted upload", + } + SwarmAccessPasswordFlag = cli.StringFlag{ + Name: "password", + Usage: "Password", + EnvVar: SWARM_ACCESS_PASSWORD, + } + SwarmDryRunFlag = cli.BoolFlag{ + Name: "dry-run", + Usage: "dry-run", + } + CorsStringFlag = cli.StringFlag{ + Name: "corsdomain", + Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", + EnvVar: SWARM_ENV_CORS, + } + SwarmStorePath = cli.StringFlag{ + Name: "store.path", + Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)", + EnvVar: SWARM_ENV_STORE_PATH, + } + SwarmStoreCapacity = cli.Uint64Flag{ + Name: "store.size", + Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)", + EnvVar: SWARM_ENV_STORE_CAPACITY, + } + SwarmStoreCacheCapacity = cli.UintFlag{ + Name: "store.cache.size", + Usage: "Number of recent chunks cached in memory (default 5000)", + EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, + } + SwarmCompressedFlag = cli.BoolFlag{ + Name: "compressed", + Usage: "Prints encryption keys in compressed form", + } + SwarmFeedNameFlag = cli.StringFlag{ + Name: "name", + Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", + } + SwarmFeedTopicFlag = cli.StringFlag{ + Name: "topic", + Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters", + } + SwarmFeedDataOnCreateFlag = cli.StringFlag{ + Name: "data", + Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x", + } + SwarmFeedManifestFlag = cli.StringFlag{ + Name: "manifest", + Usage: "Refers to the feed through a manifest", + } + SwarmFeedUserFlag = cli.StringFlag{ + Name: "user", + Usage: "Indicates the user who updates the feed", + } +) diff --git a/cmd/swarm/fs.go b/cmd/swarm/fs.go index 3dc38ca4da..b970b2e8c2 100644 --- a/cmd/swarm/fs.go +++ b/cmd/swarm/fs.go @@ -30,6 +30,43 @@ import ( "gopkg.in/urfave/cli.v1" ) +var fsCommand = cli.Command{ + Name: "fs", + CustomHelpTemplate: helpTemplate, + Usage: "perform FUSE operations", + ArgsUsage: "fs COMMAND", + Description: "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node", + Subcommands: []cli.Command{ + { + Action: mount, + CustomHelpTemplate: helpTemplate, + Name: "mount", + Flags: []cli.Flag{utils.IPCPathFlag}, + Usage: "mount a swarm hash to a mount point", + ArgsUsage: "swarm fs mount --ipcpath ", + Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", + }, + { + Action: unmount, + CustomHelpTemplate: helpTemplate, + Name: "unmount", + Flags: []cli.Flag{utils.IPCPathFlag}, + Usage: "unmount a swarmfs mount", + ArgsUsage: "swarm fs unmount --ipcpath ", + Description: "Unmounts a swarmfs mount residing at . This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", + }, + { + Action: listMounts, + CustomHelpTemplate: helpTemplate, + Name: "list", + Flags: []cli.Flag{utils.IPCPathFlag}, + Usage: "list swarmfs mounts", + ArgsUsage: "swarm fs list --ipcpath ", + Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", + }, + }, +} + func mount(cliContext *cli.Context) { args := cliContext.Args() if len(args) < 2 { diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index d679806e38..471feb53d6 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -27,6 +27,15 @@ import ( "gopkg.in/urfave/cli.v1" ) +var hashCommand = cli.Command{ + Action: hash, + CustomHelpTemplate: helpTemplate, + Name: "hash", + Usage: "print the swarm hash of a file or directory", + ArgsUsage: "", + Description: "Prints the swarm hash of file or directory", +} + func hash(ctx *cli.Context) { args := ctx.Args() if len(args) < 1 { diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go index 01b3f4ab6c..5d35154a57 100644 --- a/cmd/swarm/list.go +++ b/cmd/swarm/list.go @@ -27,6 +27,15 @@ import ( "gopkg.in/urfave/cli.v1" ) +var listCommand = cli.Command{ + Action: list, + CustomHelpTemplate: helpTemplate, + Name: "ls", + Usage: "list files and directories contained in a manifest", + ArgsUsage: " []", + Description: "Lists files and directories contained in a manifest", +} + func list(ctx *cli.Context) { args := ctx.Args() diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 88e6b0b0be..ccbb24eec3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -70,165 +70,6 @@ var ( gitCommit string // Git SHA1 commit hash of the release (set via linker flags) ) -var ( - ChequebookAddrFlag = cli.StringFlag{ - Name: "chequebook", - Usage: "chequebook contract address", - EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR, - } - SwarmAccountFlag = cli.StringFlag{ - Name: "bzzaccount", - Usage: "Swarm account key file", - EnvVar: SWARM_ENV_ACCOUNT, - } - SwarmListenAddrFlag = cli.StringFlag{ - Name: "httpaddr", - Usage: "Swarm HTTP API listening interface", - EnvVar: SWARM_ENV_LISTEN_ADDR, - } - SwarmPortFlag = cli.StringFlag{ - Name: "bzzport", - Usage: "Swarm local http api port", - EnvVar: SWARM_ENV_PORT, - } - SwarmNetworkIdFlag = cli.IntFlag{ - Name: "bzznetworkid", - Usage: "Network identifier (integer, default 3=swarm testnet)", - EnvVar: SWARM_ENV_NETWORK_ID, - } - SwarmSwapEnabledFlag = cli.BoolFlag{ - Name: "swap", - Usage: "Swarm SWAP enabled (default false)", - EnvVar: SWARM_ENV_SWAP_ENABLE, - } - SwarmSwapAPIFlag = cli.StringFlag{ - Name: "swap-api", - Usage: "URL of the Ethereum API provider to use to settle SWAP payments", - EnvVar: SWARM_ENV_SWAP_API, - } - SwarmSyncDisabledFlag = cli.BoolTFlag{ - Name: "nosync", - Usage: "Disable swarm syncing", - EnvVar: SWARM_ENV_SYNC_DISABLE, - } - SwarmSyncUpdateDelay = cli.DurationFlag{ - Name: "sync-update-delay", - Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", - EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, - } - SwarmMaxStreamPeerServersFlag = cli.IntFlag{ - Name: "max-stream-peer-servers", - Usage: "Limit of Stream peer servers, 0 denotes unlimited", - EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS, - Value: 10000, // A very large default value is possible as stream servers have very small memory footprint - } - SwarmLightNodeEnabled = cli.BoolFlag{ - Name: "lightnode", - Usage: "Enable Swarm LightNode (default false)", - EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE, - } - SwarmDeliverySkipCheckFlag = cli.BoolFlag{ - Name: "delivery-skip-check", - Usage: "Skip chunk delivery check (default false)", - EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK, - } - EnsAPIFlag = cli.StringSliceFlag{ - Name: "ens-api", - Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", - EnvVar: SWARM_ENV_ENS_API, - } - SwarmApiFlag = cli.StringFlag{ - Name: "bzzapi", - Usage: "Swarm HTTP endpoint", - Value: "http://127.0.0.1:8500", - } - SwarmRecursiveFlag = cli.BoolFlag{ - Name: "recursive", - Usage: "Upload directories recursively", - } - SwarmWantManifestFlag = cli.BoolTFlag{ - Name: "manifest", - Usage: "Automatic manifest upload (default true)", - } - SwarmUploadDefaultPath = cli.StringFlag{ - Name: "defaultpath", - Usage: "path to file served for empty url path (none)", - } - SwarmAccessGrantKeyFlag = cli.StringFlag{ - Name: "grant-key", - Usage: "grants a given public key access to an ACT", - } - SwarmAccessGrantKeysFlag = cli.StringFlag{ - Name: "grant-keys", - Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT", - } - SwarmUpFromStdinFlag = cli.BoolFlag{ - Name: "stdin", - Usage: "reads data to be uploaded from stdin", - } - SwarmUploadMimeType = cli.StringFlag{ - Name: "mime", - Usage: "Manually specify MIME type", - } - SwarmEncryptedFlag = cli.BoolFlag{ - Name: "encrypt", - Usage: "use encrypted upload", - } - SwarmAccessPasswordFlag = cli.StringFlag{ - Name: "password", - Usage: "Password", - EnvVar: SWARM_ACCESS_PASSWORD, - } - SwarmDryRunFlag = cli.BoolFlag{ - Name: "dry-run", - Usage: "dry-run", - } - CorsStringFlag = cli.StringFlag{ - Name: "corsdomain", - Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", - EnvVar: SWARM_ENV_CORS, - } - SwarmStorePath = cli.StringFlag{ - Name: "store.path", - Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)", - EnvVar: SWARM_ENV_STORE_PATH, - } - SwarmStoreCapacity = cli.Uint64Flag{ - Name: "store.size", - Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)", - EnvVar: SWARM_ENV_STORE_CAPACITY, - } - SwarmStoreCacheCapacity = cli.UintFlag{ - Name: "store.cache.size", - Usage: "Number of recent chunks cached in memory (default 5000)", - EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, - } - SwarmCompressedFlag = cli.BoolFlag{ - Name: "compressed", - Usage: "Prints encryption keys in compressed form", - } - SwarmFeedNameFlag = cli.StringFlag{ - Name: "name", - Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", - } - SwarmFeedTopicFlag = cli.StringFlag{ - Name: "topic", - Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters", - } - SwarmFeedDataOnCreateFlag = cli.StringFlag{ - Name: "data", - Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x", - } - SwarmFeedManifestFlag = cli.StringFlag{ - Name: "manifest", - Usage: "Refers to the feed through a manifest", - } - SwarmFeedUserFlag = cli.StringFlag{ - Name: "user", - Usage: "Indicates the user who updates the feed", - } -) - //declare a few constant error messages, useful for later error check comparisons in test var ( SWARM_ERR_NO_BZZACCOUNT = "bzzaccount option is required but not set; check your config file, command line or environment variables" @@ -279,267 +120,24 @@ func init() { Usage: "Print public key information", Description: "The output of this command is supposed to be machine-readable", }, - { - Action: upload, - CustomHelpTemplate: helpTemplate, - Name: "up", - Usage: "uploads a file or directory to swarm using the HTTP API", - ArgsUsage: "", - Flags: []cli.Flag{SwarmEncryptedFlag}, - Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash", - }, - { - CustomHelpTemplate: helpTemplate, - Name: "access", - Usage: "encrypts a reference and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root manifest", - Subcommands: []cli.Command{ - { - CustomHelpTemplate: helpTemplate, - Name: "new", - Usage: "encrypts a reference and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - Subcommands: []cli.Command{ - { - Action: accessNewPass, - CustomHelpTemplate: helpTemplate, - Flags: []cli.Flag{ - utils.PasswordFileFlag, - SwarmDryRunFlag, - }, - Name: "pass", - Usage: "encrypts a reference with a password and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - }, - { - Action: accessNewPK, - CustomHelpTemplate: helpTemplate, - Flags: []cli.Flag{ - utils.PasswordFileFlag, - SwarmDryRunFlag, - SwarmAccessGrantKeyFlag, - }, - Name: "pk", - Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - }, - { - Action: accessNewACT, - CustomHelpTemplate: helpTemplate, - Flags: []cli.Flag{ - SwarmAccessGrantKeysFlag, - SwarmDryRunFlag, - utils.PasswordFileFlag, - }, - Name: "act", - Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - }, - }, - }, - }, - }, - { - CustomHelpTemplate: helpTemplate, - Name: "feed", - Usage: "(Advanced) Create and update Swarm Feeds", - ArgsUsage: "", - Description: "Works with Swarm Feeds", - Subcommands: []cli.Command{ - { - Action: feedCreateManifest, - CustomHelpTemplate: helpTemplate, - Name: "create", - Usage: "creates and publishes a new feed manifest", - Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic. - The feed topic can be built in the following ways: - * use --topic to set the topic to an arbitrary binary hex string. - * use --name to set the topic to a human-readable name. - For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. - * use both --topic and --name to create named subtopics. - For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning - this feed tracks a discussion about that contract. - The --user flag allows to have this manifest refer to a user other than yourself. If not specified, - it will then default to your local account (--bzzaccount)`, - Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, - }, - { - Action: feedUpdate, - CustomHelpTemplate: helpTemplate, - Name: "update", - Usage: "updates the content of an existing Swarm Feed", - ArgsUsage: "<0x Hex data>", - Description: `publishes a new update on the specified topic - The feed topic can be built in the following ways: - * use --topic to set the topic to an arbitrary binary hex string. - * use --name to set the topic to a human-readable name. - For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. - * use both --topic and --name to create named subtopics. - For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning - this feed tracks a discussion about that contract. - - If you have a manifest, you can specify it with --manifest to refer to the feed, - instead of using --topic / --name - `, - Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag}, - }, - { - Action: feedInfo, - CustomHelpTemplate: helpTemplate, - Name: "info", - Usage: "obtains information about an existing Swarm feed", - Description: `obtains information about an existing Swarm feed - The topic can be specified directly with the --topic flag as an hex string - If no topic is specified, the default topic (zero) will be used - The --name flag can be used to specify subtopics with a specific name. - The --user flag allows to refer to a user other than yourself. If not specified, - it will then default to your local account (--bzzaccount) - If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user - to refer to the feed`, - Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, - }, - }, - }, - { - Action: list, - CustomHelpTemplate: helpTemplate, - Name: "ls", - Usage: "list files and directories contained in a manifest", - ArgsUsage: " []", - Description: "Lists files and directories contained in a manifest", - }, - { - Action: hash, - CustomHelpTemplate: helpTemplate, - Name: "hash", - Usage: "print the swarm hash of a file or directory", - ArgsUsage: "", - Description: "Prints the swarm hash of file or directory", - }, - { - Action: download, - Name: "down", - Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag}, - Usage: "downloads a swarm manifest or a file inside a manifest", - ArgsUsage: " []", - Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`, - }, - { - Name: "manifest", - CustomHelpTemplate: helpTemplate, - Usage: "perform operations on swarm manifests", - ArgsUsage: "COMMAND", - Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", - Subcommands: []cli.Command{ - { - Action: manifestAdd, - CustomHelpTemplate: helpTemplate, - Name: "add", - Usage: "add a new path to the manifest", - ArgsUsage: " ", - Description: "Adds a new path to the manifest", - }, - { - Action: manifestUpdate, - CustomHelpTemplate: helpTemplate, - Name: "update", - Usage: "update the hash for an already existing path in the manifest", - ArgsUsage: " ", - Description: "Update the hash for an already existing path in the manifest", - }, - { - Action: manifestRemove, - CustomHelpTemplate: helpTemplate, - Name: "remove", - Usage: "removes a path from the manifest", - ArgsUsage: " ", - Description: "Removes a path from the manifest", - }, - }, - }, - { - Name: "fs", - CustomHelpTemplate: helpTemplate, - Usage: "perform FUSE operations", - ArgsUsage: "fs COMMAND", - Description: "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node", - Subcommands: []cli.Command{ - { - Action: mount, - CustomHelpTemplate: helpTemplate, - Name: "mount", - Flags: []cli.Flag{utils.IPCPathFlag}, - Usage: "mount a swarm hash to a mount point", - ArgsUsage: "swarm fs mount --ipcpath ", - Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", - }, - { - Action: unmount, - CustomHelpTemplate: helpTemplate, - Name: "unmount", - Flags: []cli.Flag{utils.IPCPathFlag}, - Usage: "unmount a swarmfs mount", - ArgsUsage: "swarm fs unmount --ipcpath ", - Description: "Unmounts a swarmfs mount residing at . This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", - }, - { - Action: listMounts, - CustomHelpTemplate: helpTemplate, - Name: "list", - Flags: []cli.Flag{utils.IPCPathFlag}, - Usage: "list swarmfs mounts", - ArgsUsage: "swarm fs list --ipcpath ", - Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", - }, - }, - }, - { - Name: "db", - CustomHelpTemplate: helpTemplate, - Usage: "manage the local chunk database", - ArgsUsage: "db COMMAND", - Description: "Manage the local chunk database", - Subcommands: []cli.Command{ - { - Action: dbExport, - CustomHelpTemplate: helpTemplate, - Name: "export", - Usage: "export a local chunk database as a tar archive (use - to send to stdout)", - ArgsUsage: " ", - Description: ` -Export a local chunk database as a tar archive (use - to send to stdout). - - swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar - -The export may be quite large, consider piping the output through the Unix -pv(1) tool to get a progress bar: - - swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar -`, - }, - { - Action: dbImport, - CustomHelpTemplate: helpTemplate, - Name: "import", - Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)", - ArgsUsage: " ", - Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin). - - swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar - -The import may be quite large, consider piping the input through the Unix -pv(1) tool to get a progress bar: - - pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, - }, - }, - }, - + // See upload.go + upCommand, + // See access.go + accessCommand, + // See feeds.go + feedCommand, + // See list.go + listCommand, + // See hash.go + hashCommand, + // See download.go + downloadCommand, + // See manifest.go + manifestCommand, + // See fs.go + fsCommand, + // See db.go + dbCommand, // See config.go DumpConfigCommand, } diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 0216ffc1dd..312c72fa21 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -28,6 +28,40 @@ import ( "gopkg.in/urfave/cli.v1" ) +var manifestCommand = cli.Command{ + Name: "manifest", + CustomHelpTemplate: helpTemplate, + Usage: "perform operations on swarm manifests", + ArgsUsage: "COMMAND", + Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", + Subcommands: []cli.Command{ + { + Action: manifestAdd, + CustomHelpTemplate: helpTemplate, + Name: "add", + Usage: "add a new path to the manifest", + ArgsUsage: " ", + Description: "Adds a new path to the manifest", + }, + { + Action: manifestUpdate, + CustomHelpTemplate: helpTemplate, + Name: "update", + Usage: "update the hash for an already existing path in the manifest", + ArgsUsage: " ", + Description: "Update the hash for an already existing path in the manifest", + }, + { + Action: manifestRemove, + CustomHelpTemplate: helpTemplate, + Name: "remove", + Usage: "removes a path from the manifest", + ArgsUsage: " ", + Description: "Removes a path from the manifest", + }, + }, +} + // manifestAdd adds a new entry to the manifest at the given path. // New entry hash, the last argument, must be the hash of a manifest // with only one entry, which meta-data will be added to the original manifest. diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 2225127cf3..0dbe896e23 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -34,8 +34,17 @@ import ( "gopkg.in/urfave/cli.v1" ) -func upload(ctx *cli.Context) { +var upCommand = cli.Command{ + Action: upload, + CustomHelpTemplate: helpTemplate, + Name: "up", + Usage: "uploads a file or directory to swarm using the HTTP API", + ArgsUsage: "", + Flags: []cli.Flag{SwarmEncryptedFlag}, + Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash", +} +func upload(ctx *cli.Context) { args := ctx.Args() var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") From 862d6f2fbf55313805a96cf2e17cb2493e25aa3b Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 12 Oct 2018 16:24:00 +0200 Subject: [PATCH 004/100] cmd/swarm: Smoke test for Swarm Feed (#17892) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 323 ++++++++++++++++++ cmd/swarm/swarm-smoke/main.go | 16 +- cmd/swarm/swarm-smoke/upload_and_sync.go | 18 +- 3 files changed, 351 insertions(+), 6 deletions(-) create mode 100644 cmd/swarm/swarm-smoke/feed_upload_and_sync.go diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go new file mode 100644 index 0000000000..c7a1475d6c --- /dev/null +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -0,0 +1,323 @@ +package main + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/pborman/uuid" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/multihash" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + + colorable "github.com/mattn/go-colorable" + + cli "gopkg.in/urfave/cli.v1" +) + +const ( + feedRandomDataLength = 8 +) + +// TODO: retrieve with manifest + extract repeating code +func cliFeedUploadAndSync(c *cli.Context) error { + + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) + + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) + + generateEndpoints(scheme, cluster, from, to) + + log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing") + + // create a random private key to sign updates with and derive the address + pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") + if err != nil { + return err + } + defer pkFile.Close() + defer os.Remove(pkFile.Name()) + + privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976" + privKey, err := crypto.HexToECDSA(privkeyHex) + if err != nil { + return err + } + user := crypto.PubkeyToAddress(privKey.PublicKey) + userHex := hexutil.Encode(user.Bytes()) + + // save the private key to a file + _, err = io.WriteString(pkFile, privkeyHex) + if err != nil { + return err + } + + // keep hex strings for topic and subtopic + var topicHex string + var subTopicHex string + + // and create combination hex topics for bzz-feed retrieval + // xor'ed with topic (zero-value topic if no topic) + var subTopicOnlyHex string + var mergedSubTopicHex string + + // generate random topic and subtopic and put a hex on them + topicBytes, err := generateRandomData(feed.TopicLength) + topicHex = hexutil.Encode(topicBytes) + subTopicBytes, err := generateRandomData(8) + subTopicHex = hexutil.Encode(subTopicBytes) + if err != nil { + return err + } + mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes) + if err != nil { + return err + } + mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:]) + subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil) + if err != nil { + return err + } + subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:]) + + // create feed manifest, topic only + var out bytes.Buffer + cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic) + } + log.Debug("create topic feed", "manifest", manifestWithTopic) + out.Reset() + + // create feed manifest, subtopic only + cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest subtopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithSubTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic) + } + log.Debug("create subtopic feed", "manifest", manifestWithTopic) + out.Reset() + + // create feed manifest, merged topic + cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + log.Error(err.Error()) + return err + } + manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithMergedTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic) + } + log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic) + out.Reset() + + // create test data + data, err := generateRandomData(feedRandomDataLength) + if err != nil { + return err + } + h := md5.New() + h.Write(data) + dataHash := h.Sum(nil) + dataHex := hexutil.Encode(data) + + // update with topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update topic", "out", out) + out.Reset() + + // update with subtopic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest subtopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update subtopic", "out", out) + out.Reset() + + // update with merged topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest merged topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update mergedtopic", "out", out) + out.Reset() + + time.Sleep(3 * time.Second) + + // retrieve the data + wg := sync.WaitGroup{} + for _, endpoint := range endpoints { + // raw retrieve, topic only + for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { + wg.Add(1) + ruid := uuid.New()[:8] + go func(endpoint string, ruid string) { + for { + err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) + if err != nil { + continue + } + + wg.Done() + return + } + }(endpoint, ruid) + + } + } + wg.Wait() + log.Info("all endpoints synced random data successfully") + + // upload test file + log.Info("uploading to " + endpoints[0] + " and syncing") + + f, cleanup := generateRandomFile(filesize * 1000) + defer cleanup() + + hash, err := upload(f, endpoints[0]) + if err != nil { + return err + } + hashBytes, err := hexutil.Decode("0x" + hash) + if err != nil { + return err + } + multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes)) + + fileHash, err := digest(f) + if err != nil { + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) + + // update file with topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update topic", "out", out) + out.Reset() + + // update file with subtopic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update subtopic", "out", out) + out.Reset() + + // update file with merged topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update mergedtopic", "out", out) + out.Reset() + + time.Sleep(3 * time.Second) + + for _, endpoint := range endpoints { + + // manifest retrieve, topic only + for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { + wg.Add(1) + ruid := uuid.New()[:8] + go func(endpoint string, ruid string) { + for { + err := fetch(url, endpoint, fileHash, ruid) + if err != nil { + continue + } + + wg.Done() + return + } + }(endpoint, ruid) + } + + } + wg.Wait() + log.Info("all endpoints synced random file successfully") + + return nil +} + +func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + log.Trace("sleeping", "ruid", ruid) + time.Sleep(3 * time.Second) + + log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) + res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user) + if err != nil { + return err + } + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 70aee19229..97054dd479 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -21,7 +21,6 @@ import ( "sort" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" cli "gopkg.in/urfave/cli.v1" ) @@ -34,11 +33,10 @@ var ( filesize int from int to int + verbosity int ) func main() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) app := cli.NewApp() app.Name = "smoke-test" @@ -80,6 +78,12 @@ func main() { Usage: "file size for generated random file in KB", Destination: &filesize, }, + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity", + Destination: &verbosity, + }, } app.Commands = []cli.Command{ @@ -89,6 +93,12 @@ func main() { Usage: "upload and sync", Action: cliUploadAndSync, }, + { + Name: "feed_sync", + Aliases: []string{"f"}, + Usage: "feed update generate, upload and sync", + Action: cliFeedUploadAndSync, + }, } sort.Sort(cli.FlagsByName(app.Flags)) diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 5e0ff4b0f3..358141c7cc 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -19,7 +19,8 @@ package main import ( "bytes" "crypto/md5" - "crypto/rand" + crand "crypto/rand" + "errors" "fmt" "io" "io/ioutil" @@ -85,7 +86,6 @@ func cliUploadAndSync(c *cli.Context) error { wg := sync.WaitGroup{} for _, endpoint := range endpoints { - endpoint := endpoint ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { @@ -166,6 +166,18 @@ func digest(r io.Reader) ([]byte, error) { return h.Sum(nil), nil } +// generates random data in heap buffer +func generateRandomData(datasize int) ([]byte, error) { + b := make([]byte, datasize) + c, err := crand.Read(b) + if err != nil { + return nil, err + } else if c != datasize { + return nil, errors.New("short read") + } + return b, nil +} + // generateRandomFile is creating a temporary file with the requested byte size func generateRandomFile(size int) (f *os.File, teardown func()) { // create a tmp file @@ -181,7 +193,7 @@ func generateRandomFile(size int) (f *os.File, teardown func()) { } buf := make([]byte, size) - _, err = rand.Read(buf) + _, err = crand.Read(buf) if err != nil { panic(err) } From dc3c3fb1e177c5d01ae3ca63717130eea924271e Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 12 Oct 2018 16:25:38 +0200 Subject: [PATCH 005/100] swarm/storage: Add accessCnt for GC (#17845) --- swarm/storage/ldbstore.go | 290 +++++++++++++++++++++------------ swarm/storage/ldbstore_test.go | 219 ++++++++++++++++++++----- 2 files changed, 362 insertions(+), 147 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 2a7f51cb38..49508911f9 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -32,7 +32,6 @@ import ( "fmt" "io" "io/ioutil" - "sort" "sync" "github.com/ethereum/go-ethereum/metrics" @@ -44,8 +43,13 @@ import ( ) const ( - gcArrayFreeRatio = 0.1 - maxGCitems = 5000 // max number of items to be gc'd per call to collectGarbage() + defaultGCRatio = 10 + defaultMaxGCRound = 10000 + defaultMaxGCBatch = 5000 + + wEntryCnt = 1 << 0 + wIndexCnt = 1 << 1 + wAccessCnt = 1 << 2 ) var ( @@ -61,6 +65,7 @@ var ( keyData = byte(6) keyDistanceCnt = byte(7) keySchema = []byte{8} + keyGCIdx = byte(9) // access to chunk data index, used by garbage collection in ascending order from first entry ) var ( @@ -68,7 +73,7 @@ var ( ) type gcItem struct { - idx uint64 + idx *dpaDBIndex value uint64 idxKey []byte po uint8 @@ -89,6 +94,16 @@ func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams { } } +type garbage struct { + maxRound int // maximum number of chunks to delete in one garbage collection round + maxBatch int // maximum number of chunks to delete in one db request batch + ratio int // 1/x ratio to calculate the number of chunks to gc on a low capacity db + count int // number of chunks deleted in running round + target int // number of chunks to delete in running round + batch *dbBatch // the delete batch + runC chan struct{} // struct in chan means gc is NOT running +} + type LDBStore struct { db *LDBDatabase @@ -102,12 +117,12 @@ type LDBStore struct { hashfunc SwarmHasher po func(Address) uint8 - batchC chan bool batchesC chan struct{} closed bool batch *dbBatch lock sync.RWMutex quit chan struct{} + gc *garbage // Functions encodeDataFunc is used to bypass // the default functionality of DbStore with @@ -166,9 +181,33 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { data, _ = s.db.Get(keyDataIdx) s.dataIdx = BytesToU64(data) + // set up garbage collection + s.gc = &garbage{ + maxBatch: defaultMaxGCBatch, + maxRound: defaultMaxGCRound, + ratio: defaultGCRatio, + } + + s.gc.runC = make(chan struct{}, 1) + s.gc.runC <- struct{}{} + return s, nil } +// initialize and set values for processing of gc round +func (s *LDBStore) startGC(c int) { + + s.gc.count = 0 + // calculate the target number of deletions + if c >= s.gc.maxRound { + s.gc.target = s.gc.maxRound + } else { + s.gc.target = c / s.gc.ratio + } + s.gc.batch = newBatch() + log.Debug("startgc", "requested", c, "target", s.gc.target) +} + // NewMockDbStore creates a new instance of DbStore with // mockStore set to a provided value. If mockStore argument is nil, // this function behaves exactly as NewDbStore. @@ -225,6 +264,31 @@ func getDataKey(idx uint64, po uint8) []byte { return key } +func getGCIdxKey(index *dpaDBIndex) []byte { + key := make([]byte, 9) + key[0] = keyGCIdx + binary.BigEndian.PutUint64(key[1:], index.Access) + return key +} + +func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { + val := make([]byte, 41) // po = 1, index.Index = 8, Address = 32 + val[0] = po + binary.BigEndian.PutUint64(val[1:], index.Idx) + copy(val[9:], addr) + return val +} + +func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) { + index = &dpaDBIndex{ + Idx: binary.BigEndian.Uint64(val[1:]), + Access: binary.BigEndian.Uint64(accessCnt), + } + po = val[0] + addr = val[9:] + return +} + func encodeIndex(index *dpaDBIndex) []byte { data, _ := rlp.EncodeToBytes(index) return data @@ -247,55 +311,70 @@ func decodeData(addr Address, data []byte) (*chunk, error) { return NewChunk(addr, data[32:]), nil } -func (s *LDBStore) collectGarbage(ratio float32) { - log.Trace("collectGarbage", "ratio", ratio) +func (s *LDBStore) collectGarbage() error { + + // prevent duplicate gc from starting when one is already running + select { + case <-s.gc.runC: + default: + return nil + } + + s.lock.Lock() + entryCnt := s.entryCnt + s.lock.Unlock() metrics.GetOrRegisterCounter("ldbstore.collectgarbage", nil).Inc(1) - it := s.db.NewIterator() - defer it.Release() + // calculate the amount of chunks to collect and reset counter + s.startGC(int(entryCnt)) + log.Debug("collectGarbage", "target", s.gc.target, "entryCnt", entryCnt) - garbage := []*gcItem{} - gcnt := 0 + var totalDeleted int + for s.gc.count < s.gc.target { + it := s.db.NewIterator() + ok := it.Seek([]byte{keyGCIdx}) + var singleIterationCount int - for ok := it.Seek([]byte{keyIndex}); ok && (gcnt < maxGCitems) && (uint64(gcnt) < s.entryCnt); ok = it.Next() { - itkey := it.Key() + // every batch needs a lock so we avoid entries changing accessidx in the meantime + s.lock.Lock() + for ; ok && (singleIterationCount < s.gc.maxBatch); ok = it.Next() { - if (itkey == nil) || (itkey[0] != keyIndex) { - break + // quit if no more access index keys + itkey := it.Key() + if (itkey == nil) || (itkey[0] != keyGCIdx) { + break + } + + // get chunk data entry from access index + val := it.Value() + index, po, hash := parseGCIdxEntry(itkey[1:], val) + keyIdx := make([]byte, 33) + keyIdx[0] = keyIndex + copy(keyIdx[1:], hash) + + // add delete operation to batch + s.delete(s.gc.batch.Batch, index, keyIdx, po) + singleIterationCount++ + s.gc.count++ + + // break if target is not on max garbage batch boundary + if s.gc.count >= s.gc.target { + break + } } - // it.Key() contents change on next call to it.Next(), so we must copy it - key := make([]byte, len(it.Key())) - copy(key, it.Key()) - - val := it.Value() - - var index dpaDBIndex - - hash := key[1:] - decodeIndex(val, &index) - po := s.po(hash) - - gci := &gcItem{ - idxKey: key, - idx: index.Idx, - value: index.Access, // the smaller, the more likely to be gc'd. see sort comparator below. - po: po, - } - - garbage = append(garbage, gci) - gcnt++ + s.writeBatch(s.gc.batch, wEntryCnt) + s.lock.Unlock() + it.Release() + log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count) } - sort.Slice(garbage[:gcnt], func(i, j int) bool { return garbage[i].value < garbage[j].value }) + s.gc.runC <- struct{}{} + log.Debug("garbage collect done", "c", s.gc.count) - cutoff := int(float32(gcnt) * ratio) - metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(cutoff)) - - for i := 0; i < cutoff; i++ { - s.delete(garbage[i].idx, garbage[i].idxKey, garbage[i].po) - } + metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(totalDeleted)) + return nil } // Export writes all chunks from the store to a tar archive, returning the @@ -474,7 +553,7 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) { // if chunk is to be removed if f(c) { log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) - s.delete(index.Idx, getIndexKey(key[1:]), po) + s.deleteNow(&index, getIndexKey(key[1:]), po) removed++ errorsFound++ } @@ -526,24 +605,43 @@ func (s *LDBStore) ReIndex() { log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) } -func (s *LDBStore) Delete(addr Address) { +// Delete is removes a chunk and updates indices. +// Is thread safe +func (s *LDBStore) Delete(addr Address) error { s.lock.Lock() defer s.lock.Unlock() ikey := getIndexKey(addr) - var indx dpaDBIndex - s.tryAccessIdx(ikey, &indx) + idata, err := s.db.Get(ikey) + if err != nil { + return err + } - s.delete(indx.Idx, ikey, s.po(addr)) + var idx dpaDBIndex + decodeIndex(idata, &idx) + proximity := s.po(addr) + return s.deleteNow(&idx, ikey, proximity) } -func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { +// executes one delete operation immediately +// see *LDBStore.delete +func (s *LDBStore) deleteNow(idx *dpaDBIndex, idxKey []byte, po uint8) error { + batch := new(leveldb.Batch) + s.delete(batch, idx, idxKey, po) + return s.db.Write(batch) +} + +// adds a delete chunk operation to the provided batch +// if called directly, decrements entrycount regardless if the chunk exists upon deletion. Risk of wrap to max uint64 +func (s *LDBStore) delete(batch *leveldb.Batch, idx *dpaDBIndex, idxKey []byte, po uint8) { metrics.GetOrRegisterCounter("ldbstore.delete", nil).Inc(1) - batch := new(leveldb.Batch) + gcIdxKey := getGCIdxKey(idx) + batch.Delete(gcIdxKey) + dataKey := getDataKey(idx.Idx, po) + batch.Delete(dataKey) batch.Delete(idxKey) - batch.Delete(getDataKey(idx, po)) s.entryCnt-- dbEntryCount.Dec(1) cntKey := make([]byte, 2) @@ -551,7 +649,6 @@ func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { cntKey[1] = po batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) - s.db.Write(batch) } func (s *LDBStore) BinIndex(po uint8) uint64 { @@ -572,6 +669,9 @@ func (s *LDBStore) CurrentStorageIndex() uint64 { return s.dataIdx } +// Put adds a chunk to the database, adding indices and incrementing global counters. +// If it already exists, it merely increments the access count of the existing entry. +// Is thread safe func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { metrics.GetOrRegisterCounter("ldbstore.put", nil).Inc(1) log.Trace("ldbstore.put", "key", chunk.Address()) @@ -594,7 +694,7 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { if err != nil { s.doPut(chunk, &index, po) } else { - log.Trace("ldbstore.put: chunk already exists, only update access", "key", chunk.Address) + log.Debug("ldbstore.put: chunk already exists, only update access", "key", chunk.Address(), "po", po) decodeIndex(idata, &index) } index.Access = s.accessCnt @@ -602,6 +702,10 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { idata = encodeIndex(&index) s.batch.Put(ikey, idata) + // add the access-chunkindex index for garbage collection + gcIdxKey := getGCIdxKey(&index) + gcIdxData := getGCIdxValue(&index, po, chunk.Address()) + s.batch.Put(gcIdxKey, gcIdxData) s.lock.Unlock() select { @@ -617,7 +721,7 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { } } -// force putting into db, does not check access index +// force putting into db, does not check or update necessary indices func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { data := s.encodeDataFunc(chunk) dkey := getDataKey(s.dataIdx, po) @@ -659,38 +763,26 @@ func (s *LDBStore) writeCurrentBatch() error { if l == 0 { return nil } - e := s.entryCnt - d := s.dataIdx - a := s.accessCnt s.batch = newBatch() - b.err = s.writeBatch(b, e, d, a) + b.err = s.writeBatch(b, wEntryCnt|wAccessCnt|wIndexCnt) close(b.c) - for e > s.capacity { - log.Trace("for >", "e", e, "s.capacity", s.capacity) - // Collect garbage in a separate goroutine - // to be able to interrupt this loop by s.quit. - done := make(chan struct{}) - go func() { - s.collectGarbage(gcArrayFreeRatio) - log.Trace("collectGarbage closing done") - close(done) - }() - - select { - case <-s.quit: - return errors.New("CollectGarbage terminated due to quit") - case <-done: - } - e = s.entryCnt + if s.entryCnt >= s.capacity { + go s.collectGarbage() } return nil } // must be called non concurrently -func (s *LDBStore) writeBatch(b *dbBatch, entryCnt, dataIdx, accessCnt uint64) error { - b.Put(keyEntryCnt, U64ToBytes(entryCnt)) - b.Put(keyDataIdx, U64ToBytes(dataIdx)) - b.Put(keyAccessCnt, U64ToBytes(accessCnt)) +func (s *LDBStore) writeBatch(b *dbBatch, wFlag uint8) error { + if wFlag&wEntryCnt > 0 { + b.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) + } + if wFlag&wIndexCnt > 0 { + b.Put(keyDataIdx, U64ToBytes(s.dataIdx)) + } + if wFlag&wAccessCnt > 0 { + b.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) + } l := b.Len() if err := s.db.Write(b.Batch); err != nil { return fmt.Errorf("unable to write batch: %v", err) @@ -713,17 +805,22 @@ func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { } // try to find index; if found, update access cnt and return true -func (s *LDBStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { +func (s *LDBStore) tryAccessIdx(ikey []byte, po uint8, index *dpaDBIndex) bool { idata, err := s.db.Get(ikey) if err != nil { return false } decodeIndex(idata, index) + oldGCIdxKey := getGCIdxKey(index) s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) s.accessCnt++ index.Access = s.accessCnt idata = encodeIndex(index) s.batch.Put(ikey, idata) + newGCIdxKey := getGCIdxKey(index) + newGCIdxData := getGCIdxValue(index, po, ikey) + s.batch.Delete(oldGCIdxKey) + s.batch.Put(newGCIdxKey, newGCIdxData) select { case s.batchesC <- struct{}{}: default: @@ -755,6 +852,9 @@ func (s *LDBStore) PutSchema(schema string) error { return s.db.Put(keySchema, []byte(schema)) } +// Get retrieves the chunk matching the provided key from the database. +// If the chunk entry does not exist, it returns an error +// Updates access count and is thread safe func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) { metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1) log.Trace("ldbstore.get", "key", addr) @@ -764,12 +864,14 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) return s.get(addr) } +// TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { var indx dpaDBIndex if s.closed { return nil, ErrDBClosed } - if s.tryAccessIdx(getIndexKey(addr), &indx) { + proximity := s.po(addr) + if s.tryAccessIdx(getIndexKey(addr), proximity, &indx) { var data []byte if s.getDataFunc != nil { // if getDataFunc is defined, use it to retrieve the chunk data @@ -780,13 +882,12 @@ func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { } } else { // default DbStore functionality to retrieve chunk data - proximity := s.po(addr) datakey := getDataKey(indx.Idx, proximity) data, err = s.db.Get(datakey) log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", indx.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) if err != nil { log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err) - s.delete(indx.Idx, getIndexKey(addr), s.po(addr)) + s.deleteNow(&indx, getIndexKey(addr), s.po(addr)) return } } @@ -813,33 +914,14 @@ func newMockGetDataFunc(mockStore *mock.NodeStore) func(addr Address) (data []by } } -func (s *LDBStore) updateAccessCnt(addr Address) { - - s.lock.Lock() - defer s.lock.Unlock() - - var index dpaDBIndex - s.tryAccessIdx(getIndexKey(addr), &index) // result_chn == nil, only update access cnt - -} - func (s *LDBStore) setCapacity(c uint64) { s.lock.Lock() defer s.lock.Unlock() s.capacity = c - if s.entryCnt > c { - ratio := float32(1.01) - float32(c)/float32(s.entryCnt) - if ratio < gcArrayFreeRatio { - ratio = gcArrayFreeRatio - } - if ratio > 1 { - ratio = 1 - } - for s.entryCnt > c { - s.collectGarbage(ratio) - } + for s.entryCnt > c { + s.collectGarbage() } } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 14a42b5e3d..48af8c57c0 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -22,6 +22,8 @@ import ( "fmt" "io/ioutil" "os" + "strconv" + "strings" "testing" "time" @@ -297,27 +299,73 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { } // TestLDBStoreCollectGarbage tests that we can put more chunks than LevelDB's capacity, and -// retrieve only some of them, because garbage collection must have cleared some of them +// retrieve only some of them, because garbage collection must have partially cleared the store +// Also tests that we can delete chunks and that we can trigger garbage collection func TestLDBStoreCollectGarbage(t *testing.T) { - capacity := 500 - n := 2000 + + // below max ronud + cap := defaultMaxGCRound / 2 + t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) + t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) + + // at max round + cap = defaultMaxGCRound + t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) + t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) + + // more than max around, not on threshold + cap = defaultMaxGCRound * 1.1 + t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) + t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) + +} + +func testLDBStoreCollectGarbage(t *testing.T) { + params := strings.Split(t.Name(), "/") + capacity, err := strconv.Atoi(params[2]) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(params[3]) + if err != nil { + t.Fatal(err) + } ldb, cleanup := newLDBStore(t) ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) - if err != nil { - t.Fatal(err.Error()) + // retrieve the gc round target count for the db capacity + ldb.startGC(capacity) + roundTarget := ldb.gc.target + + // split put counts to gc target count threshold, and wait for gc to finish in between + var allChunks []Chunk + remaining := n + for remaining > 0 { + var putCount int + if remaining < roundTarget { + putCount = remaining + } else { + putCount = roundTarget + } + remaining -= putCount + chunks, err := mputRandomChunks(ldb, putCount, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) + } + allChunks = append(allChunks, chunks...) + log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) } - log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) - - // wait for garbage collection to kick in on the responsible actor - time.Sleep(1 * time.Second) + // attempt gets on all put chunks var missing int - for _, ch := range chunks { - ret, err := ldb.Get(context.Background(), ch.Address()) + for _, ch := range allChunks { + ret, err := ldb.Get(context.TODO(), ch.Address()) if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { missing++ continue @@ -333,8 +381,10 @@ func TestLDBStoreCollectGarbage(t *testing.T) { log.Trace("got back chunk", "chunk", ret) } - if missing < n-capacity { - t.Fatalf("gc failure: expected to miss %v chunks, but only %v are actually missing", n-capacity, missing) + // all surplus chunks should be missing + expectMissing := roundTarget + (((n - capacity) / roundTarget) * roundTarget) + if missing != expectMissing { + t.Fatalf("gc failure: expected to miss %v chunks, but only %v are actually missing", expectMissing, missing) } log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) @@ -367,7 +417,6 @@ func TestLDBStoreAddRemove(t *testing.T) { if i%2 == 0 { // expect even chunks to be missing if err == nil { - // if err != ErrChunkNotFound { t.Fatal("expected chunk to be missing, but got no error") } } else { @@ -383,30 +432,48 @@ func TestLDBStoreAddRemove(t *testing.T) { } } -// TestLDBStoreRemoveThenCollectGarbage tests that we can delete chunks and that we can trigger garbage collection -func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { - capacity := 11 - surplus := 4 +func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { + + params := strings.Split(t.Name(), "/") + capacity, err := strconv.Atoi(params[2]) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(params[3]) + if err != nil { + t.Fatal(err) + } ldb, cleanup := newLDBStore(t) + defer cleanup() ldb.setCapacity(uint64(capacity)) - n := capacity - - chunks := []Chunk{} - for i := 0; i < n+surplus; i++ { + // put capacity count number of chunks + chunks := make([]Chunk, n) + for i := 0; i < n; i++ { c := GenerateRandomChunk(ch.DefaultSize) - chunks = append(chunks, c) + chunks[i] = c log.Trace("generate random chunk", "idx", i, "chunk", c) } for i := 0; i < n; i++ { - ldb.Put(context.TODO(), chunks[i]) + err := ldb.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatal(err) + } } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) + // delete all chunks + // (only count the ones actually deleted, the rest will have been gc'd) + deletes := 0 for i := 0; i < n; i++ { - ldb.Delete(chunks[i].Address()) + if ldb.Delete(chunks[i].Address()) == nil { + deletes++ + } } log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) @@ -415,37 +482,49 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { t.Fatalf("ldb.entrCnt expected 0 got %v", ldb.entryCnt) } - expAccessCnt := uint64(n * 2) + // the manual deletes will have increased accesscnt, so we need to add this when we verify the current count + expAccessCnt := uint64(n) if ldb.accessCnt != expAccessCnt { - t.Fatalf("ldb.accessCnt expected %v got %v", expAccessCnt, ldb.entryCnt) + t.Fatalf("ldb.accessCnt expected %v got %v", expAccessCnt, ldb.accessCnt) } - cleanup() + // retrieve the gc round target count for the db capacity + ldb.startGC(capacity) + roundTarget := ldb.gc.target - ldb, cleanup = newLDBStore(t) - capacity = 10 - ldb.setCapacity(uint64(capacity)) - defer cleanup() + remaining := n + var puts int + for remaining > 0 { + var putCount int + if remaining < roundTarget { + putCount = remaining + } else { + putCount = roundTarget + } + remaining -= putCount + for putCount > 0 { + ldb.Put(context.TODO(), chunks[puts]) + log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n, "puts", puts, "remaining", remaining, "roundtarget", roundTarget) + puts++ + putCount-- + } - n = capacity + surplus - - for i := 0; i < n; i++ { - ldb.Put(context.TODO(), chunks[i]) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) } - // wait for garbage collection - time.Sleep(1 * time.Second) - // expect first surplus chunks to be missing, because they have the smallest access value - for i := 0; i < surplus; i++ { + expectMissing := roundTarget + (((n - capacity) / roundTarget) * roundTarget) + for i := 0; i < expectMissing; i++ { _, err := ldb.Get(context.TODO(), chunks[i].Address()) if err == nil { - t.Fatal("expected surplus chunk to be missing, but got no error") + t.Fatalf("expected surplus chunk %d to be missing, but got no error", i) } } // expect last chunks to be present, as they have the largest access value - for i := surplus; i < surplus+capacity; i++ { + for i := expectMissing; i < n; i++ { ret, err := ldb.Get(context.TODO(), chunks[i].Address()) if err != nil { t.Fatalf("chunk %v: expected no error, but got %s", i, err) @@ -455,3 +534,57 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { } } } + +// TestLDBStoreCollectGarbageAccessUnlikeIndex tests garbage collection where accesscount differs from indexcount +func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { + + capacity := defaultMaxGCRound * 2 + n := capacity - 1 + + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(uint64(capacity)) + defer cleanup() + + chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) + } + log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) + + // set first added capacity/2 chunks to highest accesscount + for i := 0; i < capacity/2; i++ { + _, err := ldb.Get(context.TODO(), chunks[i].Address()) + if err != nil { + t.Fatalf("fail add chunk #%d - %s: %v", i, chunks[i].Address(), err) + } + } + _, err = mputRandomChunks(ldb, 2, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) + } + + // wait for garbage collection to kick in on the responsible actor + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) + + var missing int + for i, ch := range chunks[2 : capacity/2] { + ret, err := ldb.Get(context.TODO(), ch.Address()) + if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { + t.Fatalf("fail find chunk #%d - %s: %v", i, ch.Address(), err) + } + + if !bytes.Equal(ret.Data(), ch.Data()) { + t.Fatal("expected to get the same data back, but got smth else") + } + log.Trace("got back chunk", "chunk", ret) + } + + log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) +} + +func waitGc(ctx context.Context, ldb *LDBStore) { + <-ldb.gc.runC + ldb.gc.runC <- struct{}{} +} From 6566a0a3b82f5d24d478d3876d5fa2b1b0e8684c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Fri, 12 Oct 2018 16:26:16 +0200 Subject: [PATCH 006/100] swarm/network/stream: generalise setting of next batch (#17818) * swarm/network/stream: generalize SetNextBatch and add Server SessionIndex * swarm/network/stream: fix a typo in comment * swarm/network/stream: remove live argument from NewSwarmSyncerServer --- swarm/network/stream/delivery.go | 7 +++- swarm/network/stream/delivery_test.go | 6 +-- swarm/network/stream/intervals_test.go | 16 ++------ swarm/network/stream/peer.go | 13 +++++-- swarm/network/stream/stream.go | 30 ++++++++++++++- swarm/network/stream/streamer_test.go | 42 +++++++++++---------- swarm/network/stream/syncer.go | 52 ++++++++------------------ 7 files changed, 89 insertions(+), 77 deletions(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index c2adb1009e..0429c4dffc 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -96,6 +96,11 @@ func (s *SwarmChunkServer) processDeliveries() { } } +// SessionIndex returns zero in all cases for SwarmChunkServer. +func (s *SwarmChunkServer) SessionIndex() (uint64, error) { + return 0, nil +} + // SetNextBatch func (s *SwarmChunkServer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { select { @@ -141,7 +146,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * "retrieve.request") defer osp.Finish() - s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", false)) + s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", true)) if err != nil { return err } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index b021b87714..c6ebae3f0e 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -88,7 +88,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { peer := streamer.getPeer(node.ID()) peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{ - Stream: NewStream(swarmChunkServerStreamName, "", false), + Stream: NewStream(swarmChunkServerStreamName, "", true), History: nil, Priority: Top, }) @@ -136,7 +136,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { node := tester.Nodes[0] peer := streamer.getPeer(node.ID()) - stream := NewStream(swarmChunkServerStreamName, "", false) + stream := NewStream(swarmChunkServerStreamName, "", true) peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{ Stream: stream, @@ -409,7 +409,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return fmt.Errorf("No registry") } registry := item.(*Registry) - err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", false), NewRange(0, 0), Top) + err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), NewRange(0, 0), Top) if err != nil { return err } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 2692594238..3164193b34 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -345,8 +345,6 @@ func (c *testExternalClient) BatchDone(Stream, uint64, []byte, []byte) func() (* func (c *testExternalClient) Close() {} -const testExternalServerBatchSize = 10 - type testExternalServer struct { t string keyFunc func(key []byte, index uint64) @@ -366,17 +364,11 @@ func newTestExternalServer(t string, sessionAt, maxKeys uint64, keyFunc func(key } } +func (s *testExternalServer) SessionIndex() (uint64, error) { + return s.sessionAt, nil +} + func (s *testExternalServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { - if from == 0 && to == 0 { - from = s.sessionAt - to = s.sessionAt + testExternalServerBatchSize - } - if to-from > testExternalServerBatchSize { - to = from + testExternalServerBatchSize - 1 - } - if from >= s.maxKeys && to > s.maxKeys { - return nil, 0, 0, nil, io.EOF - } if to > s.maxKeys { to = s.maxKeys } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index ef6bbdf70d..89d135ad52 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -166,7 +166,7 @@ func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error { "send.offered.hashes") defer sp.Finish() - hashes, from, to, proof, err := s.SetNextBatch(f, t) + hashes, from, to, proof, err := s.setNextBatch(f, t) if err != nil { return err } @@ -214,10 +214,15 @@ func (p *Peer) setServer(s Stream, o Server, priority uint8) (*server, error) { return nil, ErrMaxPeerServers } + sessionIndex, err := o.SessionIndex() + if err != nil { + return nil, err + } os := &server{ - Server: o, - stream: s, - priority: priority, + Server: o, + stream: s, + priority: priority, + sessionIndex: sessionIndex, } p.servers[s] = os return os, nil diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 1eda06c6a7..3861cfcf6a 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -375,7 +375,7 @@ func (r *Registry) Run(p *network.BzzPeer) error { defer sp.close() if r.doRetrieve { - err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", false), nil, Top) + err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", true), nil, Top) if err != nil { return err } @@ -500,10 +500,38 @@ type server struct { stream Stream priority uint8 currentBatch []byte + sessionIndex uint64 +} + +// setNextBatch adjusts passed interval based on session index and whether +// stream is live or history. It calls Server SetNextBatch with adjusted +// interval and returns batch hashes and their interval. +func (s *server) setNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { + if s.stream.Live { + if from == 0 { + from = s.sessionIndex + } + if to <= from || from >= s.sessionIndex { + to = math.MaxUint64 + } + } else { + if (to < from && to != 0) || from > s.sessionIndex { + return nil, 0, 0, nil, nil + } + if to == 0 || to > s.sessionIndex { + to = s.sessionIndex + } + } + return s.SetNextBatch(from, to) } // Server interface for outgoing peer Streamer type Server interface { + // SessionIndex is called when a server is initialized + // to get the current cursor state of the stream data. + // Based on this index, live and history stream intervals + // will be adjusted before calling SetNextBatch. + SessionIndex() (uint64, error) SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) GetData(context.Context, []byte) ([]byte, error) Close() diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index 5d91eecfd7..e7f79e7a12 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -107,15 +107,21 @@ func (self *testClient) BatchDone(Stream, uint64, []byte, []byte) func() (*Takeo func (self *testClient) Close() {} type testServer struct { - t string + t string + sessionIndex uint64 } -func newTestServer(t string) *testServer { +func newTestServer(t string, sessionIndex uint64) *testServer { return &testServer{ - t: t, + t: t, + sessionIndex: sessionIndex, } } +func (s *testServer) SessionIndex() (uint64, error) { + return s.sessionIndex, nil +} + func (self *testServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { return make([]byte, HashSize), from + 1, to + 1, nil, nil } @@ -230,7 +236,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { stream := NewStream("foo", "", false) streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 10), nil }) node := tester.Nodes[0] @@ -297,7 +303,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { stream := NewStream("foo", "", true) streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 0), nil }) node := tester.Nodes[0] @@ -324,7 +330,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { }, Hashes: make([]byte, HashSize), From: 1, - To: 1, + To: 0, }, Peer: node.ID(), }, @@ -361,7 +367,7 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { } streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 0), nil }) stream := NewStream("bar", "", true) @@ -407,9 +413,7 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { stream := NewStream("foo", "", true) streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return &testServer{ - t: t, - }, nil + return newTestServer(t, 10), nil }) node := tester.Nodes[0] @@ -448,8 +452,8 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { HandoverProof: &HandoverProof{ Handover: &Handover{}, }, - From: 1, - To: 1, + From: 11, + To: 0, Hashes: make([]byte, HashSize), }, Peer: node.ID(), @@ -634,7 +638,7 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { } streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 10), nil }) node := tester.Nodes[0] @@ -694,8 +698,8 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { HandoverProof: &HandoverProof{ Handover: &Handover{}, }, - From: 1, - To: 1, + From: 11, + To: 0, Hashes: make([]byte, HashSize), }, Peer: node.ID(), @@ -769,7 +773,7 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) { } streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 0), nil }) node := tester.Nodes[0] @@ -799,7 +803,7 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) { }, Hashes: make([]byte, HashSize), From: 1, - To: 1, + To: 0, }, Peer: node.ID(), }, @@ -843,7 +847,7 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { } streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 0), nil }) node := tester.Nodes[0] @@ -903,7 +907,7 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { }, Hashes: make([]byte, HashSize), From: 1, - To: 1, + To: 0, }, Peer: node.ID(), }, diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index 38b3078d2f..4bfbac8b0d 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -18,7 +18,6 @@ package stream import ( "context" - "math" "strconv" "time" @@ -36,38 +35,27 @@ const ( // * live request delivery with or without checkback // * (live/non-live historical) chunk syncing per proximity bin type SwarmSyncerServer struct { - po uint8 - store storage.SyncChunkStore - sessionAt uint64 - start uint64 - live bool - quit chan struct{} + po uint8 + store storage.SyncChunkStore + quit chan struct{} } -// NewSwarmSyncerServer is contructor for SwarmSyncerServer -func NewSwarmSyncerServer(live bool, po uint8, syncChunkStore storage.SyncChunkStore) (*SwarmSyncerServer, error) { - sessionAt := syncChunkStore.BinIndex(po) - var start uint64 - if live { - start = sessionAt - } +// NewSwarmSyncerServer is constructor for SwarmSyncerServer +func NewSwarmSyncerServer(po uint8, syncChunkStore storage.SyncChunkStore) (*SwarmSyncerServer, error) { return &SwarmSyncerServer{ - po: po, - store: syncChunkStore, - sessionAt: sessionAt, - start: start, - live: live, - quit: make(chan struct{}), + po: po, + store: syncChunkStore, + quit: make(chan struct{}), }, nil } func RegisterSwarmSyncerServer(streamer *Registry, syncChunkStore storage.SyncChunkStore) { - streamer.RegisterServerFunc("SYNC", func(p *Peer, t string, live bool) (Server, error) { + streamer.RegisterServerFunc("SYNC", func(_ *Peer, t string, _ bool) (Server, error) { po, err := ParseSyncBinKey(t) if err != nil { return nil, err } - return NewSwarmSyncerServer(live, po, syncChunkStore) + return NewSwarmSyncerServer(po, syncChunkStore) }) // streamer.RegisterServerFunc(stream, func(p *Peer) (Server, error) { // return NewOutgoingProvableSwarmSyncer(po, db) @@ -88,25 +76,15 @@ func (s *SwarmSyncerServer) GetData(ctx context.Context, key []byte) ([]byte, er return chunk.Data(), nil } +// SessionIndex returns current storage bin (po) index. +func (s *SwarmSyncerServer) SessionIndex() (uint64, error) { + return s.store.BinIndex(s.po), nil +} + // GetBatch retrieves the next batch of hashes from the dbstore func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { var batch []byte i := 0 - if s.live { - if from == 0 { - from = s.start - } - if to <= from || from >= s.sessionAt { - to = math.MaxUint64 - } - } else { - if (to < from && to != 0) || from > s.sessionAt { - return nil, 0, 0, nil, nil - } - if to == 0 || to > s.sessionAt { - to = s.sessionAt - } - } var ticker *time.Ticker defer func() { From 2e98631c5e5724bea1a29b877929b4911bf50b86 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 15 Oct 2018 10:56:04 +0200 Subject: [PATCH 007/100] rpc: fix client shutdown hang when Close races with Unsubscribe (#17894) Fixes #17837 --- rpc/client.go | 59 +++++++++-------------------------------- rpc/client_test.go | 24 +++++++++++++++++ rpc/stdio.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 47 deletions(-) create mode 100644 rpc/stdio.go diff --git a/rpc/client.go b/rpc/client.go index d96189a2d8..6254c95ffd 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -25,7 +25,6 @@ import ( "fmt" "net" "net/url" - "os" "reflect" "strconv" "strings" @@ -118,7 +117,8 @@ type Client struct { // for dispatch close chan struct{} - didQuit chan struct{} // closed when client quits + closing chan struct{} // closed when client is quitting + didClose chan struct{} // closed when client quits reconnected chan net.Conn // where write/reconnect sends the new connection readErr chan error // errors from read readResp chan []*jsonrpcMessage // valid messages from read @@ -181,45 +181,6 @@ func DialContext(ctx context.Context, rawurl string) (*Client, error) { } } -type StdIOConn struct{} - -func (io StdIOConn) Read(b []byte) (n int, err error) { - return os.Stdin.Read(b) -} - -func (io StdIOConn) Write(b []byte) (n int, err error) { - return os.Stdout.Write(b) -} - -func (io StdIOConn) Close() error { - return nil -} - -func (io StdIOConn) LocalAddr() net.Addr { - return &net.UnixAddr{Name: "stdio", Net: "stdio"} -} - -func (io StdIOConn) RemoteAddr() net.Addr { - return &net.UnixAddr{Name: "stdio", Net: "stdio"} -} - -func (io StdIOConn) SetDeadline(t time.Time) error { - return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} -} - -func (io StdIOConn) SetReadDeadline(t time.Time) error { - return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} -} - -func (io StdIOConn) SetWriteDeadline(t time.Time) error { - return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} -} -func DialStdIO(ctx context.Context) (*Client, error) { - return newClient(ctx, func(_ context.Context) (net.Conn, error) { - return StdIOConn{}, nil - }) -} - func newClient(initctx context.Context, connectFunc func(context.Context) (net.Conn, error)) (*Client, error) { conn, err := connectFunc(initctx) if err != nil { @@ -231,7 +192,8 @@ func newClient(initctx context.Context, connectFunc func(context.Context) (net.C isHTTP: isHTTP, connectFunc: connectFunc, close: make(chan struct{}), - didQuit: make(chan struct{}), + closing: make(chan struct{}), + didClose: make(chan struct{}), reconnected: make(chan net.Conn), readErr: make(chan error), readResp: make(chan []*jsonrpcMessage), @@ -268,8 +230,8 @@ func (c *Client) Close() { } select { case c.close <- struct{}{}: - <-c.didQuit - case <-c.didQuit: + <-c.didClose + case <-c.didClose: } } @@ -469,7 +431,9 @@ func (c *Client) send(ctx context.Context, op *requestOp, msg interface{}) error // This can happen if the client is overloaded or unable to keep up with // subscription notifications. return ctx.Err() - case <-c.didQuit: + case <-c.closing: + return ErrClientQuit + case <-c.didClose: return ErrClientQuit } } @@ -504,7 +468,7 @@ func (c *Client) reconnect(ctx context.Context) error { case c.reconnected <- newconn: c.writeConn = newconn return nil - case <-c.didQuit: + case <-c.didClose: newconn.Close() return ErrClientQuit } @@ -522,8 +486,9 @@ func (c *Client) dispatch(conn net.Conn) { requestOpLock = c.requestOp // nil while the send lock is held reading = true // if true, a read loop is running ) - defer close(c.didQuit) + defer close(c.didClose) defer func() { + close(c.closing) c.closeRequestOps(ErrClientQuit) conn.Close() if reading { diff --git a/rpc/client_test.go b/rpc/client_test.go index 4f354d389e..a8195c0af8 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -323,6 +323,30 @@ func TestClientSubscribeClose(t *testing.T) { } } +// This test reproduces https://github.com/ethereum/go-ethereum/issues/17837 where the +// client hangs during shutdown when Unsubscribe races with Client.Close. +func TestClientCloseUnsubscribeRace(t *testing.T) { + service := &NotificationTestService{} + server := newTestServer("eth", service) + defer server.Stop() + + for i := 0; i < 20; i++ { + client := DialInProc(server) + nc := make(chan int) + sub, err := client.EthSubscribe(context.Background(), nc, "someSubscription", 3, 1) + if err != nil { + t.Fatal(err) + } + go client.Close() + go sub.Unsubscribe() + select { + case <-sub.Err(): + case <-time.After(5 * time.Second): + t.Fatal("subscription not closed within timeout") + } + } +} + // This test checks that Client doesn't lock up when a single subscriber // doesn't read subscription events. func TestClientNotificationStorm(t *testing.T) { diff --git a/rpc/stdio.go b/rpc/stdio.go new file mode 100644 index 0000000000..ea552cca28 --- /dev/null +++ b/rpc/stdio.go @@ -0,0 +1,66 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rpc + +import ( + "context" + "errors" + "net" + "os" + "time" +) + +// DialStdIO creates a client on stdin/stdout. +func DialStdIO(ctx context.Context) (*Client, error) { + return newClient(ctx, func(_ context.Context) (net.Conn, error) { + return stdioConn{}, nil + }) +} + +type stdioConn struct{} + +func (io stdioConn) Read(b []byte) (n int, err error) { + return os.Stdin.Read(b) +} + +func (io stdioConn) Write(b []byte) (n int, err error) { + return os.Stdout.Write(b) +} + +func (io stdioConn) Close() error { + return nil +} + +func (io stdioConn) LocalAddr() net.Addr { + return &net.UnixAddr{Name: "stdio", Net: "stdio"} +} + +func (io stdioConn) RemoteAddr() net.Addr { + return &net.UnixAddr{Name: "stdio", Net: "stdio"} +} + +func (io stdioConn) SetDeadline(t time.Time) error { + return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} +} + +func (io stdioConn) SetReadDeadline(t time.Time) error { + return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} +} + +func (io stdioConn) SetWriteDeadline(t time.Time) error { + return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} +} From 60827dc50fc0892f63fc79704c57b45cf34f991e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 16 Oct 2018 00:26:47 +0200 Subject: [PATCH 008/100] tests: update tests, implement no-pow blocks (#17902) This commit updates our tests with the latest and greatest from ethereum/tests. It also contains implementation of NoProof for blockchain tests. --- tests/block_test.go | 8 ++++++-- tests/block_test_util.go | 23 +++++++++++++++-------- tests/difficulty_test.go | 19 ++++--------------- tests/state_test.go | 5 +---- tests/testdata | 2 +- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index 8315728a63..711a3f8695 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -30,8 +30,6 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcForgedTest/bcForkUncle\.json`) bt.skipLoad(`^bcMultiChainTest/(ChainAtoChainB_blockorder|CallContractFromNotBestBlock)`) bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) - // This test is broken - bt.fails(`blockhashNonConstArg_Constantinople`, "Broken test") // Slow tests bt.slow(`^bcExploitTest/DelegateCallSpam.json`) bt.slow(`^bcExploitTest/ShanghaiLove.json`) @@ -40,6 +38,12 @@ func TestBlockchain(t *testing.T) { bt.slow(`^bcGasPricerTest/RPC_API_Test.json`) bt.slow(`^bcWalletTest/`) + // Still failing tests that we need to look into + //bt.fails(`^bcStateTests/suicideThenCheckBalance.json/suicideThenCheckBalance_Constantinople`, "TODO: investigate") + //bt.fails(`^bcStateTests/suicideStorageCheckVCreate2.json/suicideStorageCheckVCreate2_Constantinople`, "TODO: investigate") + //bt.fails(`^bcStateTests/suicideStorageCheckVCreate.json/suicideStorageCheckVCreate_Constantinople`, "TODO: investigate") + //bt.fails(`^bcStateTests/suicideStorageCheck.json/suicideStorageCheck_Constantinople`, "TODO: investigate") + bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { if err := bt.checkFailure(t, name, test.Run()); err != nil { t.Error(err) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 427a94958a..12cba3244f 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" @@ -48,12 +49,13 @@ func (t *BlockTest) UnmarshalJSON(in []byte) error { } type btJSON struct { - Blocks []btBlock `json:"blocks"` - Genesis btHeader `json:"genesisBlockHeader"` - Pre core.GenesisAlloc `json:"pre"` - Post core.GenesisAlloc `json:"postState"` - BestBlock common.UnprefixedHash `json:"lastblockhash"` - Network string `json:"network"` + Blocks []btBlock `json:"blocks"` + Genesis btHeader `json:"genesisBlockHeader"` + Pre core.GenesisAlloc `json:"pre"` + Post core.GenesisAlloc `json:"postState"` + BestBlock common.UnprefixedHash `json:"lastblockhash"` + Network string `json:"network"` + SealEngine string `json:"sealEngine"` } type btBlock struct { @@ -110,8 +112,13 @@ func (t *BlockTest) Run() error { if gblock.Root() != t.json.Genesis.StateRoot { return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", gblock.Root().Bytes()[:6], t.json.Genesis.StateRoot[:6]) } - - chain, err := core.NewBlockChain(db, nil, config, ethash.NewShared(), vm.Config{}, nil) + var engine consensus.Engine + if t.json.SealEngine == "NoProof" { + engine = ethash.NewFaker() + } else { + engine = ethash.NewShared() + } + chain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil) if err != nil { return err } diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 56c3fc297b..fde9db3ad4 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -36,20 +36,6 @@ var ( EIP158Block: big.NewInt(2675000), ByzantiumBlock: big.NewInt(4370000), } - - // Ropsten without the Constantinople bump in bomb delay - RopstenNoConstantinople = params.ChainConfig{ - ChainID: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1700000), - ConstantinopleBlock: nil, - } ) func TestDifficulty(t *testing.T) { @@ -69,7 +55,7 @@ func TestDifficulty(t *testing.T) { dt.skipLoad("difficultyMorden\\.json") dt.skipLoad("difficultyOlimpic\\.json") - dt.config("Ropsten", RopstenNoConstantinople) + dt.config("Ropsten", *params.TestnetChainConfig) dt.config("Morden", *params.TestnetChainConfig) dt.config("Frontier", params.ChainConfig{}) @@ -84,6 +70,9 @@ func TestDifficulty(t *testing.T) { dt.config("Frontier", *params.TestnetChainConfig) dt.config("MainNetwork", mainnetChainConfig) dt.config("CustomMainNetwork", mainnetChainConfig) + dt.config("Constantinople", params.ChainConfig{ + ConstantinopleBlock: big.NewInt(0), + }) dt.config("difficulty.json", mainnetChainConfig) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, test *DifficultyTest) { diff --git a/tests/state_test.go b/tests/state_test.go index c52e9abb8e..ad77e4f339 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -45,14 +45,11 @@ func TestState(t *testing.T) { // Expected failures: st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch.json/Constantinople`, "bug in test") st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { subtest := subtest - if subtest.Fork == "Constantinople" { - // Skipping constantinople due to net sstore gas changes affecting all tests - continue - } key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key t.Run(key, func(t *testing.T) { diff --git a/tests/testdata b/tests/testdata index ad2184adca..95a3092038 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit ad2184adca367c0b68c65b44519dba16e1d0b9e2 +Subproject commit 95a309203890e6244c6d4353ca411671973c13b5 From 3e92c853fb74fa887dff6497c5d92952c31ff7ec Mon Sep 17 00:00:00 2001 From: Grachev Mikhail Date: Tue, 16 Oct 2018 01:33:09 +0300 Subject: [PATCH 009/100] cmd/clef: fix typos in README (#17908) --- cmd/clef/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/clef/README.md b/cmd/clef/README.md index c02ac44d85..c9461be101 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -91,7 +91,7 @@ invoking methods with the following info: * [x] Version info about the signer * [x] Address of API (http/ipc) * [ ] List of known accounts -* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected. +* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected. * [ ] `account_signRawTransaction` * [ ] `account_bulkSignTransactions([] transactions)` should * only exist if enabled via config/flag @@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification). Some of these call can require user interaction. Clients must be aware that responses -may be delayed significanlty or may never be received if a users decides to ignore the confirmation request. +may be delayed significantly or may never be received if a users decides to ignore the confirmation request. The External API is **untrusted** : it does not accept credentials over this api, nor does it expect that requests have any authority. @@ -862,7 +862,7 @@ A UI should conform to the following rules. * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts * The signer provides accounts -* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled +* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled along with the UI. From 331fa6d3075cb89fc295b36d9039620ec1d741ad Mon Sep 17 00:00:00 2001 From: Evgeny Date: Tue, 16 Oct 2018 01:34:50 +0300 Subject: [PATCH 010/100] accounts/usbwallet: simplify code using -= operator (#17904) --- accounts/usbwallet/ledger.go | 2 +- accounts/usbwallet/trezor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 2583fbc4da..7d5f67908d 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction signer = new(types.HomesteadSigner) } else { signer = types.NewEIP155Signer(chainID) - signature[64] = signature[64] - byte(chainID.Uint64()*2+35) + signature[64] -= byte(chainID.Uint64()*2 + 35) } signed, err := tx.WithSignature(signer, signature) if err != nil { diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index b84a955992..a9d2e9959e 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction signer = new(types.HomesteadSigner) } else { signer = types.NewEIP155Signer(chainID) - signature[64] = signature[64] - byte(chainID.Uint64()*2+35) + signature[64] -= byte(chainID.Uint64()*2 + 35) } // Inject the final signature into the transaction and sanity check the sender signed, err := tx.WithSignature(signer, signature) From 16e4d0e0055f7fce620ff6881a1393d955c06cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Tue, 16 Oct 2018 01:40:51 +0300 Subject: [PATCH 011/100] p2p: meter peer traffic, emit metered peer events (#17695) This change extends the peer metrics collection: - traces the life-cycle of the peers - meters the peer traffic separately for every peer - creates event feed for the peer events - emits the peer events --- p2p/dial.go | 2 +- p2p/metrics.go | 196 ++++++++++++++++++++++++++++++++++++++++++++----- p2p/server.go | 9 ++- 3 files changed, 188 insertions(+), 19 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index d228514fc6..075a0f9368 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -350,7 +350,7 @@ func (t *dialTask) dial(srv *Server, dest *enode.Node) error { if err != nil { return &dialError{err} } - mfd := newMeteredConn(fd, false) + mfd := newMeteredConn(fd, false, dest.IP()) return srv.SetupConn(mfd, t.flags, dest) } diff --git a/p2p/metrics.go b/p2p/metrics.go index 2d52fd1fd1..6a7c0bad33 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -19,53 +19,215 @@ package p2p import ( + "fmt" "net" + "sync" + "sync/atomic" + "time" + "github.com/ethereum/go-ethereum/p2p/enode" + + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" ) -var ( - ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil) - ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil) - egressConnectMeter = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil) - egressTrafficMeter = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil) +const ( + MetricsInboundConnects = "p2p/InboundConnects" // Name for the registered inbound connects meter + MetricsInboundTraffic = "p2p/InboundTraffic" // Name for the registered inbound traffic meter + MetricsOutboundConnects = "p2p/OutboundConnects" // Name for the registered outbound connects meter + MetricsOutboundTraffic = "p2p/OutboundTraffic" // Name for the registered outbound traffic meter + + MeteredPeerLimit = 1024 // This amount of peers are individually metered ) +var ( + ingressConnectMeter = metrics.NewRegisteredMeter(MetricsInboundConnects, nil) // Meter counting the ingress connections + ingressTrafficMeter = metrics.NewRegisteredMeter(MetricsInboundTraffic, nil) // Meter metering the cumulative ingress traffic + egressConnectMeter = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections + egressTrafficMeter = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil) // Meter metering the cumulative egress traffic + + PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress + PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress + + meteredPeerFeed event.Feed // Event feed for peer metrics + meteredPeerCount int32 // Actually stored peer connection count +) + +// MeteredPeerEventType is the type of peer events emitted by a metered connection. +type MeteredPeerEventType int + +const ( + // PeerConnected is the type of event emitted when a peer successfully + // made the handshake. + PeerConnected MeteredPeerEventType = iota + + // PeerDisconnected is the type of event emitted when a peer disconnects. + PeerDisconnected + + // PeerHandshakeFailed is the type of event emitted when a peer fails to + // make the handshake or disconnects before the handshake. + PeerHandshakeFailed +) + +// MeteredPeerEvent is an event emitted when peers connect or disconnect. +type MeteredPeerEvent struct { + Type MeteredPeerEventType // Type of peer event + IP net.IP // IP address of the peer + ID string // NodeID of the peer + Elapsed time.Duration // Time elapsed between the connection and the handshake/disconnection + Ingress uint64 // Ingress count at the moment of the event + Egress uint64 // Egress count at the moment of the event +} + +// SubscribeMeteredPeerEvent registers a subscription for peer life-cycle events +// if metrics collection is enabled. +func SubscribeMeteredPeerEvent(ch chan<- MeteredPeerEvent) event.Subscription { + return meteredPeerFeed.Subscribe(ch) +} + // meteredConn is a wrapper around a net.Conn that meters both the // inbound and outbound network traffic. type meteredConn struct { net.Conn // Network connection to wrap with metering + + connected time.Time // Connection time of the peer + ip net.IP // IP address of the peer + id string // NodeID of the peer + + // trafficMetered denotes if the peer is registered in the traffic registries. + // Its value is true if the metered peer count doesn't reach the limit in the + // moment of the peer's connection. + trafficMetered bool + ingressMeter metrics.Meter // Meter for the read bytes of the peer + egressMeter metrics.Meter // Meter for the written bytes of the peer + + lock sync.RWMutex // Lock protecting the metered connection's internals } -// newMeteredConn creates a new metered connection, also bumping the ingress or -// egress connection meter. If the metrics system is disabled, this function -// returns the original object. -func newMeteredConn(conn net.Conn, ingress bool) net.Conn { +// newMeteredConn creates a new metered connection, bumps the ingress or egress +// connection meter and also increases the metered peer count. If the metrics +// system is disabled or the IP address is unspecified, this function returns +// the original object. +func newMeteredConn(conn net.Conn, ingress bool, ip net.IP) net.Conn { // Short circuit if metrics are disabled if !metrics.Enabled { return conn } - // Otherwise bump the connection counters and wrap the connection + if ip.IsUnspecified() { + log.Warn("Peer IP is unspecified") + return conn + } + // Bump the connection counters and wrap the connection if ingress { ingressConnectMeter.Mark(1) } else { egressConnectMeter.Mark(1) } - return &meteredConn{Conn: conn} + return &meteredConn{ + Conn: conn, + ip: ip, + connected: time.Now(), + } } -// Read delegates a network read to the underlying connection, bumping the ingress -// traffic meter along the way. +// Read delegates a network read to the underlying connection, bumping the common +// and the peer ingress traffic meters along the way. func (c *meteredConn) Read(b []byte) (n int, err error) { n, err = c.Conn.Read(b) ingressTrafficMeter.Mark(int64(n)) - return + c.lock.RLock() + if c.trafficMetered { + c.ingressMeter.Mark(int64(n)) + } + c.lock.RUnlock() + return n, err } -// Write delegates a network write to the underlying connection, bumping the -// egress traffic meter along the way. +// Write delegates a network write to the underlying connection, bumping the common +// and the peer egress traffic meters along the way. func (c *meteredConn) Write(b []byte) (n int, err error) { n, err = c.Conn.Write(b) egressTrafficMeter.Mark(int64(n)) - return + c.lock.RLock() + if c.trafficMetered { + c.egressMeter.Mark(int64(n)) + } + c.lock.RUnlock() + return n, err +} + +// handshakeDone is called when a peer handshake is done. Registers the peer to +// the ingress and the egress traffic registries using the peer's IP and node ID, +// also emits connect event. +func (c *meteredConn) handshakeDone(nodeID enode.ID) { + id := nodeID.String() + if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit { + // Don't register the peer in the traffic registries. + atomic.AddInt32(&meteredPeerCount, -1) + c.lock.Lock() + c.id, c.trafficMetered = id, false + c.lock.Unlock() + log.Warn("Metered peer count reached the limit") + } else { + key := fmt.Sprintf("%s/%s", c.ip, id) + c.lock.Lock() + c.id, c.trafficMetered = id, true + c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry) + c.egressMeter = metrics.NewRegisteredMeter(key, PeerEgressRegistry) + c.lock.Unlock() + } + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerConnected, + IP: c.ip, + ID: id, + Elapsed: time.Since(c.connected), + }) +} + +// Close delegates a close operation to the underlying connection, unregisters +// the peer from the traffic registries and emits close event. +func (c *meteredConn) Close() error { + err := c.Conn.Close() + c.lock.RLock() + if c.id == "" { + // If the peer disconnects before the handshake. + c.lock.RUnlock() + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerHandshakeFailed, + IP: c.ip, + Elapsed: time.Since(c.connected), + }) + return err + } + id := c.id + if !c.trafficMetered { + // If the peer isn't registered in the traffic registries. + c.lock.RUnlock() + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerDisconnected, + IP: c.ip, + ID: id, + }) + return err + } + ingress, egress := uint64(c.ingressMeter.Count()), uint64(c.egressMeter.Count()) + c.lock.RUnlock() + + // Decrement the metered peer count + atomic.AddInt32(&meteredPeerCount, -1) + + // Unregister the peer from the traffic registries + key := fmt.Sprintf("%s/%s", c.ip, id) + PeerIngressRegistry.Unregister(key) + PeerEgressRegistry.Unregister(key) + + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerDisconnected, + IP: c.ip, + ID: id, + Ingress: ingress, + Egress: egress, + }) + return err } diff --git a/p2p/server.go b/p2p/server.go index 6482c04019..38a881f7bd 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -864,7 +864,11 @@ func (srv *Server) listenLoop() { } } - fd = newMeteredConn(fd, true) + var ip net.IP + if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok { + ip = tcp.IP + } + fd = newMeteredConn(fd, true, ip) srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) go func() { srv.SetupConn(fd, inboundConn, nil) @@ -917,6 +921,9 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro } else { c.node = nodeFromConn(remotePubkey, c.fd) } + if conn, ok := c.fd.(*meteredConn); ok { + conn.handshakeDone(c.node.ID()) + } clog := srv.log.New("id", c.node.ID(), "addr", c.fd.RemoteAddr(), "conn", c.flags) err = srv.checkpoint(c, srv.posthandshake) if err != nil { From 6a7695e3676ef8275afd5eafde6e045b7a7ab024 Mon Sep 17 00:00:00 2001 From: Dmitrij Koniajev Date: Tue, 16 Oct 2018 01:47:25 +0300 Subject: [PATCH 012/100] ethdb, rpc: support building on js/wasm (#17709) The changes allow building WebAssembly applications which use ethclient.Client. --- ethdb/database.go | 70 ++------------------------------------- ethdb/database_js.go | 68 +++++++++++++++++++++++++++++++++++++ ethdb/database_js_test.go | 25 ++++++++++++++ ethdb/database_test.go | 2 ++ ethdb/table.go | 51 ++++++++++++++++++++++++++++ ethdb/table_batch.go | 51 ++++++++++++++++++++++++++++ rpc/ipc_js.go | 37 +++++++++++++++++++++ 7 files changed, 236 insertions(+), 68 deletions(-) create mode 100644 ethdb/database_js.go create mode 100644 ethdb/database_js_test.go create mode 100644 ethdb/table.go create mode 100644 ethdb/table_batch.go create mode 100644 rpc/ipc_js.go diff --git a/ethdb/database.go b/ethdb/database.go index 99abd09b99..6c62d6a386 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// +build !js + package ethdb import ( @@ -380,71 +382,3 @@ func (b *ldbBatch) Reset() { b.b.Reset() b.size = 0 } - -type table struct { - db Database - prefix string -} - -// NewTable returns a Database object that prefixes all keys with a given -// string. -func NewTable(db Database, prefix string) Database { - return &table{ - db: db, - prefix: prefix, - } -} - -func (dt *table) Put(key []byte, value []byte) error { - return dt.db.Put(append([]byte(dt.prefix), key...), value) -} - -func (dt *table) Has(key []byte) (bool, error) { - return dt.db.Has(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Get(key []byte) ([]byte, error) { - return dt.db.Get(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Delete(key []byte) error { - return dt.db.Delete(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Close() { - // Do nothing; don't close the underlying DB. -} - -type tableBatch struct { - batch Batch - prefix string -} - -// NewTableBatch returns a Batch object which prefixes all keys with a given string. -func NewTableBatch(db Database, prefix string) Batch { - return &tableBatch{db.NewBatch(), prefix} -} - -func (dt *table) NewBatch() Batch { - return &tableBatch{dt.db.NewBatch(), dt.prefix} -} - -func (tb *tableBatch) Put(key, value []byte) error { - return tb.batch.Put(append([]byte(tb.prefix), key...), value) -} - -func (tb *tableBatch) Delete(key []byte) error { - return tb.batch.Delete(append([]byte(tb.prefix), key...)) -} - -func (tb *tableBatch) Write() error { - return tb.batch.Write() -} - -func (tb *tableBatch) ValueSize() int { - return tb.batch.ValueSize() -} - -func (tb *tableBatch) Reset() { - tb.batch.Reset() -} diff --git a/ethdb/database_js.go b/ethdb/database_js.go new file mode 100644 index 0000000000..ba6eeb5a23 --- /dev/null +++ b/ethdb/database_js.go @@ -0,0 +1,68 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build js + +package ethdb + +import ( + "errors" +) + +var errNotSupported = errors.New("ethdb: not supported") + +type LDBDatabase struct { +} + +// NewLDBDatabase returns a LevelDB wrapped object. +func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { + return nil, errNotSupported +} + +// Path returns the path to the database directory. +func (db *LDBDatabase) Path() string { + return "" +} + +// Put puts the given key / value to the queue +func (db *LDBDatabase) Put(key []byte, value []byte) error { + return errNotSupported +} + +func (db *LDBDatabase) Has(key []byte) (bool, error) { + return false, errNotSupported +} + +// Get returns the given key if it's present. +func (db *LDBDatabase) Get(key []byte) ([]byte, error) { + return nil, errNotSupported +} + +// Delete deletes the key from the queue and database +func (db *LDBDatabase) Delete(key []byte) error { + return errNotSupported +} + +func (db *LDBDatabase) Close() { +} + +// Meter configures the database metrics collectors and +func (db *LDBDatabase) Meter(prefix string) { +} + +func (db *LDBDatabase) NewBatch() Batch { + return nil +} diff --git a/ethdb/database_js_test.go b/ethdb/database_js_test.go new file mode 100644 index 0000000000..b4c12ae0bc --- /dev/null +++ b/ethdb/database_js_test.go @@ -0,0 +1,25 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build js + +package ethdb_test + +import ( + "github.com/ethereum/go-ethereum/ethdb" +) + +var _ ethdb.Database = ðdb.LDBDatabase{} diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 74675cbe63..382fedbf9e 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// +build !js + package ethdb_test import ( diff --git a/ethdb/table.go b/ethdb/table.go new file mode 100644 index 0000000000..28069c078e --- /dev/null +++ b/ethdb/table.go @@ -0,0 +1,51 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethdb + +type table struct { + db Database + prefix string +} + +// NewTable returns a Database object that prefixes all keys with a given +// string. +func NewTable(db Database, prefix string) Database { + return &table{ + db: db, + prefix: prefix, + } +} + +func (dt *table) Put(key []byte, value []byte) error { + return dt.db.Put(append([]byte(dt.prefix), key...), value) +} + +func (dt *table) Has(key []byte) (bool, error) { + return dt.db.Has(append([]byte(dt.prefix), key...)) +} + +func (dt *table) Get(key []byte) ([]byte, error) { + return dt.db.Get(append([]byte(dt.prefix), key...)) +} + +func (dt *table) Delete(key []byte) error { + return dt.db.Delete(append([]byte(dt.prefix), key...)) +} + +func (dt *table) Close() { + // Do nothing; don't close the underlying DB. +} diff --git a/ethdb/table_batch.go b/ethdb/table_batch.go new file mode 100644 index 0000000000..ae83e79ced --- /dev/null +++ b/ethdb/table_batch.go @@ -0,0 +1,51 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethdb + +type tableBatch struct { + batch Batch + prefix string +} + +// NewTableBatch returns a Batch object which prefixes all keys with a given string. +func NewTableBatch(db Database, prefix string) Batch { + return &tableBatch{db.NewBatch(), prefix} +} + +func (dt *table) NewBatch() Batch { + return &tableBatch{dt.db.NewBatch(), dt.prefix} +} + +func (tb *tableBatch) Put(key, value []byte) error { + return tb.batch.Put(append([]byte(tb.prefix), key...), value) +} + +func (tb *tableBatch) Delete(key []byte) error { + return tb.batch.Delete(append([]byte(tb.prefix), key...)) +} + +func (tb *tableBatch) Write() error { + return tb.batch.Write() +} + +func (tb *tableBatch) ValueSize() int { + return tb.batch.ValueSize() +} + +func (tb *tableBatch) Reset() { + tb.batch.Reset() +} diff --git a/rpc/ipc_js.go b/rpc/ipc_js.go new file mode 100644 index 0000000000..eceef050e7 --- /dev/null +++ b/rpc/ipc_js.go @@ -0,0 +1,37 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build js + +package rpc + +import ( + "context" + "errors" + "net" +) + +var errNotSupported = errors.New("rpc: not supported") + +// ipcListen will create a named pipe on the given endpoint. +func ipcListen(endpoint string) (net.Listener, error) { + return nil, errNotSupported +} + +// newIPCConnection will connect to a named pipe with the given endpoint as name. +func newIPCConnection(ctx context.Context, endpoint string) (net.Conn, error) { + return nil, errNotSupported +} From a352de6a08db2c68383c7b1fdcf8267184b7dcea Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 16 Oct 2018 00:51:39 +0200 Subject: [PATCH 013/100] core/vm: add shortcuts for trivial exp cases (#16851) --- core/vm/instructions.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index e94a2777b4..b7c3ca5323 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -124,10 +124,22 @@ func opSmod(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory func opExp(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { base, exponent := stack.pop(), stack.pop() - stack.push(math.Exp(base, exponent)) - - interpreter.intPool.put(base, exponent) - + // some shortcuts + cmpToOne := exponent.Cmp(big1) + if cmpToOne < 0 { // Exponent is zero + // x ^ 0 == 1 + stack.push(base.SetUint64(1)) + } else if base.Sign() == 0 { + // 0 ^ y, if y != 0, == 0 + stack.push(base.SetUint64(0)) + } else if cmpToOne == 0 { // Exponent is one + // x ^ 1 == x + stack.push(base) + } else { + stack.push(math.Exp(base, exponent)) + interpreter.intPool.put(base) + } + interpreter.intPool.put(exponent) return nil, nil } From 6c313fff7bd7c1935d3193986df4c084b2e34c1c Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Mon, 15 Oct 2018 19:02:53 -0500 Subject: [PATCH 014/100] cmd/geth: don't set GOMAXPROCS by default (#17148) This is no longer needed because Go uses all CPUs by default. The change allows setting GOMAXPROCS in environment if needed. --- cmd/geth/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index fae4b57181..0288b33806 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -21,7 +21,6 @@ import ( "fmt" "math" "os" - "runtime" godebug "runtime/debug" "sort" "strconv" @@ -209,8 +208,6 @@ func init() { app.Flags = append(app.Flags, metricsFlags...) app.Before = func(ctx *cli.Context) error { - runtime.GOMAXPROCS(runtime.NumCPU()) - logdir := "" if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) { logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs") From 2868acd80b05e3dfca9f32c19372f22849e40eaf Mon Sep 17 00:00:00 2001 From: Smilenator Date: Tue, 16 Oct 2018 13:45:28 +0300 Subject: [PATCH 015/100] core/types: fix comment for func SignatureValues (#17921) --- core/types/transaction_signing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 1997755dca..63132048ee 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -136,7 +136,7 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true) } -// WithSignature returns a new transaction with the given signature. This signature +// SignatureValues returns signature values. This signature // needs to be in the [R || S || V] format where V is 0 or 1. func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig) From 4466c7b971837bded610dd2225ddc6065c0cb3c3 Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 16 Oct 2018 09:22:51 -0500 Subject: [PATCH 016/100] metrics: added NewCounterForced (#17919) --- metrics/counter.go | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/metrics/counter.go b/metrics/counter.go index c7f2b4bd3a..2f78c90d5c 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -1,6 +1,8 @@ package metrics -import "sync/atomic" +import ( + "sync/atomic" +) // Counters hold an int64 value that can be incremented and decremented. type Counter interface { @@ -20,6 +22,17 @@ func GetOrRegisterCounter(name string, r Registry) Counter { return r.GetOrRegister(name, NewCounter).(Counter) } +// GetOrRegisterCounterForced returns an existing Counter or constructs and registers a +// new Counter no matter the global switch is enabled or not. +// Be sure to unregister the counter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterCounterForced(name string, r Registry) Counter { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewCounterForced).(Counter) +} + // NewCounter constructs a new StandardCounter. func NewCounter() Counter { if !Enabled { @@ -28,6 +41,12 @@ func NewCounter() Counter { return &StandardCounter{0} } +// NewCounterForced constructs a new StandardCounter and returns it no matter if +// the global switch is enabled or not. +func NewCounterForced() Counter { + return &StandardCounter{0} +} + // NewRegisteredCounter constructs and registers a new StandardCounter. func NewRegisteredCounter(name string, r Registry) Counter { c := NewCounter() @@ -38,6 +57,19 @@ func NewRegisteredCounter(name string, r Registry) Counter { return c } +// NewRegisteredCounterForced constructs and registers a new StandardCounter +// and launches a goroutine no matter the global switch is enabled or not. +// Be sure to unregister the counter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredCounterForced(name string, r Registry) Counter { + c := NewCounterForced() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + // CounterSnapshot is a read-only copy of another Counter. type CounterSnapshot int64 From 4e693ad5a6f118dcf4f3586938c17fc8ef5822b5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 17 Oct 2018 14:46:59 +0200 Subject: [PATCH 017/100] swarm/tracing: disable stdout logging for opentracing (#17931) --- swarm/tracing/tracing.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/swarm/tracing/tracing.go b/swarm/tracing/tracing.go index b84cfb3102..f95fa41b8f 100644 --- a/swarm/tracing/tracing.go +++ b/swarm/tracing/tracing.go @@ -9,7 +9,6 @@ import ( "github.com/ethereum/go-ethereum/log" jaeger "github.com/uber/jaeger-client-go" jaegercfg "github.com/uber/jaeger-client-go/config" - jaegerlog "github.com/uber/jaeger-client-go/log" cli "gopkg.in/urfave/cli.v1" ) @@ -85,13 +84,13 @@ func initTracer(endpoint, svc string) (closer io.Closer) { // Example logger and metrics factory. Use github.com/uber/jaeger-client-go/log // and github.com/uber/jaeger-lib/metrics respectively to bind to real logging and metrics // frameworks. - jLogger := jaegerlog.StdLogger + //jLogger := jaegerlog.StdLogger //jMetricsFactory := metrics.NullFactory // Initialize tracer with a logger and a metrics factory closer, err := cfg.InitGlobalTracer( svc, - jaegercfg.Logger(jLogger), + //jaegercfg.Logger(jLogger), //jaegercfg.Metrics(jMetricsFactory), //jaegercfg.Observer(rpcmetrics.NewObserver(jMetricsFactory, rpcmetrics.DefaultNameNormalizer)), ) From cdf5982cfca2cd7d5fea85c226af5e48fde837df Mon Sep 17 00:00:00 2001 From: Attila Gazso Date: Wed, 17 Oct 2018 19:22:37 +0200 Subject: [PATCH 018/100] swarm: Lightnode mode: disable sync, retrieve, subscription (#17899) * swarm: Lightnode mode: disable sync, retrieve, subscription * swarm/network/stream: assign error and check in one line * swarm: restructured RegistryOption initializing * swarm: empty commit to retrigger CI build * swarm/network/stream: Added comments explaining RegistryOptions --- swarm/network/stream/delivery_test.go | 15 +- swarm/network/stream/lightnode_test.go | 210 ++++++++++++++++++ swarm/network/stream/messages.go | 11 +- .../network/stream/snapshot_retrieval_test.go | 1 + swarm/network/stream/snapshot_sync_test.go | 6 +- swarm/network/stream/stream.go | 22 +- swarm/swarm.go | 10 +- 7 files changed, 260 insertions(+), 15 deletions(-) create mode 100644 swarm/network/stream/lightnode_test.go diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index c6ebae3f0e..ec6148a0bf 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -75,7 +75,9 @@ func TestStreamerRetrieveRequest(t *testing.T) { } func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + DoServeRetrieve: true, + }) defer teardown() if err != nil { t.Fatal(err) @@ -127,7 +129,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // upstream request server receives a retrieve Request and responds with // offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { - tester, streamer, localStore, teardown, err := newStreamerTester(t, nil) + tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ + DoServeRetrieve: true, + }) defer teardown() if err != nil { t.Fatal(err) @@ -221,7 +225,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { - tester, streamer, localStore, teardown, err := newStreamerTester(t, nil) + tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ + DoServeRetrieve: true, + }) defer teardown() if err != nil { t.Fatal(err) @@ -336,7 +342,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - SkipCheck: skipCheck, + SkipCheck: skipCheck, + DoServeRetrieve: true, }) bucket.Store(bucketKeyRegistry, r) diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go new file mode 100644 index 0000000000..0d3bc7f544 --- /dev/null +++ b/swarm/network/stream/lightnode_test.go @@ -0,0 +1,210 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package stream + +import ( + "testing" + + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" +) + +// This test checks the default behavior of the server, that is +// when it is serving Retrieve requests. +func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { + registryOptions := &RegistryOptions{ + DoServeRetrieve: true, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + stream := NewStream(swarmChunkServerStreamName, "", false) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "SubscribeMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + }, + Peer: node.ID(), + }, + }, + }) + if err != nil { + t.Fatalf("Got %v", err) + } + + err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: node.ID()}) + if err == nil || err.Error() != "timed out waiting for peers to disconnect" { + t.Fatalf("Expected no disconnect, got %v", err) + } +} + +// This test checks the Lightnode behavior of server, when serving Retrieve +// requests are disabled +func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { + registryOptions := &RegistryOptions{ + DoServeRetrieve: false, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + stream := NewStream(swarmChunkServerStreamName, "", false) + + err = tester.TestExchanges( + p2ptest.Exchange{ + Label: "SubscribeMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream RETRIEVE_REQUEST not registered", + }, + Peer: node.ID(), + }, + }, + }) + if err != nil { + t.Fatalf("Got %v", err) + } +} + +// This test checks the default behavior of the server, that is +// when syncing is enabled. +func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { + registryOptions := &RegistryOptions{ + DoSync: true, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + syncStream := NewStream("SYNC", FormatSyncBinKey(1), false) + + err = tester.TestExchanges( + p2ptest.Exchange{ + Label: "RequestSubscription", + Triggers: []p2ptest.Trigger{ + { + Code: 8, + Msg: &RequestSubscriptionMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatalf("Got %v", err) + } +} + +// This test checks the Lightnode behavior of the server, that is +// when syncing is disabled. +func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) { + registryOptions := &RegistryOptions{ + DoSync: false, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + syncStream := NewStream("SYNC", FormatSyncBinKey(1), false) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RequestSubscription", + Triggers: []p2ptest.Trigger{ + { + Code: 8, + Msg: &RequestSubscriptionMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream SYNC not registered", + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ + Label: "RequestSubscription", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream SYNC not registered", + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatalf("Got %v", err) + } +} diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 74c785d587..68503fe1f0 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -76,7 +76,16 @@ type RequestSubscriptionMsg struct { func (p *Peer) handleRequestSubscription(ctx context.Context, req *RequestSubscriptionMsg) (err error) { log.Debug(fmt.Sprintf("handleRequestSubscription: streamer %s to subscribe to %s with stream %s", p.streamer.addr, p.ID(), req.Stream)) - return p.streamer.Subscribe(p.ID(), req.Stream, req.History, req.Priority) + if err = p.streamer.Subscribe(p.ID(), req.Stream, req.History, req.Priority); err != nil { + // The error will be sent as a subscribe error message + // and will not be returned as it will prevent any new message + // exchange between peers over p2p. Instead, error will be returned + // only if there is one from sending subscribe error message. + err = p.Send(ctx, SubscribeErrorMsg{ + Error: err.Error(), + }) + } + return err } func (p *Peer) handleSubscribeMsg(ctx context.Context, req *SubscribeMsg) (err error) { diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 09d915d48d..7625e02d7a 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -130,6 +130,7 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no DoSync: true, SyncUpdateDelay: 3 * time.Second, DoRetrieve: true, + DoServeRetrieve: true, }) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 0d58494873..96e92c5cf1 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -166,6 +166,7 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ DoSync: true, + DoServeRetrieve: true, SyncUpdateDelay: 3 * time.Second, }) @@ -358,7 +359,10 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) delivery := NewDelivery(kad, netStore) netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), nil) + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + DoServeRetrieve: true, + DoSync: true, + }) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 3861cfcf6a..1dc2a8cba8 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -66,8 +66,9 @@ type Registry struct { // RegistryOptions holds optional values for NewRegistry constructor. type RegistryOptions struct { SkipCheck bool - DoSync bool - DoRetrieve bool + DoSync bool // Sets if the server syncs with peers. Default is true, set to false by lightnode or nosync flags. + DoRetrieve bool // Sets if the server issues Retrieve requests. Default is true. + DoServeRetrieve bool // Sets if the server serves Retrieve requests. Default is true, set to false by lightnode flag. SyncUpdateDelay time.Duration MaxPeerServers int // The limit of servers for each peer in registry } @@ -93,14 +94,21 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy } streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer - streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { - return NewSwarmChunkServer(delivery.chunkStore), nil - }) + + if options.DoServeRetrieve { + streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { + return NewSwarmChunkServer(delivery.chunkStore), nil + }) + } + streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) }) - RegisterSwarmSyncerServer(streamer, syncChunkStore) - RegisterSwarmSyncerClient(streamer, syncChunkStore) + + if options.DoSync { + RegisterSwarmSyncerServer(streamer, syncChunkStore) + RegisterSwarmSyncerClient(streamer, syncChunkStore) + } if options.DoSync { // latestIntC function ensures that diff --git a/swarm/swarm.go b/swarm/swarm.go index aea0989a1e..7214abbda4 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -175,13 +175,19 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil { return nil, err } - self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, &stream.RegistryOptions{ + registryOptions := &stream.RegistryOptions{ SkipCheck: config.DeliverySkipCheck, DoSync: config.SyncEnabled, DoRetrieve: true, + DoServeRetrieve: true, SyncUpdateDelay: config.SyncUpdateDelay, MaxPeerServers: config.MaxStreamPeerServers, - }) + } + if config.LightNodeEnabled { + registryOptions.DoSync = false + registryOptions.DoRetrieve = false + } + self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) From 97fb08342d227bbd126516083b0ddaf74e6e8468 Mon Sep 17 00:00:00 2001 From: Simon Jentzsch Date: Thu, 18 Oct 2018 21:41:22 +0200 Subject: [PATCH 019/100] EIP-1186 eth_getProof (#17737) * first impl of eth_getProof * fixed docu * added comments and refactored based on comments from holiman * created structs * handle errors correctly * change Value to *hexutil.Big in order to have the same output as parity * use ProofList as return type --- common/bytes.go | 9 ++++++ core/state/statedb.go | 20 +++++++++++++ core/vm/interface.go | 10 +++++++ internal/ethapi/api.go | 66 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+) diff --git a/common/bytes.go b/common/bytes.go index 0c257a1ee0..c82e616241 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -31,6 +31,15 @@ func ToHex(b []byte) string { return "0x" + hex } +// ToHexArray creates a array of hex-string based on []byte +func ToHexArray(b [][]byte) []string { + r := make([]string, len(b)) + for i := range b { + r[i] = ToHex(b[i]) + } + return r +} + // FromHex returns the bytes represented by the hexadecimal string s. // s may be prefixed with "0x". func FromHex(s string) []byte { diff --git a/core/state/statedb.go b/core/state/statedb.go index 216667ce98..b1bb53ed04 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,6 +18,7 @@ package state import ( + "errors" "fmt" "math/big" "sort" @@ -25,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -256,6 +258,24 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash return common.Hash{} } +// GetProof returns the MerkleProof for a given Account +func (self *StateDB) GetProof(a common.Address) (vm.ProofList, error) { + var proof vm.ProofList + err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) + return proof, err +} + +// GetProof returns the StorageProof for given key +func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) (vm.ProofList, error) { + var proof vm.ProofList + trie := self.StorageTrie(a) + if trie == nil { + return proof, errors.New("storage trie for requested address does not exist") + } + err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + return proof, err +} + // GetCommittedState retrieves a value from the given account's committed storage trie. func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := self.getStateObject(addr) diff --git a/core/vm/interface.go b/core/vm/interface.go index fc15082f18..1ae98cf7a2 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -35,6 +35,8 @@ type StateDB interface { SetNonce(common.Address, uint64) GetCodeHash(common.Address) common.Hash + GetProof(common.Address) (ProofList, error) + GetStorageProof(common.Address, common.Hash) (ProofList, error) GetCode(common.Address) []byte SetCode(common.Address, []byte) GetCodeSize(common.Address) int @@ -78,3 +80,11 @@ type CallContext interface { // Create a new contract Create(env *EVM, me ContractRef, data []byte, gas, value *big.Int) ([]byte, common.Address, error) } + +// MerkleProof +type ProofList [][]byte + +func (n *ProofList) Put(key []byte, value []byte) error { + *n = append(*n, value) + return nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 196be43e03..0c751c3282 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -508,6 +508,72 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add return (*hexutil.Big)(state.GetBalance(address)), state.Error() } +// Result structs for GetProof +type AccountResult struct { + Address common.Address `json:"address"` + AccountProof []string `json:"accountProof"` + Balance *hexutil.Big `json:"balance"` + CodeHash common.Hash `json:"codeHash"` + Nonce hexutil.Uint64 `json:"nonce"` + StorageHash common.Hash `json:"storageHash"` + StorageProof []StorageResult `json:"storageProof"` +} +type StorageResult struct { + Key string `json:"key"` + Value *hexutil.Big `json:"value"` + Proof []string `json:"proof"` +} + +// GetProof returns the Merkle-proof for a given account and optionally some storage keys. +func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNr rpc.BlockNumber) (*AccountResult, error) { + state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) + if state == nil || err != nil { + return nil, err + } + + storageTrie := state.StorageTrie(address) + storageHash := types.EmptyRootHash + codeHash := state.GetCodeHash(address) + storageProof := make([]StorageResult, len(storageKeys)) + + // if we have a storageTrie, (which means the account exists), we can update the storagehash + if storageTrie != nil { + storageHash = storageTrie.Hash() + } else { + // no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray. + codeHash = crypto.Keccak256Hash(nil) + } + + // create the proof for the storageKeys + for i, key := range storageKeys { + if storageTrie != nil { + proof, storageError := state.GetStorageProof(address, common.HexToHash(key)) + if storageError != nil { + return nil, storageError + } + storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)} + } else { + storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}} + } + } + + // create the accountProof + accountProof, proofErr := state.GetProof(address) + if proofErr != nil { + return nil, proofErr + } + + return &AccountResult{ + Address: address, + AccountProof: common.ToHexArray(accountProof), + Balance: (*hexutil.Big)(state.GetBalance(address)), + CodeHash: codeHash, + Nonce: hexutil.Uint64(state.GetNonce(address)), + StorageHash: storageHash, + StorageProof: storageProof, + }, state.Error() +} + // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all // transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { From aeb733623e79d9b8f03036ec1a4c0717185b527d Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 19 Oct 2018 10:50:25 +0200 Subject: [PATCH 020/100] swarm/network: disallow historical retrieval requests (#17936) --- swarm/network/stream/delivery_test.go | 2 +- swarm/network/stream/messages.go | 1 + swarm/network/stream/snapshot_retrieval_test.go | 4 ++-- swarm/network/stream/snapshot_sync_test.go | 4 ++-- swarm/network/stream/stream.go | 7 +++++-- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index ec6148a0bf..949645558b 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -416,7 +416,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return fmt.Errorf("No registry") } registry := item.(*Registry) - err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), NewRange(0, 0), Top) + err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), nil, Top) if err != nil { return err } diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 68503fe1f0..0fe3e5eb4f 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -158,6 +158,7 @@ type SubscribeErrorMsg struct { } func (p *Peer) handleSubscribeErrorMsg(req *SubscribeErrorMsg) (err error) { + //TODO the error should be channeled to whoever calls the subscribe return fmt.Errorf("subscribe to peer %s: %v", p.ID(), req.Error) } diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 7625e02d7a..b81cfc0ca0 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -218,7 +218,7 @@ func runFileRetrievalTest(nodeCount int) error { reader, _ := fileStore.Retrieve(context.TODO(), hash) //check that we can read the file size and that it corresponds to the generated file size if s, err := reader.Size(ctx, nil); err != nil || s != int64(len(randomFiles[i])) { - log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id) + log.Debug("Retrieve error", "err", err, "hash", hash, "nodeId", id) time.Sleep(500 * time.Millisecond) continue REPEAT } @@ -309,7 +309,7 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { reader, _ := fileStore.Retrieve(context.TODO(), hash) //check that we can read the chunk size and that it corresponds to the generated chunk size if s, err := reader.Size(ctx, nil); err != nil || s != int64(chunkSize) { - log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id, "size", s) + log.Debug("Retrieve error", "err", err, "hash", hash, "nodeId", id, "size", s) time.Sleep(500 * time.Millisecond) continue REPEAT } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 96e92c5cf1..8d89f28cba 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -310,7 +310,7 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio _, err = lstore.Get(ctx, chunk) } if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) + log.Debug(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) // Do not get crazy with logging the warn message time.Sleep(500 * time.Millisecond) continue REPEAT @@ -514,7 +514,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) _, err = lstore.Get(ctx, chunk) } if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) + log.Debug(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) // Do not get crazy with logging the warn message time.Sleep(500 * time.Millisecond) continue REPEAT diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 1dc2a8cba8..9d0e6c68b8 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -18,6 +18,7 @@ package stream import ( "context" + "errors" "fmt" "math" "sync" @@ -96,7 +97,10 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy delivery.getPeer = streamer.getPeer if options.DoServeRetrieve { - streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { + streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) { + if !live { + return nil, errors.New("only live retrieval requests supported") + } return NewSwarmChunkServer(delivery.chunkStore), nil }) } @@ -279,7 +283,6 @@ func (r *Registry) Subscribe(peerId enode.ID, s Stream, h *Range, priority uint8 if err != nil { return err } - if s.Live && h != nil { if err := peer.setClientParams( getHistoryStream(s), From d98c45f70f5c9e414193ff4f79fbfc04a3d7751f Mon Sep 17 00:00:00 2001 From: Wuxiang Date: Fri, 19 Oct 2018 21:33:27 +0800 Subject: [PATCH 021/100] core: fix a typo (#17941) --- core/rawdb/accessors_chain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index da54328326..6660e17de1 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -278,7 +278,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece if len(data) == 0 { return nil } - // Convert the revceipts from their storage form to their internal representation + // Convert the receipts from their storage form to their internal representation storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { log.Error("Invalid receipt array RLP", "hash", hash, "err", err) From 6ff97bf2e5623d3f6eb09e2630c3fdbd770acf0b Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Fri, 19 Oct 2018 08:40:10 -0500 Subject: [PATCH 022/100] accounts: wallet derivation path comment is mistaken (#17934) --- accounts/hd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accounts/hd.go b/accounts/hd.go index 277f688e48..6ed6318078 100644 --- a/accounts/hd.go +++ b/accounts/hd.go @@ -30,8 +30,8 @@ import ( var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0} // DefaultBaseDerivationPath is the base path from which custom derivation endpoints -// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second -// at m/44'/60'/0'/1, etc. +// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second +// at m/44'/60'/0'/0/1, etc. var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0} // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints From 75060ef96ec38b0c6ca4dba4f4543e594025507f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 19 Oct 2018 15:41:27 +0200 Subject: [PATCH 023/100] cmd/bootnode: fix -writeaddress output (#17932) --- cmd/bootnode/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 346523ddb6..32f7d63bea 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -38,7 +38,7 @@ func main() { var ( listenAddr = flag.String("addr", ":30301", "listen address") genKey = flag.String("genkey", "", "generate a node key") - writeAddr = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit") + writeAddr = flag.Bool("writeaddress", false, "write out the node's public key and quit") nodeKeyFile = flag.String("nodekey", "", "private key filename") nodeKeyHex = flag.String("nodekeyhex", "", "private key as hex (for testing)") natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:)") @@ -86,7 +86,7 @@ func main() { } if *writeAddr { - fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey)) + fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:]) os.Exit(0) } From 66debd91d9268067000c061093a674ce34f18d48 Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 19 Oct 2018 16:02:44 +0200 Subject: [PATCH 024/100] swarm/api/http: remove ModTime=now for direct and multipart uploads (#17945) --- swarm/api/http/server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 370aca5a75..b4294b0581 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -409,7 +409,6 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api Path: path, ContentType: part.Header.Get("Content-Type"), Size: size, - ModTime: time.Now(), } log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path) contentKey, err := mw.AddEntry(r.Context(), reader, entry) @@ -428,7 +427,6 @@ func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) err ContentType: r.Header.Get("Content-Type"), Mode: 0644, Size: r.ContentLength, - ModTime: time.Now(), }) if err != nil { return err From 88b41a9e680a764aa079051aa7c71b3c6879d60a Mon Sep 17 00:00:00 2001 From: holisticode Date: Sun, 21 Oct 2018 02:30:41 -0500 Subject: [PATCH 025/100] =?UTF-8?q?swarm/network/stream:=20disambiguate=20?= =?UTF-8?q?chunk=20delivery=20messages=20(retrieval=E2=80=A6=20(#17920)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * swarm/network/stream: disambiguate chunk delivery messages (retrieval vs syncing) * swarm/network/stream: addressed PR comments * swarm/network/stream: stream protocol version change due to new message types in this PR --- swarm/network/stream/delivery.go | 13 ++++++++++++- swarm/network/stream/messages.go | 3 ++- swarm/network/stream/peer.go | 29 +++++++++++++++++++++++------ swarm/network/stream/stream.go | 14 ++++++++++---- 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 0429c4dffc..9092ffe3ed 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -173,7 +173,8 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return } if req.SkipCheck { - err = sp.Deliver(ctx, chunk, s.priority) + syncing := false + err = sp.Deliver(ctx, chunk, s.priority, syncing) if err != nil { log.Warn("ERROR in handleRetrieveRequestMsg", "err", err) } @@ -189,12 +190,22 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return nil } +//Chunk delivery always uses the same message type.... type ChunkDeliveryMsg struct { Addr storage.Address SData []byte // the stored chunk Data (incl size) peer *Peer // set in handleChunkDeliveryMsg } +//...but swap accounting needs to disambiguate if it is a delivery for syncing or for retrieval +//as it decides based on message type if it needs to account for this message or not + +//defines a chunk delivery for retrieval (with accounting) +type ChunkDeliveryMsgRetrieval ChunkDeliveryMsg + +//defines a chunk delivery for syncing (without accounting) +type ChunkDeliveryMsgSyncing ChunkDeliveryMsg + // TODO: Fix context SNAFU func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *ChunkDeliveryMsg) error { var osp opentracing.Span diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 0fe3e5eb4f..eb1b2983e1 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -357,7 +357,8 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err) } chunk := storage.NewChunk(hash, data) - if err := p.Deliver(ctx, chunk, s.priority); err != nil { + syncing := true + if err := p.Deliver(ctx, chunk, s.priority, syncing); err != nil { return err } } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 89d135ad52..4bccf56f5d 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -128,17 +128,34 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { } // Deliver sends a storeRequestMsg protocol message to the peer -func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8) error { +// Depending on the `syncing` parameter we send different message types +func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8, syncing bool) error { var sp opentracing.Span + var msg interface{} + + spanName := "send.chunk.delivery" + + //we send different types of messages if delivery is for syncing or retrievals, + //even if handling and content of the message are the same, + //because swap accounting decides which messages need accounting based on the message type + if syncing { + msg = &ChunkDeliveryMsgSyncing{ + Addr: chunk.Address(), + SData: chunk.Data(), + } + spanName += ".syncing" + } else { + msg = &ChunkDeliveryMsgRetrieval{ + Addr: chunk.Address(), + SData: chunk.Data(), + } + spanName += ".retrieval" + } ctx, sp = spancontext.StartSpan( ctx, - "send.chunk.delivery") + spanName) defer sp.Finish() - msg := &ChunkDeliveryMsg{ - Addr: chunk.Address(), - SData: chunk.Data(), - } return p.SendPriority(ctx, msg, priority) } diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 9d0e6c68b8..0ac374def1 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -489,8 +489,13 @@ func (p *Peer) HandleMsg(ctx context.Context, msg interface{}) error { case *WantedHashesMsg: return p.handleWantedHashesMsg(ctx, msg) - case *ChunkDeliveryMsg: - return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, msg) + case *ChunkDeliveryMsgRetrieval: + //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) + + case *ChunkDeliveryMsgSyncing: + //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) case *RetrieveRequestMsg: return p.streamer.delivery.handleRetrieveRequestMsg(ctx, p, msg) @@ -681,7 +686,7 @@ func (c *clientParams) clientCreated() { // Spec is the spec of the streamer protocol var Spec = &protocols.Spec{ Name: "stream", - Version: 7, + Version: 8, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ UnsubscribeMsg{}, @@ -690,10 +695,11 @@ var Spec = &protocols.Spec{ TakeoverProofMsg{}, SubscribeMsg{}, RetrieveRequestMsg{}, - ChunkDeliveryMsg{}, + ChunkDeliveryMsgRetrieval{}, SubscribeErrorMsg{}, RequestSubscriptionMsg{}, QuitMsg{}, + ChunkDeliveryMsgSyncing{}, }, } From 3088c122d8497acf176f03a3f19f6292e817cab7 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Tue, 23 Oct 2018 06:21:16 -0500 Subject: [PATCH 026/100] eth/downloader: fix comment typos (#17956) --- eth/downloader/downloader.go | 6 +++--- eth/downloader/queue.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 9cfc8a978d..f01a8fdbd8 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -1246,7 +1246,7 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er } // If no headers were retrieved at all, the peer violated its TD promise that it had a // better chain compared to ours. The only exception is if its promised blocks were - // already imported by other means (e.g. fecher): + // already imported by other means (e.g. fetcher): // // R , L : Both at block 10 // R: Mine block 11, and propagate it to L @@ -1415,7 +1415,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { defer stateSync.Cancel() go func() { if err := stateSync.Wait(); err != nil && err != errCancelStateFetch { - d.queue.Close() // wake up WaitResults + d.queue.Close() // wake up Results } }() // Figure out the ideal pivot block. Note, that this goalpost may move if the @@ -1473,7 +1473,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { defer stateSync.Cancel() go func() { if err := stateSync.Wait(); err != nil && err != errCancelStateFetch { - d.queue.Close() // wake up WaitResults + d.queue.Close() // wake up Results } }() oldPivot = P diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index a0e8a6d48c..c6b635aff1 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -143,7 +143,7 @@ func (q *queue) Reset() { q.resultOffset = 0 } -// Close marks the end of the sync, unblocking WaitResults. +// Close marks the end of the sync, unblocking Results. // It may be called even if the queue is already closed. func (q *queue) Close() { q.lock.Lock() @@ -545,7 +545,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common taskQueue.Push(header, -int64(header.Number.Uint64())) } if progress { - // Wake WaitResults, resultCache was modified + // Wake Results, resultCache was modified q.active.Signal() } // Assemble and return the block download request @@ -857,7 +857,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ taskQueue.Push(header, -int64(header.Number.Uint64())) } } - // Wake up WaitResults + // Wake up Results if accepted > 0 { q.active.Signal() } From 4c0883e20d78b987dc95acd46498f326626aaee3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 23 Oct 2018 16:28:18 +0200 Subject: [PATCH 027/100] core/vm: adds refund as part of the json standard trace (#17910) This adds the global accumulated refund counter to the standard json output as a numeric json value. Previously this was not very interesting since it was not used much, but with the new sstore gas changes the value is a lot more interesting from a consensus investigation perspective. --- cmd/evm/json_logger.go | 17 +++++++------ core/vm/gen_structlog.go | 52 ++++++++++++++++++++++---------------- core/vm/logger.go | 23 +++++++++-------- core/vm/logger_test.go | 11 +++++--- eth/tracers/tracer.go | 16 ++++++++---- eth/tracers/tracer_test.go | 11 ++++++-- 6 files changed, 79 insertions(+), 51 deletions(-) diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go index f16424fbe6..50cb4f0e42 100644 --- a/cmd/evm/json_logger.go +++ b/cmd/evm/json_logger.go @@ -45,14 +45,15 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create // CaptureState outputs state information on the logger. func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { log := vm.StructLog{ - Pc: pc, - Op: op, - Gas: gas, - GasCost: cost, - MemorySize: memory.Len(), - Storage: nil, - Depth: depth, - Err: err, + Pc: pc, + Op: op, + Gas: gas, + GasCost: cost, + MemorySize: memory.Len(), + Storage: nil, + Depth: depth, + RefundCounter: env.StateDB.GetRefund(), + Err: err, } if !l.cfg.DisableMemory { log.Memory = memory.Data() diff --git a/core/vm/gen_structlog.go b/core/vm/gen_structlog.go index ade3ca6318..726012e59e 100644 --- a/core/vm/gen_structlog.go +++ b/core/vm/gen_structlog.go @@ -13,20 +13,22 @@ import ( var _ = (*structLogMarshaling)(nil) +// MarshalJSON marshals as JSON. func (s StructLog) MarshalJSON() ([]byte, error) { type StructLog struct { - Pc uint64 `json:"pc"` - Op OpCode `json:"op"` - Gas math.HexOrDecimal64 `json:"gas"` - GasCost math.HexOrDecimal64 `json:"gasCost"` - Memory hexutil.Bytes `json:"memory"` - MemorySize int `json:"memSize"` - Stack []*math.HexOrDecimal256 `json:"stack"` - Storage map[common.Hash]common.Hash `json:"-"` - Depth int `json:"depth"` - Err error `json:"-"` - OpName string `json:"opName"` - ErrorString string `json:"error"` + Pc uint64 `json:"pc"` + Op OpCode `json:"op"` + Gas math.HexOrDecimal64 `json:"gas"` + GasCost math.HexOrDecimal64 `json:"gasCost"` + Memory hexutil.Bytes `json:"memory"` + MemorySize int `json:"memSize"` + Stack []*math.HexOrDecimal256 `json:"stack"` + Storage map[common.Hash]common.Hash `json:"-"` + Depth int `json:"depth"` + RefundCounter uint64 `json:"refund"` + Err error `json:"-"` + OpName string `json:"opName"` + ErrorString string `json:"error"` } var enc StructLog enc.Pc = s.Pc @@ -43,24 +45,27 @@ func (s StructLog) MarshalJSON() ([]byte, error) { } enc.Storage = s.Storage enc.Depth = s.Depth + enc.RefundCounter = s.RefundCounter enc.Err = s.Err enc.OpName = s.OpName() enc.ErrorString = s.ErrorString() return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (s *StructLog) UnmarshalJSON(input []byte) error { type StructLog struct { - Pc *uint64 `json:"pc"` - Op *OpCode `json:"op"` - Gas *math.HexOrDecimal64 `json:"gas"` - GasCost *math.HexOrDecimal64 `json:"gasCost"` - Memory *hexutil.Bytes `json:"memory"` - MemorySize *int `json:"memSize"` - Stack []*math.HexOrDecimal256 `json:"stack"` - Storage map[common.Hash]common.Hash `json:"-"` - Depth *int `json:"depth"` - Err error `json:"-"` + Pc *uint64 `json:"pc"` + Op *OpCode `json:"op"` + Gas *math.HexOrDecimal64 `json:"gas"` + GasCost *math.HexOrDecimal64 `json:"gasCost"` + Memory *hexutil.Bytes `json:"memory"` + MemorySize *int `json:"memSize"` + Stack []*math.HexOrDecimal256 `json:"stack"` + Storage map[common.Hash]common.Hash `json:"-"` + Depth *int `json:"depth"` + RefundCounter *uint64 `json:"refund"` + Err error `json:"-"` } var dec StructLog if err := json.Unmarshal(input, &dec); err != nil { @@ -96,6 +101,9 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { if dec.Depth != nil { s.Depth = *dec.Depth } + if dec.RefundCounter != nil { + s.RefundCounter = *dec.RefundCounter + } if dec.Err != nil { s.Err = dec.Err } diff --git a/core/vm/logger.go b/core/vm/logger.go index 85acb8d6d3..1733bf2700 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -56,16 +56,17 @@ type LogConfig struct { // StructLog is emitted to the EVM each cycle and lists information about the current internal state // prior to the execution of the statement. type StructLog struct { - Pc uint64 `json:"pc"` - Op OpCode `json:"op"` - Gas uint64 `json:"gas"` - GasCost uint64 `json:"gasCost"` - Memory []byte `json:"memory"` - MemorySize int `json:"memSize"` - Stack []*big.Int `json:"stack"` - Storage map[common.Hash]common.Hash `json:"-"` - Depth int `json:"depth"` - Err error `json:"-"` + Pc uint64 `json:"pc"` + Op OpCode `json:"op"` + Gas uint64 `json:"gas"` + GasCost uint64 `json:"gasCost"` + Memory []byte `json:"memory"` + MemorySize int `json:"memSize"` + Stack []*big.Int `json:"stack"` + Storage map[common.Hash]common.Hash `json:"-"` + Depth int `json:"depth"` + RefundCounter uint64 `json:"refund"` + Err error `json:"-"` } // overrides for gencodec @@ -177,7 +178,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui storage = l.changedValues[contract.Address()].Copy() } // create a new snaptshot of the EVM. - log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, err} + log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, env.StateDB.GetRefund(), err} l.logs = append(l.logs, log) return nil diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index cdbb70dc42..cba7c7a0ef 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/params" ) @@ -41,9 +42,15 @@ func (d *dummyContractRef) SetBalance(*big.Int) {} func (d *dummyContractRef) SetNonce(uint64) {} func (d *dummyContractRef) Balance() *big.Int { return new(big.Int) } +type dummyStatedb struct { + state.StateDB +} + +func (dummyStatedb) GetRefund() uint64 { return 1337 } + func TestStoreCapture(t *testing.T) { var ( - env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, &dummyStatedb{}, params.TestChainConfig, Config{}) logger = NewStructLogger(nil) mem = NewMemory() stack = newstack() @@ -51,9 +58,7 @@ func TestStoreCapture(t *testing.T) { ) stack.push(big.NewInt(1)) stack.push(big.NewInt(0)) - var index common.Hash - logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, contract, 0, nil) if len(logger.changedValues[contract.Address()]) == 0 { t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()])) diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index b519236f2d..3533a831f1 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -290,11 +290,12 @@ type Tracer struct { contractWrapper *contractWrapper // Wrapper around the contract object dbWrapper *dbWrapper // Wrapper around the VM environment - pcValue *uint // Swappable pc value wrapped by a log accessor - gasValue *uint // Swappable gas value wrapped by a log accessor - costValue *uint // Swappable cost value wrapped by a log accessor - depthValue *uint // Swappable depth value wrapped by a log accessor - errorValue *string // Swappable error value wrapped by a log accessor + pcValue *uint // Swappable pc value wrapped by a log accessor + gasValue *uint // Swappable gas value wrapped by a log accessor + costValue *uint // Swappable cost value wrapped by a log accessor + depthValue *uint // Swappable depth value wrapped by a log accessor + errorValue *string // Swappable error value wrapped by a log accessor + refundValue *uint // Swappable refund value wrapped by a log accessor ctx map[string]interface{} // Transaction context gathered throughout execution err error // Error, if one has occurred @@ -323,6 +324,7 @@ func New(code string) (*Tracer, error) { gasValue: new(uint), costValue: new(uint), depthValue: new(uint), + refundValue: new(uint), } // Set up builtins for this environment tracer.vm.PushGlobalGoFunction("toHex", func(ctx *duktape.Context) int { @@ -442,6 +444,9 @@ func New(code string) (*Tracer, error) { tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.depthValue); return 1 }) tracer.vm.PutPropString(logObject, "getDepth") + tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.refundValue); return 1 }) + tracer.vm.PutPropString(logObject, "getRefund") + tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { if tracer.errorValue != nil { ctx.PushString(*tracer.errorValue) @@ -527,6 +532,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost *jst.gasValue = uint(gas) *jst.costValue = uint(cost) *jst.depthValue = uint(depth) + *jst.refundValue = uint(env.StateDB.GetRefund()) jst.errorValue = nil if err != nil { diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 58b6247245..52f29c83fc 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" ) @@ -43,8 +44,14 @@ func (account) ReturnGas(*big.Int) {} func (account) SetCode(common.Hash, []byte) {} func (account) ForEachStorage(cb func(key, value common.Hash) bool) {} +type dummyStatedb struct { + state.StateDB +} + +func (dummyStatedb) GetRefund() uint64 { return 1337 } + func runTrace(tracer *Tracer) (json.RawMessage, error) { - env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) + env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} @@ -126,7 +133,7 @@ func TestHaltBetweenSteps(t *testing.T) { t.Fatal(err) } - env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) + env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil) From 7f22b59f87cfda85a14eed6481da68e3dfba44a1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 23 Oct 2018 21:51:41 +0200 Subject: [PATCH 028/100] core/state: simplify proof methods (#17965) This fixes the import cycle build error in core/vm tests. There is no need to refer to core/vm for a type definition. --- core/state/statedb.go | 20 +++++++++++++------- core/vm/interface.go | 10 ---------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index b1bb53ed04..f0d7cdb6eb 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -46,6 +45,13 @@ var ( emptyCode = crypto.Keccak256Hash(nil) ) +type proofList [][]byte + +func (n *proofList) Put(key []byte, value []byte) error { + *n = append(*n, value) + return nil +} + // StateDBs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: @@ -259,21 +265,21 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash } // GetProof returns the MerkleProof for a given Account -func (self *StateDB) GetProof(a common.Address) (vm.ProofList, error) { - var proof vm.ProofList +func (self *StateDB) GetProof(a common.Address) ([][]byte, error) { + var proof proofList err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) - return proof, err + return [][]byte(proof), err } // GetProof returns the StorageProof for given key -func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) (vm.ProofList, error) { - var proof vm.ProofList +func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { + var proof proofList trie := self.StorageTrie(a) if trie == nil { return proof, errors.New("storage trie for requested address does not exist") } err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) - return proof, err + return [][]byte(proof), err } // GetCommittedState retrieves a value from the given account's committed storage trie. diff --git a/core/vm/interface.go b/core/vm/interface.go index 1ae98cf7a2..fc15082f18 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -35,8 +35,6 @@ type StateDB interface { SetNonce(common.Address, uint64) GetCodeHash(common.Address) common.Hash - GetProof(common.Address) (ProofList, error) - GetStorageProof(common.Address, common.Hash) (ProofList, error) GetCode(common.Address) []byte SetCode(common.Address, []byte) GetCodeSize(common.Address) int @@ -80,11 +78,3 @@ type CallContext interface { // Create a new contract Create(env *EVM, me ContractRef, data []byte, gas, value *big.Int) ([]byte, common.Address, error) } - -// MerkleProof -type ProofList [][]byte - -func (n *ProofList) Put(key []byte, value []byte) error { - *n = append(*n, value) - return nil -} From 68109336405a36e3c3fdcab6a87e8bc024a44a49 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Tue, 23 Oct 2018 18:27:49 -0500 Subject: [PATCH 029/100] eth/downloader: SetBlocksIdle is not used (#17962) __ <(o )___ ( ._> / `---' --- eth/downloader/peer.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 428a60f8a1..60f86d0e14 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -229,13 +229,6 @@ func (p *peerConnection) SetHeadersIdle(delivered int) { p.setIdle(p.headerStarted, delivered, &p.headerThroughput, &p.headerIdle) } -// SetBlocksIdle sets the peer to idle, allowing it to execute new block retrieval -// requests. Its estimated block retrieval throughput is updated with that measured -// just now. -func (p *peerConnection) SetBlocksIdle(delivered int) { - p.setIdle(p.blockStarted, delivered, &p.blockThroughput, &p.blockIdle) -} - // SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval // requests. Its estimated body retrieval throughput is updated with that measured // just now. From 80d390776742a2a3cfc2f3041fd01ffe82f43d23 Mon Sep 17 00:00:00 2001 From: Johns Beharry Date: Thu, 25 Oct 2018 21:45:56 +0200 Subject: [PATCH 030/100] cmd/clef: replace password arg with prompt (#17897) * cmd/clef: replace password arg with prompt (#17829) Entering passwords on the command line is not secure as it is easy to recover from bash_history or the process table. 1. The clef command addpw was renamed to setpw to better describe the functionality 2. The argument was removed and replaced with an interactive prompt * cmd/clef: remove undeclared variable --- cmd/clef/main.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 6098b1ac21..519d63b3c1 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -157,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell Clef that the file is 'safe' to execute.`, } - addCredentialCommand = cli.Command{ - Action: utils.MigrateFlags(addCredential), - Name: "addpw", + setCredentialCommand = cli.Command{ + Action: utils.MigrateFlags(setCredential), + Name: "setpw", Usage: "Store a credential for a keystore file", - ArgsUsage: "
", + ArgsUsage: "
", Flags: []cli.Flag{ logLevelFlag, configdirFlag, signerSecretFlag, }, Description: ` -The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will + The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will remove any stored credential for that address (keyfile) `, } @@ -200,7 +200,7 @@ func init() { advancedMode, } app.Action = signer - app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand} + app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand} } func main() { @@ -293,14 +293,17 @@ func attestFile(ctx *cli.Context) error { return nil } -func addCredential(ctx *cli.Context) error { +func setCredential(ctx *cli.Context) error { if len(ctx.Args()) < 1 { - utils.Fatalf("This command requires at leaste one argument.") + utils.Fatalf("This command requires an address to be passed as an argument.") } if err := initialize(ctx); err != nil { return err } + address := ctx.Args().First() + password := getPassPhrase("Enter a passphrase to store with this address.", true) + stretchedKey, err := readMasterKey(ctx, nil) if err != nil { utils.Fatalf(err.Error()) @@ -311,13 +314,8 @@ func addCredential(ctx *cli.Context) error { // Initialize the encrypted storages pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) - key := ctx.Args().First() - value := "" - if len(ctx.Args()) > 1 { - value = ctx.Args().Get(1) - } - pwStorage.Put(key, value) - log.Info("Credential store updated", "key", key) + pwStorage.Put(address, password) + log.Info("Credential store updated", "key", address) return nil } From 8ed4739176f435d09dfa36d8b2e2a3c8c6f407dd Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 25 Oct 2018 17:26:31 -0500 Subject: [PATCH 031/100] p2p accounting (#17951) * p2p/protocols: introduced protocol accounting * p2p/protocols: added TestExchange simulation * p2p/protocols: add accounting simulation * p2p/protocols: remove unnecessary tests * p2p/protocols: comments for accounting simulation * p2p/protocols: addressed PR comments * p2p/protocols: finalized accounting implementation * p2p/protocols: removed unused code * p2p/protocols: addressed @nonsense PR comments --- p2p/protocols/accounting.go | 172 +++++++++++ p2p/protocols/accounting_simulation_test.go | 310 ++++++++++++++++++++ p2p/protocols/accounting_test.go | 223 ++++++++++++++ p2p/protocols/protocol.go | 30 ++ p2p/protocols/protocol_test.go | 202 +++++++++++++ 5 files changed, 937 insertions(+) create mode 100644 p2p/protocols/accounting.go create mode 100644 p2p/protocols/accounting_simulation_test.go create mode 100644 p2p/protocols/accounting_test.go diff --git a/p2p/protocols/accounting.go b/p2p/protocols/accounting.go new file mode 100644 index 0000000000..06a1a58454 --- /dev/null +++ b/p2p/protocols/accounting.go @@ -0,0 +1,172 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import "github.com/ethereum/go-ethereum/metrics" + +//define some metrics +var ( + //NOTE: these metrics just define the interfaces and are currently *NOT persisted* over sessions + //All metrics are cumulative + + //total amount of units credited + mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", nil) + //total amount of units debited + mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", nil) + //total amount of bytes credited + mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", nil) + //total amount of bytes debited + mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", nil) + //total amount of credited messages + mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", nil) + //total amount of debited messages + mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", nil) + //how many times local node had to drop remote peers + mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", nil) + //how many times local node overdrafted and dropped + mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", nil) +) + +//Prices defines how prices are being passed on to the accounting instance +type Prices interface { + //Return the Price for a message + Price(interface{}) *Price +} + +type Payer bool + +const ( + Sender = Payer(true) + Receiver = Payer(false) +) + +//Price represents the costs of a message +type Price struct { + Value uint64 // + PerByte bool //True if the price is per byte or for unit + Payer Payer +} + +//For gives back the price for a message +//A protocol provides the message price in absolute value +//This method then returns the correct signed amount, +//depending on who pays, which is identified by the `payer` argument: +//`Send` will pass a `Sender` payer, `Receive` will pass the `Receiver` argument. +//Thus: If Sending and sender pays, amount positive, otherwise negative +//If Receiving, and receiver pays, amount positive, otherwise negative +func (p *Price) For(payer Payer, size uint32) int64 { + price := p.Value + if p.PerByte { + price *= uint64(size) + } + if p.Payer == payer { + return 0 - int64(price) + } + return int64(price) +} + +//Balance is the actual accounting instance +//Balance defines the operations needed for accounting +//Implementations internally maintain the balance for every peer +type Balance interface { + //Adds amount to the local balance with remote node `peer`; + //positive amount = credit local node + //negative amount = debit local node + Add(amount int64, peer *Peer) error +} + +//Accounting implements the Hook interface +//It interfaces to the balances through the Balance interface, +//while interfacing with protocols and its prices through the Prices interface +type Accounting struct { + Balance //interface to accounting logic + Prices //interface to prices logic +} + +func NewAccounting(balance Balance, po Prices) *Accounting { + ah := &Accounting{ + Prices: po, + Balance: balance, + } + return ah +} + +//Implement Hook.Send +// Send takes a peer, a size and a msg and +// - calculates the cost for the local node sending a msg of size to peer using the Prices interface +// - credits/debits local node using balance interface +func (ah *Accounting) Send(peer *Peer, size uint32, msg interface{}) error { + //get the price for a message (through the protocol spec) + price := ah.Price(msg) + //this message doesn't need accounting + if price == nil { + return nil + } + //evaluate the price for sending messages + costToLocalNode := price.For(Sender, size) + //do the accounting + err := ah.Add(costToLocalNode, peer) + //record metrics: just increase counters for user-facing metrics + ah.doMetrics(costToLocalNode, size, err) + return err +} + +//Implement Hook.Receive +// Receive takes a peer, a size and a msg and +// - calculates the cost for the local node receiving a msg of size from peer using the Prices interface +// - credits/debits local node using balance interface +func (ah *Accounting) Receive(peer *Peer, size uint32, msg interface{}) error { + //get the price for a message (through the protocol spec) + price := ah.Price(msg) + //this message doesn't need accounting + if price == nil { + return nil + } + //evaluate the price for receiving messages + costToLocalNode := price.For(Receiver, size) + //do the accounting + err := ah.Add(costToLocalNode, peer) + //record metrics: just increase counters for user-facing metrics + ah.doMetrics(costToLocalNode, size, err) + return err +} + +//record some metrics +//this is not an error handling. `err` is returned by both `Send` and `Receive` +//`err` will only be non-nil if a limit has been violated (overdraft), in which case the peer has been dropped. +//if the limit has been violated and `err` is thus not nil: +// * if the price is positive, local node has been credited; thus `err` implicitly signals the REMOTE has been dropped +// * if the price is negative, local node has been debited, thus `err` implicitly signals LOCAL node "overdraft" +func (ah *Accounting) doMetrics(price int64, size uint32, err error) { + if price > 0 { + mBalanceCredit.Inc(price) + mBytesCredit.Inc(int64(size)) + mMsgCredit.Inc(1) + if err != nil { + //increase the number of times a remote node has been dropped due to "overdraft" + mPeerDrops.Inc(1) + } + } else { + mBalanceDebit.Inc(price) + mBytesDebit.Inc(int64(size)) + mMsgDebit.Inc(1) + if err != nil { + //increase the number of times the local node has done an "overdraft" in respect to other nodes + mSelfDrops.Inc(1) + } + } +} diff --git a/p2p/protocols/accounting_simulation_test.go b/p2p/protocols/accounting_simulation_test.go new file mode 100644 index 0000000000..65b737abed --- /dev/null +++ b/p2p/protocols/accounting_simulation_test.go @@ -0,0 +1,310 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "context" + "flag" + "fmt" + "math/rand" + "reflect" + "sync" + "testing" + "time" + + "github.com/mattn/go-colorable" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +const ( + content = "123456789" +) + +var ( + nodes = flag.Int("nodes", 30, "number of nodes to create (default 30)") + msgs = flag.Int("msgs", 100, "number of messages sent by node (default 100)") + loglevel = flag.Int("loglevel", 0, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") +) + +func init() { + flag.Parse() + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog)))) +} + +//TestAccountingSimulation runs a p2p/simulations simulation +//It creates a *nodes number of nodes, connects each one with each other, +//then sends out a random selection of messages up to *msgs amount of messages +//from the test protocol spec. +//The spec has some accounted messages defined through the Prices interface. +//The test does accounting for all the message exchanged, and then checks +//that every node has the same balance with a peer, but with opposite signs. +//Balance(AwithB) = 0 - Balance(BwithA) or Abs|Balance(AwithB)| == Abs|Balance(BwithA)| +func TestAccountingSimulation(t *testing.T) { + //setup the balances objects for every node + bal := newBalances(*nodes) + //define the node.Service for this test + services := adapters.Services{ + "accounting": func(ctx *adapters.ServiceContext) (node.Service, error) { + return bal.newNode(), nil + }, + } + //setup the simulation + adapter := adapters.NewSimAdapter(services) + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{DefaultService: "accounting"}) + defer net.Shutdown() + + // we send msgs messages per node, wait for all messages to arrive + bal.wg.Add(*nodes * *msgs) + trigger := make(chan enode.ID) + go func() { + // wait for all of them to arrive + bal.wg.Wait() + // then trigger a check + // the selected node for the trigger is irrelevant, + // we just want to trigger the end of the simulation + trigger <- net.Nodes[0].ID() + }() + + // create nodes and start them + for i := 0; i < *nodes; i++ { + conf := adapters.RandomNodeConfig() + bal.id2n[conf.ID] = i + if _, err := net.NewNodeWithConfig(conf); err != nil { + t.Fatal(err) + } + if err := net.Start(conf.ID); err != nil { + t.Fatal(err) + } + } + // fully connect nodes + for i, n := range net.Nodes { + for _, m := range net.Nodes[i+1:] { + if err := net.Connect(n.ID(), m.ID()); err != nil { + t.Fatal(err) + } + } + } + + // empty action + action := func(ctx context.Context) error { + return nil + } + // check always checks out + check := func(ctx context.Context, id enode.ID) (bool, error) { + return true, nil + } + + // run simulation + timeout := 30 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ + Action: action, + Trigger: trigger, + Expect: &simulations.Expectation{ + Nodes: []enode.ID{net.Nodes[0].ID()}, + Check: check, + }, + }) + + if result.Error != nil { + t.Fatal(result.Error) + } + + // check if balance matrix is symmetric + if err := bal.symmetric(); err != nil { + t.Fatal(err) + } +} + +// matrix is a matrix of nodes and its balances +// matrix is in fact a linear array of size n*n, +// so the balance for any node A with B is at index +// A*n + B, while the balance of node B with A is at +// B*n + A +// (n entries in the array will not be filled - +// the balance of a node with itself) +type matrix struct { + n int //number of nodes + m []int64 //array of balances +} + +// create a new matrix +func newMatrix(n int) *matrix { + return &matrix{ + n: n, + m: make([]int64, n*n), + } +} + +// called from the testBalance's Add accounting function: register balance change +func (m *matrix) add(i, j int, v int64) error { + // index for the balance of local node i with remote nodde j is + // i * number of nodes + remote node + mi := i*m.n + j + // register that balance + m.m[mi] += v + return nil +} + +// check that the balances are symmetric: +// balance of node i with node j is the same as j with i but with inverted signs +func (m *matrix) symmetric() error { + //iterate all nodes + for i := 0; i < m.n; i++ { + //iterate starting +1 + for j := i + 1; j < m.n; j++ { + log.Debug("bal", "1", i, "2", j, "i,j", m.m[i*m.n+j], "j,i", m.m[j*m.n+i]) + if m.m[i*m.n+j] != -m.m[j*m.n+i] { + return fmt.Errorf("value mismatch. m[%v, %v] = %v; m[%v, %v] = %v", i, j, m.m[i*m.n+j], j, i, m.m[j*m.n+i]) + } + } + } + return nil +} + +// all the balances +type balances struct { + i int + *matrix + id2n map[enode.ID]int + wg *sync.WaitGroup +} + +func newBalances(n int) *balances { + return &balances{ + matrix: newMatrix(n), + id2n: make(map[enode.ID]int), + wg: &sync.WaitGroup{}, + } +} + +// create a new testNode for every node created as part of the service +func (b *balances) newNode() *testNode { + defer func() { b.i++ }() + return &testNode{ + bal: b, + i: b.i, + peers: make([]*testPeer, b.n), //a node will be connected to n-1 peers + } +} + +type testNode struct { + bal *balances + i int + lock sync.Mutex + peers []*testPeer + peerCount int +} + +// do the accounting for the peer's test protocol +// testNode implements protocols.Balance +func (t *testNode) Add(a int64, p *Peer) error { + //get the index for the remote peer + remote := t.bal.id2n[p.ID()] + log.Debug("add", "local", t.i, "remote", remote, "amount", a) + return t.bal.add(t.i, remote, a) +} + +//run the p2p protocol +//for every node, represented by testNode, create a remote testPeer +func (t *testNode) run(p *p2p.Peer, rw p2p.MsgReadWriter) error { + spec := createTestSpec() + //create accounting hook + spec.Hook = NewAccounting(t, &dummyPrices{}) + + //create a peer for this node + tp := &testPeer{NewPeer(p, rw, spec), t.i, t.bal.id2n[p.ID()], t.bal.wg} + t.lock.Lock() + t.peers[t.bal.id2n[p.ID()]] = tp + t.peerCount++ + if t.peerCount == t.bal.n-1 { + //when all peer connections are established, start sending messages from this peer + go t.send() + } + t.lock.Unlock() + return tp.Run(tp.handle) +} + +// p2p message receive handler function +func (tp *testPeer) handle(ctx context.Context, msg interface{}) error { + tp.wg.Done() + log.Debug("receive", "from", tp.remote, "to", tp.local, "type", reflect.TypeOf(msg), "msg", msg) + return nil +} + +type testPeer struct { + *Peer + local, remote int + wg *sync.WaitGroup +} + +func (t *testNode) send() { + log.Debug("start sending") + for i := 0; i < *msgs; i++ { + //determine randomly to which peer to send + whom := rand.Intn(t.bal.n - 1) + if whom >= t.i { + whom++ + } + t.lock.Lock() + p := t.peers[whom] + t.lock.Unlock() + + //determine a random message from the spec's messages to be sent + which := rand.Intn(len(p.spec.Messages)) + msg := p.spec.Messages[which] + switch msg.(type) { + case *perBytesMsgReceiverPays: + msg = &perBytesMsgReceiverPays{Content: content[:rand.Intn(len(content))]} + case *perBytesMsgSenderPays: + msg = &perBytesMsgSenderPays{Content: content[:rand.Intn(len(content))]} + } + log.Debug("send", "from", t.i, "to", whom, "type", reflect.TypeOf(msg), "msg", msg) + p.Send(context.TODO(), msg) + } +} + +// define the protocol +func (t *testNode) Protocols() []p2p.Protocol { + return []p2p.Protocol{{ + Length: 100, + Run: t.run, + }} +} + +func (t *testNode) APIs() []rpc.API { + return nil +} + +func (t *testNode) Start(server *p2p.Server) error { + return nil +} + +func (t *testNode) Stop() error { + return nil +} diff --git a/p2p/protocols/accounting_test.go b/p2p/protocols/accounting_test.go new file mode 100644 index 0000000000..3810ae2c9b --- /dev/null +++ b/p2p/protocols/accounting_test.go @@ -0,0 +1,223 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "testing" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" +) + +//dummy Balance implementation +type dummyBalance struct { + amount int64 + peer *Peer +} + +//dummy Prices implementation +type dummyPrices struct{} + +//a dummy message which needs size based accounting +//sender pays +type perBytesMsgSenderPays struct { + Content string +} + +//a dummy message which needs size based accounting +//receiver pays +type perBytesMsgReceiverPays struct { + Content string +} + +//a dummy message which is paid for per unit +//sender pays +type perUnitMsgSenderPays struct{} + +//receiver pays +type perUnitMsgReceiverPays struct{} + +//a dummy message which has zero as its price +type zeroPriceMsg struct{} + +//a dummy message which has no accounting +type nilPriceMsg struct{} + +//return the price for the defined messages +func (d *dummyPrices) Price(msg interface{}) *Price { + switch msg.(type) { + //size based message cost, receiver pays + case *perBytesMsgReceiverPays: + return &Price{ + PerByte: true, + Value: uint64(100), + Payer: Receiver, + } + //size based message cost, sender pays + case *perBytesMsgSenderPays: + return &Price{ + PerByte: true, + Value: uint64(100), + Payer: Sender, + } + //unitary cost, receiver pays + case *perUnitMsgReceiverPays: + return &Price{ + PerByte: false, + Value: uint64(99), + Payer: Receiver, + } + //unitary cost, sender pays + case *perUnitMsgSenderPays: + return &Price{ + PerByte: false, + Value: uint64(99), + Payer: Sender, + } + case *zeroPriceMsg: + return &Price{ + PerByte: false, + Value: uint64(0), + Payer: Sender, + } + case *nilPriceMsg: + return nil + } + return nil +} + +//dummy accounting implementation, only stores values for later check +func (d *dummyBalance) Add(amount int64, peer *Peer) error { + d.amount = amount + d.peer = peer + return nil +} + +type testCase struct { + msg interface{} + size uint32 + sendResult int64 + recvResult int64 +} + +//lowest level unit test +func TestBalance(t *testing.T) { + //create instances + balance := &dummyBalance{} + prices := &dummyPrices{} + //create the spec + spec := createTestSpec() + //create the accounting hook for the spec + acc := NewAccounting(balance, prices) + //create a peer + id := adapters.RandomNodeConfig().ID + p := p2p.NewPeer(id, "testPeer", nil) + peer := NewPeer(p, &dummyRW{}, spec) + //price depends on size, receiver pays + msg := &perBytesMsgReceiverPays{Content: "testBalance"} + size, _ := rlp.EncodeToBytes(msg) + + testCases := []testCase{ + { + msg, + uint32(len(size)), + int64(len(size) * 100), + int64(len(size) * -100), + }, + { + &perBytesMsgSenderPays{Content: "testBalance"}, + uint32(len(size)), + int64(len(size) * -100), + int64(len(size) * 100), + }, + { + &perUnitMsgSenderPays{}, + 0, + int64(-99), + int64(99), + }, + { + &perUnitMsgReceiverPays{}, + 0, + int64(99), + int64(-99), + }, + { + &zeroPriceMsg{}, + 0, + int64(0), + int64(0), + }, + { + &nilPriceMsg{}, + 0, + int64(0), + int64(0), + }, + } + checkAccountingTestCases(t, testCases, acc, peer, balance, true) + checkAccountingTestCases(t, testCases, acc, peer, balance, false) +} + +func checkAccountingTestCases(t *testing.T, cases []testCase, acc *Accounting, peer *Peer, balance *dummyBalance, send bool) { + for _, c := range cases { + var err error + var expectedResult int64 + //reset balance before every check + balance.amount = 0 + if send { + err = acc.Send(peer, c.size, c.msg) + expectedResult = c.sendResult + } else { + err = acc.Receive(peer, c.size, c.msg) + expectedResult = c.recvResult + } + + checkResults(t, err, balance, peer, expectedResult) + } +} + +func checkResults(t *testing.T, err error, balance *dummyBalance, peer *Peer, result int64) { + if err != nil { + t.Fatal(err) + } + if balance.peer != peer { + t.Fatalf("expected Add to be called with peer %v, got %v", peer, balance.peer) + } + if balance.amount != result { + t.Fatalf("Expected balance to be %d but is %d", result, balance.amount) + } +} + +//create a test spec +func createTestSpec() *Spec { + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + &perBytesMsgReceiverPays{}, + &perBytesMsgSenderPays{}, + &perUnitMsgReceiverPays{}, + &perUnitMsgSenderPays{}, + &zeroPriceMsg{}, + &nilPriceMsg{}, + }, + } + return spec +} diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index 615f74b569..7dddd852fa 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -122,6 +122,16 @@ type WrappedMsg struct { Payload []byte } +//For accounting, the design is to allow the Spec to describe which and how its messages are priced +//To access this functionality, we provide a Hook interface which will call accounting methods +//NOTE: there could be more such (horizontal) hooks in the future +type Hook interface { + //A hook for sending messages + Send(peer *Peer, size uint32, msg interface{}) error + //A hook for receiving messages + Receive(peer *Peer, size uint32, msg interface{}) error +} + // Spec is a protocol specification including its name and version as well as // the types of messages which are exchanged type Spec struct { @@ -141,6 +151,9 @@ type Spec struct { // each message must have a single unique data type Messages []interface{} + //hook for accounting (could be extended to multiple hooks in the future) + Hook Hook + initOnce sync.Once codes map[reflect.Type]uint64 types map[uint64]reflect.Type @@ -274,6 +287,15 @@ func (p *Peer) Send(ctx context.Context, msg interface{}) error { Payload: r, } + //if the accounting hook is set, call it + if p.spec.Hook != nil { + err := p.spec.Hook.Send(p, wmsg.Size, msg) + if err != nil { + p.Drop(err) + return err + } + } + code, found := p.spec.GetCode(msg) if !found { return errorf(ErrInvalidMsgType, "%v", code) @@ -336,6 +358,14 @@ func (p *Peer) handleIncoming(handle func(ctx context.Context, msg interface{}) return errorf(ErrDecode, "<= %v: %v", msg, err) } + //if the accounting hook is set, call it + if p.spec.Hook != nil { + err := p.spec.Hook.Receive(p, wmsg.Size, val) + if err != nil { + return err + } + } + // call the registered handler callbacks // a registered callback take the decoded message as argument as an interface // which the handler is supposed to cast to the appropriate type diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index 4755db3e62..2874af48d2 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -17,12 +17,15 @@ package protocols import ( + "bytes" "context" "errors" "fmt" "testing" "time" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" @@ -185,6 +188,169 @@ func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) { } } +type dummyHook struct { + peer *Peer + size uint32 + msg interface{} + send bool + err error + waitC chan struct{} +} + +type dummyMsg struct { + Content string +} + +func (d *dummyHook) Send(peer *Peer, size uint32, msg interface{}) error { + d.peer = peer + d.size = size + d.msg = msg + d.send = true + return d.err +} + +func (d *dummyHook) Receive(peer *Peer, size uint32, msg interface{}) error { + d.peer = peer + d.size = size + d.msg = msg + d.send = false + d.waitC <- struct{}{} + return d.err +} + +func TestProtocolHook(t *testing.T) { + testHook := &dummyHook{ + waitC: make(chan struct{}, 1), + } + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + dummyMsg{}, + }, + Hook: testHook, + } + + runFunc := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := NewPeer(p, rw, spec) + ctx := context.TODO() + err := peer.Send(ctx, &dummyMsg{ + Content: "handshake"}) + + if err != nil { + t.Fatal(err) + } + + handle := func(ctx context.Context, msg interface{}) error { + return nil + } + + return peer.Run(handle) + } + + conf := adapters.RandomNodeConfig() + tester := p2ptest.NewProtocolTester(t, conf.ID, 2, runFunc) + err := tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { + Code: 0, + Msg: &dummyMsg{Content: "handshake"}, + Peer: tester.Nodes[0].ID(), + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if testHook.msg == nil || testHook.msg.(*dummyMsg).Content != "handshake" { + t.Fatal("Expected msg to be set, but it is not") + } + if !testHook.send { + t.Fatal("Expected a send message, but it is not") + } + if testHook.peer == nil || testHook.peer.ID() != tester.Nodes[0].ID() { + t.Fatal("Expected peer ID to be set correctly, but it is not") + } + if testHook.size != 11 { //11 is the length of the encoded message + t.Fatalf("Expected size to be %d, but it is %d ", 1, testHook.size) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &dummyMsg{Content: "response"}, + Peer: tester.Nodes[1].ID(), + }, + }, + }) + + <-testHook.waitC + + if err != nil { + t.Fatal(err) + } + if testHook.msg == nil || testHook.msg.(*dummyMsg).Content != "response" { + t.Fatal("Expected msg to be set, but it is not") + } + if testHook.send { + t.Fatal("Expected a send message, but it is not") + } + if testHook.peer == nil || testHook.peer.ID() != tester.Nodes[1].ID() { + t.Fatal("Expected peer ID to be set correctly, but it is not") + } + if testHook.size != 10 { //11 is the length of the encoded message + t.Fatalf("Expected size to be %d, but it is %d ", 1, testHook.size) + } + + testHook.err = fmt.Errorf("dummy error") + err = tester.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &dummyMsg{Content: "response"}, + Peer: tester.Nodes[1].ID(), + }, + }, + }) + + <-testHook.waitC + + time.Sleep(100 * time.Millisecond) + err = tester.TestDisconnected(&p2ptest.Disconnect{tester.Nodes[1].ID(), testHook.err}) + if err != nil { + t.Fatalf("Expected a specific disconnect error, but got different one: %v", err) + } + +} + +//We need to test that if the hook is not defined, then message infrastructure +//(send,receive) still works +func TestNoHook(t *testing.T) { + //create a test spec + spec := createTestSpec() + //a random node + id := adapters.RandomNodeConfig().ID + //a peer + p := p2p.NewPeer(id, "testPeer", nil) + rw := &dummyRW{} + peer := NewPeer(p, rw, spec) + ctx := context.TODO() + msg := &perBytesMsgSenderPays{Content: "testBalance"} + //send a message + err := peer.Send(ctx, msg) + if err != nil { + t.Fatal(err) + } + //simulate receiving a message + rw.msg = msg + peer.handleIncoming(func(ctx context.Context, msg interface{}) error { + return nil + }) + //all should just work and not result in any error +} + func TestProtoHandshakeVersionMismatch(t *testing.T) { runProtoHandshake(t, &protoHandshake{41, "420"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 41 (!= 42)").Error())) } @@ -386,3 +552,39 @@ func XTestMultiplePeersDropOther(t *testing.T) { fmt.Errorf("subprotocol error"), ) } + +//dummy implementation of a MsgReadWriter +//this allows for quick and easy unit tests without +//having to build up the complete protocol +type dummyRW struct { + msg interface{} + size uint32 + code uint64 +} + +func (d *dummyRW) WriteMsg(msg p2p.Msg) error { + return nil +} + +func (d *dummyRW) ReadMsg() (p2p.Msg, error) { + enc := bytes.NewReader(d.getDummyMsg()) + return p2p.Msg{ + Code: d.code, + Size: d.size, + Payload: enc, + ReceivedAt: time.Now(), + }, nil +} + +func (d *dummyRW) getDummyMsg() []byte { + r, _ := rlp.EncodeToBytes(d.msg) + var b bytes.Buffer + wmsg := WrappedMsg{ + Context: b.Bytes(), + Size: uint32(len(r)), + Payload: r, + } + rr, _ := rlp.EncodeToBytes(wmsg) + d.size = uint32(len(rr)) + return rr +} From 1b6fd032e3def058133161a45a000dae4d0db729 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 26 Oct 2018 08:52:41 +0200 Subject: [PATCH 032/100] core/vm: check empty in extcodehash --- core/vm/instructions.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index b7c3ca5323..6696c6e3d1 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -544,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract, // this account should be regarded as a non-existent account and zero should be returned. func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { slot := stack.peek() - slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes()) + address := common.BigToAddress(slot) + if interpreter.evm.StateDB.Empty(address) { + slot.SetUint64(0) + } else { + slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes()) + } return nil, nil } From 54f650a3be2ccf7cd44e9929e3e132ef93f101ad Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Sat, 27 Oct 2018 16:18:42 +0200 Subject: [PATCH 033/100] swarm: clean up unused private types and functions (#17989) * swarm: clean up unused private types and functions Those that were identified by code inspection tool. * swarm/storage: move/add Proximity GoDoc from deleted private function The mentioned proximity() private function was deleted in: 1ca8fc1e6fa0ab4ab1aaca06d6fb32e173cd5f2f --- swarm/api/http/server.go | 7 ------ swarm/network/protocol_test.go | 4 --- swarm/pot/address.go | 40 ------------------------------ swarm/pss/client/client_test.go | 4 --- swarm/pss/pss_test.go | 5 ---- swarm/pss/types.go | 4 --- swarm/storage/common_test.go | 11 -------- swarm/storage/feed/handler_test.go | 13 ---------- swarm/storage/ldbstore.go | 20 --------------- swarm/storage/types.go | 13 ++++++++++ 10 files changed, 13 insertions(+), 108 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index b4294b0581..de1eb70dcf 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -41,16 +41,9 @@ import ( "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/rs/cors" ) -type resourceResponse struct { - Manifest storage.Address `json:"manifest"` - Resource string `json:"resource"` - Update storage.Address `json:"update"` -} - var ( postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil) postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil) diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 4b83c7a278..cdf370f359 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -50,10 +50,6 @@ type testStore struct { values map[string][]byte } -func newTestStore() *testStore { - return &testStore{values: make(map[string][]byte)} -} - func (t *testStore) Load(key string) ([]byte, error) { t.Lock() defer t.Unlock() diff --git a/swarm/pot/address.go b/swarm/pot/address.go index 3974ebcaac..728dac14ee 100644 --- a/swarm/pot/address.go +++ b/swarm/pot/address.go @@ -79,46 +79,6 @@ func (a Address) Bytes() []byte { return a[:] } -/* -Proximity(x, y) returns the proximity order of the MSB distance between x and y - -The distance metric MSB(x, y) of two equal length byte sequences x an y is the -value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. -the binary cast is big endian: most significant bit first (=MSB). - -Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. -It is defined as the reverse rank of the integer part of the base 2 -logarithm of the distance. -It is calculated by counting the number of common leading zeros in the (MSB) -binary representation of the x^y. - -(0 farthest, 255 closest, 256 self) -*/ -func proximity(one, other Address) (ret int, eq bool) { - return posProximity(one, other, 0) -} - -// posProximity(a, b, pos) returns proximity order of b wrt a (symmetric) pretending -// the first pos bits match, checking only bits index >= pos -func posProximity(one, other Address, pos int) (ret int, eq bool) { - for i := pos / 8; i < len(one); i++ { - if one[i] == other[i] { - continue - } - oxo := one[i] ^ other[i] - start := 0 - if i == pos/8 { - start = pos % 8 - } - for j := start; j < 8; j++ { - if (oxo>>uint8(7-j))&0x01 != 0 { - return i*8 + j, false - } - } - } - return len(one) * 8, true -} - // ProxCmp compares the distances a->target and b->target. // Returns -1 if a is closer to target, 1 if b is closer to target // and 0 if they are equal. diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 48edc6ccef..cfef3c7947 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -288,10 +288,6 @@ type testStore struct { values map[string][]byte } -func newTestStore() *testStore { - return &testStore{values: make(map[string][]byte)} -} - func (t *testStore) Load(key string) ([]byte, error) { return nil, nil } diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 574714114b..66a90be620 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -976,11 +976,6 @@ func TestNetwork10000(t *testing.T) { } func testNetwork(t *testing.T) { - type msgnotifyC struct { - id enode.ID - msgIdx int - } - paramstring := strings.Split(t.Name(), "/") nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0) msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0) diff --git a/swarm/pss/types.go b/swarm/pss/types.go index 1e33ecdcac..56c2c51dc0 100644 --- a/swarm/pss/types.go +++ b/swarm/pss/types.go @@ -169,10 +169,6 @@ type stateStore struct { values map[string][]byte } -func newStateStore() *stateStore { - return &stateStore{values: make(map[string][]byte)} -} - func (store *stateStore) Load(key string) ([]byte, error) { return nil, nil } diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 33133edd74..600be164a9 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -88,17 +88,6 @@ func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) return mput(store, n, GenerateRandomChunk) } -func mputChunks(store ChunkStore, chunks ...Chunk) error { - i := 0 - f := func(n int64) Chunk { - chunk := chunks[i] - i++ - return chunk - } - _, err := mput(store, len(chunks), f) - return err -} - func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error) { // put to localstore and wait for stored channel // does not check delivery error state diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go index cf95bc1f57..fb2ef3a6ba 100644 --- a/swarm/storage/feed/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -27,7 +27,6 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" @@ -506,15 +505,3 @@ func newCharlieSigner() *GenericSigner { privKey, _ := crypto.HexToECDSA("facadefacadefacadefacadefacadefacadefacadefacadefacadefacadefaca") return NewGenericSigner(privKey) } - -func getUpdateDirect(rh *Handler, addr storage.Address) ([]byte, error) { - chunk, err := rh.chunkStore.Get(context.TODO(), addr) - if err != nil { - return nil, err - } - var r Request - if err := r.fromChunk(addr, chunk.Data()); err != nil { - return nil, err - } - return r.data, nil -} diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 49508911f9..9feb687413 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -39,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/opt" ) const ( @@ -72,13 +71,6 @@ var ( ErrDBClosed = errors.New("LDBStore closed") ) -type gcItem struct { - idx *dpaDBIndex - value uint64 - idxKey []byte - po uint8 -} - type LDBStoreParams struct { *StoreParams Path string @@ -961,15 +953,3 @@ func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Add } return it.Error() } - -func databaseExists(path string) bool { - o := &opt.Options{ - ErrorIfMissing: true, - } - tdb, err := leveldb.OpenFile(path, o) - if err != nil { - return false - } - defer tdb.Close() - return true -} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 8c70f45848..ada86831fc 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -80,6 +80,19 @@ func (a Address) bits(i, j uint) uint { return res } +// Proximity(x, y) returns the proximity order of the MSB distance between x and y +// +// The distance metric MSB(x, y) of two equal length byte sequences x an y is the +// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. +// the binary cast is big endian: most significant bit first (=MSB). +// +// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. +// It is defined as the reverse rank of the integer part of the base 2 +// logarithm of the distance. +// It is calculated by counting the number of common leading zeros in the (MSB) +// binary representation of the x^y. +// +// (0 farthest, 255 closest, 256 self) func Proximity(one, other []byte) (ret int) { b := (MaxPO-1)/8 + 1 if b > len(one) { From 3e1cfbae934f72c94c00e515ca872997530eede8 Mon Sep 17 00:00:00 2001 From: Roc Yu Date: Mon, 29 Oct 2018 17:00:00 +0800 Subject: [PATCH 034/100] cmd/swarm/swarm-smoke: fix issue that loop variable capture in func (#17992) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index c7a1475d6c..1371d66548 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -13,16 +13,13 @@ import ( "sync" "time" - "github.com/pborman/uuid" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" - colorable "github.com/mattn/go-colorable" - + "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -190,7 +187,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { wg.Add(1) ruid := uuid.New()[:8] - go func(endpoint string, ruid string) { + go func(hex string, endpoint string, ruid string) { for { err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) if err != nil { @@ -200,7 +197,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(endpoint, ruid) + }(hex, endpoint, ruid) } } @@ -268,7 +265,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { wg.Add(1) ruid := uuid.New()[:8] - go func(endpoint string, ruid string) { + go func(url string, endpoint string, ruid string) { for { err := fetch(url, endpoint, fileHash, ruid) if err != nil { @@ -278,7 +275,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(endpoint, ruid) + }(url, endpoint, ruid) } } From f08f596a378e1bc0cfcb476868767b1f1d602c17 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 31 Oct 2018 23:36:33 +0100 Subject: [PATCH 035/100] all: updated code owners file (#17987) --- .github/CODEOWNERS | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5a717da009..f7f6916c54 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,23 +9,26 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi +p2p/simulations @lmars +p2p/protocols @zelig +swarm/api/http @justelad swarm/bmt @zelig swarm/dev @lmars swarm/fuse @jmozah @holisticode swarm/grafana_dashboards @nonsense swarm/metrics @nonsense @holisticode swarm/multihash @nolash -swarm/network/bitvector @zelig @janos @gbalint -swarm/network/priorityqueue @zelig @janos @gbalint -swarm/network/simulations @zelig -swarm/network/stream @janos @zelig @gbalint @holisticode @justelad +swarm/network/bitvector @zelig @janos +swarm/network/priorityqueue @zelig @janos +swarm/network/simulations @zelig @janos +swarm/network/stream @janos @zelig @holisticode @justelad swarm/network/stream/intervals @janos swarm/network/stream/testing @zelig swarm/pot @zelig swarm/pss @nolash @zelig @nonsense swarm/services @zelig swarm/state @justelad -swarm/storage/encryption @gbalint @zelig @nagydani +swarm/storage/encryption @zelig @nagydani swarm/storage/mock @janos swarm/storage/feed @nolash @jpeletier swarm/testutil @lmars From 126dfde6c9de1f8122bb3991453d2429b47542f8 Mon Sep 17 00:00:00 2001 From: Elad Date: Sun, 4 Nov 2018 07:59:58 +0100 Subject: [PATCH 036/100] cmd/swarm: auto resolve default path according to env flag (#17960) --- cmd/swarm/config.go | 1 + cmd/swarm/upload.go | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 16001010df..3eea3057b3 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -80,6 +80,7 @@ const ( SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" + SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" GETH_ENV_DATADIR = "GETH_DATADIR" ) diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 0dbe896e23..992f2d6e97 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -26,8 +26,10 @@ import ( "os/user" "path" "path/filepath" + "strconv" "strings" + "github.com/ethereum/go-ethereum/log" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/cmd/utils" @@ -47,17 +49,24 @@ var upCommand = cli.Command{ func upload(ctx *cli.Context) { args := ctx.Args() var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name) - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) - fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) - mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) - client = swarm.NewClient(bzzapi) - toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) - file string + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name) + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) + fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) + mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) + client = swarm.NewClient(bzzapi) + toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) + autoDefaultPath = false + file string ) - + if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" { + b, err := strconv.ParseBool(autoDefaultPathString) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err) + } + autoDefaultPath = b + } if len(args) != 1 { if fromStdin { tmp, err := ioutil.TempFile("", "swarm-stdin") @@ -106,6 +115,15 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } + if autoDefaultPath && defaultPath == "" { + defaultEntryCandidate := path.Join(file, "index.html") + log.Debug("trying to find default path", "path", defaultEntryCandidate) + defaultEntryStat, err := os.Stat(defaultEntryCandidate) + if err == nil && !defaultEntryStat.IsDir() { + log.Debug("setting auto detected default path", "path", defaultEntryCandidate) + defaultPath = defaultEntryCandidate + } + } if defaultPath != "" { // construct absolute default path absDefaultPath, _ := filepath.Abs(defaultPath) From baee850471603d7fbf37eed0074c9de01df92ee7 Mon Sep 17 00:00:00 2001 From: KimMachineGun Date: Tue, 6 Nov 2018 17:54:19 +0900 Subject: [PATCH 037/100] swarm: modify context key (#17925) * swarm: modify context key * gofmt sctx.go --- swarm/api/http/sctx.go | 14 +++++--------- swarm/sctx/sctx.go | 12 +++++------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/swarm/api/http/sctx.go b/swarm/api/http/sctx.go index 431e117354..b8dafab0b7 100644 --- a/swarm/api/http/sctx.go +++ b/swarm/api/http/sctx.go @@ -7,14 +7,10 @@ import ( "github.com/ethereum/go-ethereum/swarm/sctx" ) -type contextKey int - -const ( - uriKey contextKey = iota -) +type uriKey struct{} func GetRUID(ctx context.Context) string { - v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string) + v, ok := ctx.Value(sctx.HTTPRequestIDKey{}).(string) if ok { return v } @@ -22,11 +18,11 @@ func GetRUID(ctx context.Context) string { } func SetRUID(ctx context.Context, ruid string) context.Context { - return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid) + return context.WithValue(ctx, sctx.HTTPRequestIDKey{}, ruid) } func GetURI(ctx context.Context) *api.URI { - v, ok := ctx.Value(uriKey).(*api.URI) + v, ok := ctx.Value(uriKey{}).(*api.URI) if ok { return v } @@ -34,5 +30,5 @@ func GetURI(ctx context.Context) *api.URI { } func SetURI(ctx context.Context, uri *api.URI) context.Context { - return context.WithValue(ctx, uriKey, uri) + return context.WithValue(ctx, uriKey{}, uri) } diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index bed2b11458..fb7d35b000 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -2,19 +2,17 @@ package sctx import "context" -type ContextKey int - -const ( - HTTPRequestIDKey ContextKey = iota - requestHostKey +type ( + HTTPRequestIDKey struct{} + requestHostKey struct{} ) func SetHost(ctx context.Context, domain string) context.Context { - return context.WithValue(ctx, requestHostKey, domain) + return context.WithValue(ctx, requestHostKey{}, domain) } func GetHost(ctx context.Context) string { - v, ok := ctx.Value(requestHostKey).(string) + v, ok := ctx.Value(requestHostKey{}).(string) if ok { return v } From 53eb4e0b0fffdc105fbe9f5eed671b96de6e2ba1 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 6 Nov 2018 12:34:34 +0100 Subject: [PATCH 038/100] swarm/api: unexport Respond methods (#18037) --- swarm/api/http/middleware.go | 2 +- swarm/api/http/response.go | 25 ++++++------ swarm/api/http/server.go | 76 ++++++++++++++++++------------------ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 3b2dcc7d59..ccc040c54d 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -50,7 +50,7 @@ func ParseURI(h http.Handler) http.Handler { uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) if err != nil { w.WriteHeader(http.StatusBadRequest) - RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) + respondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) return } if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") { diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index c9fb9d2855..d4e81d7f67 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -53,23 +53,23 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) msg := "" if list.Entries == nil { - RespondError(w, r, "Could not resolve", http.StatusInternalServerError) + respondError(w, r, "Could not resolve", http.StatusInternalServerError) return } requestUri := strings.TrimPrefix(r.RequestURI, "/") uri, err := api.Parse(requestUri) if err != nil { - RespondError(w, r, "Bad Request", http.StatusBadRequest) + respondError(w, r, "Bad Request", http.StatusBadRequest) } uri.Scheme = "bzz-list" msg += fmt.Sprintf("Disambiguation:
Your request may refer to multiple choices.
Click here if your browser does not redirect you within 5 seconds.
", "/"+uri.String()) - RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices) + respondTemplate(w, r, "error", msg, http.StatusMultipleChoices) } -func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { - log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) +func respondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { + log.Debug("respondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) respond(w, r, &ResponseParams{ Code: code, Msg: template.HTML(msg), @@ -78,13 +78,12 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s }) } -func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) { - log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) - RespondTemplate(w, r, "error", msg, code) +func respondError(w http.ResponseWriter, r *http.Request, msg string, code int) { + log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) + respondTemplate(w, r, "error", msg, code) } func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { - w.WriteHeader(params.Code) if params.Code >= 400 { @@ -96,7 +95,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { // this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8" if strings.Contains(acceptHeader, "application/json") { if err := respondJSON(w, r, params); err != nil { - RespondError(w, r, "Internal server error", http.StatusInternalServerError) + respondError(w, r, "Internal server error", http.StatusInternalServerError) } } else if strings.Contains(acceptHeader, "text/html") { respondHTML(w, r, params) @@ -107,7 +106,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) { htmlCounter.Inc(1) - log.Debug("respondHTML", "ruid", GetRUID(r.Context())) + log.Info("respondHTML", "ruid", GetRUID(r.Context()), "code", params.Code) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -116,14 +115,14 @@ func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { jsonCounter.Inc(1) - log.Debug("respondJSON", "ruid", GetRUID(r.Context())) + log.Info("respondJSON", "ruid", GetRUID(r.Context()), "code", params.Code) w.Header().Set("Content-Type", "application/json") return json.NewEncoder(w).Encode(params) } func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { plaintextCounter.Inc(1) - log.Debug("respondPlaintext", "ruid", GetRUID(r.Context())) + log.Info("respondPlaintext", "ruid", GetRUID(r.Context()), "code", params.Code) w.Header().Set("Content-Type", "text/plain") strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n" strToWrite += "Message: " + string(params.Msg) + "\n" diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index de1eb70dcf..e9005104ee 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -184,10 +184,10 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { if err != nil { if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String())) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) return } defer reader.Close() @@ -211,7 +211,7 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { switch r.RequestURI { case "/": - RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) + respondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) return case "/robots.txt": w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) @@ -220,7 +220,7 @@ func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(faviconBytes) default: - RespondError(w, r, "Not Found", http.StatusNotFound) + respondError(w, r, "Not Found", http.StatusNotFound) } } @@ -240,26 +240,26 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { if uri.Path != "" { postRawFail.Inc(1) - RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) + respondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) return } if uri.Addr != "" && uri.Addr != "encrypt" { postRawFail.Inc(1) - RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) + respondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { postRawFail.Inc(1) - RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) + respondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -283,7 +283,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusBadRequest) + respondError(w, r, err.Error(), http.StatusBadRequest) return } @@ -298,7 +298,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { addr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) return } log.Debug("resolved key", "ruid", ruid, "key", addr) @@ -306,7 +306,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { addr, err = s.api.NewManifest(r.Context(), toEncrypt) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } log.Debug("new manifest", "ruid", ruid, "key", addr) @@ -317,7 +317,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { case "application/x-tar": _, err := s.handleTarUpload(r, mw) if err != nil { - RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) return err } return nil @@ -330,7 +330,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { }) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } @@ -439,7 +439,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) { newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path) if err != nil { deleteFail.Inc(1) - RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) return } @@ -460,7 +460,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // Creation and update must send feed.updateRequestJSON JSON structure body, err := ioutil.ReadAll(r.Body) if err != nil { - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -471,7 +471,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { httpStatus = http.StatusNotFound } - RespondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus) + respondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus) return } @@ -480,7 +480,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters - RespondError(w, r, err.Error(), http.StatusBadRequest) + respondError(w, r, err.Error(), http.StatusBadRequest) return } @@ -489,12 +489,12 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // to update this feed // Check this early, to avoid creating a feed and then not being able to set its first update. if err = updateRequest.Verify(); err != nil { - RespondError(w, r, err.Error(), http.StatusForbidden) + respondError(w, r, err.Error(), http.StatusForbidden) return } _, err = s.api.FeedsUpdate(r.Context(), &updateRequest) if err != nil { - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } } @@ -505,7 +505,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // feed identification used to retrieve feed updates later m, err := s.api.NewFeedManifest(r.Context(), &updateRequest.Feed) if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError) return } // the key to the manifest will be passed back to the client @@ -513,7 +513,7 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { // the manifest key can be set as content in the resolver of the ENS name outdata, err := json.Marshal(m) if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) return } fmt.Fprint(w, string(outdata)) @@ -550,7 +550,7 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { httpStatus = http.StatusNotFound } - RespondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus) + respondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus) return } @@ -559,12 +559,12 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { unsignedUpdateRequest, err := s.api.FeedsNewRequest(r.Context(), fd) if err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound) return } rawResponse, err := unsignedUpdateRequest.MarshalJSON() if err != nil { - RespondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError) return } w.Header().Add("Content-type", "application/json") @@ -575,7 +575,7 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { lookupParams := &feed.Query{Feed: *fd} if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version - RespondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest) + respondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest) return } @@ -584,7 +584,7 @@ func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { // any error from the switch statement will end up here if err != nil { code, err2 := s.translateFeedError(w, r, "feed lookup fail", err) - RespondError(w, r, err2.Error(), code) + respondError(w, r, err2.Error(), code) return } @@ -630,7 +630,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { addr, err := s.api.ResolveURI(r.Context(), uri, pass) if err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. @@ -654,7 +654,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { reader, isEncrypted := s.api.Retrieve(r.Context(), addr) if _, err := reader.Size(r.Context(), nil); err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) return } @@ -694,7 +694,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { addr, err := s.api.Resolve(r.Context(), uri.Addr) if err != nil { getListFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr) @@ -704,10 +704,10 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { getListFail.Inc(1) if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String())) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -755,7 +755,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFileFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } } else { @@ -779,17 +779,17 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { if err != nil { if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } switch status { case http.StatusNotFound: getFileNotFound.Inc(1) - RespondError(w, r, err.Error(), http.StatusNotFound) + respondError(w, r, err.Error(), http.StatusNotFound) default: getFileFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) } return } @@ -802,10 +802,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { getFileFail.Inc(1) if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -818,7 +818,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(r.Context(), nil); err != nil { getFileNotFound.Inc(1) - RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) return } From 79c7a69ac8066cc28ceee2ebaab3d0221a8adf57 Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 6 Nov 2018 18:04:18 -0500 Subject: [PATCH 039/100] swarm: Better syncing and retrieval option definition (#17986) * swarm: Better syncing and retrieval option definition * swarm/network/stream: better comments * swarm/network/stream: addressed PR comments --- swarm/network/stream/delivery_test.go | 97 +++++++++++++------ swarm/network/stream/intervals_test.go | 2 + swarm/network/stream/lightnode_test.go | 12 ++- .../network/stream/snapshot_retrieval_test.go | 5 +- swarm/network/stream/snapshot_sync_test.go | 8 +- swarm/network/stream/stream.go | 57 ++++++++--- swarm/network/stream/streamer_test.go | 2 + swarm/network/stream/syncer_test.go | 2 + swarm/swarm.go | 20 ++-- 9 files changed, 147 insertions(+), 58 deletions(-) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 949645558b..29b4f2f69a 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -38,8 +38,13 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) +//Tests initializing a retrieve request func TestStreamerRetrieveRequest(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + regOpts := &RegistryOptions{ + Retrieval: RetrievalClientOnly, + Syncing: SyncingDisabled, + } + tester, streamer, _, teardown, err := newStreamerTester(t, regOpts) defer teardown() if err != nil { t.Fatal(err) @@ -55,10 +60,21 @@ func TestStreamerRetrieveRequest(t *testing.T) { ) streamer.delivery.RequestFromPeers(ctx, req) + stream := NewStream(swarmChunkServerStreamName, "", true) + err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Expects: []p2ptest.Expect{ - { + { //start expecting a subscription for RETRIEVE_REQUEST due to `RetrievalClientOnly` + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + { //expect a retrieve request message for the given hash Code: 5, Msg: &RetrieveRequestMsg{ Addr: hash0[:], @@ -74,9 +90,12 @@ func TestStreamerRetrieveRequest(t *testing.T) { } } +//Test requesting a chunk from a peer then issuing a "empty" OfferedHashesMsg (no hashes available yet) +//Should time out as the peer does not have the chunk (no syncing happened previously) func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalEnabled, + Syncing: SyncingDisabled, //do no syncing }) defer teardown() if err != nil { @@ -89,16 +108,31 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { peer := streamer.getPeer(node.ID()) + stream := NewStream(swarmChunkServerStreamName, "", true) + //simulate pre-subscription to RETRIEVE_REQUEST stream on peer peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{ - Stream: NewStream(swarmChunkServerStreamName, "", true), + Stream: stream, History: nil, Priority: Top, }) + //test the exchange err = tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { //first expect a subscription to the RETRIEVE_REQUEST stream + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ - { + { //then the actual RETRIEVE_REQUEST.... Code: 5, Msg: &RetrieveRequestMsg{ Addr: chunk.Address()[:], @@ -107,7 +141,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - { + { //to which the peer responds with offered hashes Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: nil, @@ -120,7 +154,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }) - expectedError := `exchange #0 "RetrieveRequestMsg": timed out` + //should fail with a timeout as the peer we are requesting + //the chunk from does not have the chunk + expectedError := `exchange #1 "RetrieveRequestMsg": timed out` if err == nil || err.Error() != expectedError { t.Fatalf("Expected error %v, got %v", expectedError, err) } @@ -130,7 +166,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalEnabled, + Syncing: SyncingDisabled, }) defer teardown() if err != nil { @@ -138,6 +175,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } node := tester.Nodes[0] + peer := streamer.getPeer(node.ID()) stream := NewStream(swarmChunkServerStreamName, "", true) @@ -156,6 +194,18 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } err = tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ { @@ -226,7 +276,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, }) defer teardown() if err != nil { @@ -241,6 +292,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { node := tester.Nodes[0] + //subscribe to custom stream stream := NewStream("foo", "", true) err = streamer.Subscribe(node.ID(), stream, NewRange(5, 8), Top) if err != nil { @@ -253,7 +305,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Expects: []p2ptest.Expect{ - { + { //first expect subscription to the custom stream... Code: 4, Msg: &SubscribeMsg{ Stream: stream, @@ -267,7 +319,8 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { p2ptest.Exchange{ Label: "ChunkDelivery message", Triggers: []p2ptest.Trigger{ - { + { //...then trigger a chunk delivery for the given chunk from peer in order for + //local node to get the chunk delivered Code: 6, Msg: &ChunkDeliveryMsg{ Addr: chunkKey, @@ -342,8 +395,9 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - SkipCheck: skipCheck, - DoServeRetrieve: true, + SkipCheck: skipCheck, + Syncing: SyncingDisabled, + Retrieval: RetrievalEnabled, }) bucket.Store(bucketKeyRegistry, r) @@ -408,20 +462,6 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return err } - //each of the nodes (except pivot node) subscribes to the stream of the next node - for j, node := range nodeIDs[0 : nodes-1] { - sid := nodeIDs[j+1] - item, ok := sim.NodeItem(node, bucketKeyRegistry) - if !ok { - return fmt.Errorf("No registry") - } - registry := item.(*Registry) - err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), nil, Top) - if err != nil { - return err - } - } - //get the pivot node's filestore item, ok := sim.NodeItem(*sim.PivotNodeID(), bucketKeyFileStore) if !ok { @@ -530,7 +570,8 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, - DoSync: true, + Syncing: SyncingDisabled, + Retrieval: RetrievalDisabled, SyncUpdateDelay: 0, }) diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 3164193b34..0c95fabb70 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -83,6 +83,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, SkipCheck: skipCheck, }) bucket.Store(bucketKeyRegistry, r) diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go index 0d3bc7f544..65cde2411c 100644 --- a/swarm/network/stream/lightnode_test.go +++ b/swarm/network/stream/lightnode_test.go @@ -25,7 +25,8 @@ import ( // when it is serving Retrieve requests. func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { registryOptions := &RegistryOptions{ - DoServeRetrieve: true, + Retrieval: RetrievalClientOnly, + Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() @@ -63,7 +64,8 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { // requests are disabled func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { registryOptions := &RegistryOptions{ - DoServeRetrieve: false, + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() @@ -106,7 +108,8 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { // when syncing is enabled. func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { registryOptions := &RegistryOptions{ - DoSync: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() @@ -150,7 +153,8 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { // when syncing is disabled. func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) { registryOptions := &RegistryOptions{ - DoSync: false, + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(t, registryOptions) defer teardown() diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index b81cfc0ca0..ad15193414 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -127,10 +127,9 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, + Retrieval: RetrievalEnabled, + Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, - DoRetrieve: true, - DoServeRetrieve: true, }) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 8d89f28cba..2ddbed9368 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -165,8 +165,8 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, - DoServeRetrieve: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, }) @@ -360,8 +360,8 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoServeRetrieve: true, - DoSync: true, + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, }) bucket.Store(bucketKeyRegistry, r) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 0ac374def1..695ff0c502 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -47,6 +47,31 @@ const ( HashSize = 32 ) +//Enumerate options for syncing and retrieval +type SyncingOption int +type RetrievalOption int + +//Syncing options +const ( + //Syncing disabled + SyncingDisabled SyncingOption = iota + //Register the client and the server but not subscribe + SyncingRegisterOnly + //Both client and server funcs are registered, subscribe sent automatically + SyncingAutoSubscribe +) + +const ( + //Retrieval disabled. Used mostly for tests to isolate syncing features (i.e. syncing only) + RetrievalDisabled RetrievalOption = iota + //Only the client side of the retrieve request is registered. + //(light nodes do not serve retrieve requests) + //once the client is registered, subscription to retrieve request stream is always sent + RetrievalClientOnly + //Both client and server funcs are registered, subscribe sent automatically + RetrievalEnabled +) + // Registry registry for outgoing and incoming streamer constructors type Registry struct { addr enode.ID @@ -60,16 +85,15 @@ type Registry struct { peers map[enode.ID]*Peer delivery *Delivery intervalsStore state.Store - doRetrieve bool + autoRetrieval bool //automatically subscribe to retrieve request stream maxPeerServers int } // RegistryOptions holds optional values for NewRegistry constructor. type RegistryOptions struct { SkipCheck bool - DoSync bool // Sets if the server syncs with peers. Default is true, set to false by lightnode or nosync flags. - DoRetrieve bool // Sets if the server issues Retrieve requests. Default is true. - DoServeRetrieve bool // Sets if the server serves Retrieve requests. Default is true, set to false by lightnode flag. + Syncing SyncingOption //Defines syncing behavior + Retrieval RetrievalOption //Defines retrieval behavior SyncUpdateDelay time.Duration MaxPeerServers int // The limit of servers for each peer in registry } @@ -82,6 +106,9 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy if options.SyncUpdateDelay <= 0 { options.SyncUpdateDelay = 15 * time.Second } + //check if retriaval has been disabled + retrieval := options.Retrieval != RetrievalDisabled + streamer := &Registry{ addr: localID, skipCheck: options.SkipCheck, @@ -90,13 +117,14 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy peers: make(map[enode.ID]*Peer), delivery: delivery, intervalsStore: intervalsStore, - doRetrieve: options.DoRetrieve, + autoRetrieval: retrieval, maxPeerServers: options.MaxPeerServers, } streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer - if options.DoServeRetrieve { + //if retrieval is enabled, register the server func, so that retrieve requests will be served (non-light nodes only) + if options.Retrieval == RetrievalEnabled { streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) { if !live { return nil, errors.New("only live retrieval requests supported") @@ -105,16 +133,21 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy }) } - streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { - return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) - }) + //if retrieval is not disabled, register the client func (both light nodes and normal nodes can issue retrieve requests) + if options.Retrieval != RetrievalDisabled { + streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { + return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) + }) + } - if options.DoSync { + //If syncing is not disabled, the syncing functions are registered (both client and server) + if options.Syncing != SyncingDisabled { RegisterSwarmSyncerServer(streamer, syncChunkStore) RegisterSwarmSyncerClient(streamer, syncChunkStore) } - if options.DoSync { + //if syncing is set to automatically subscribe to the syncing stream, start the subscription process + if options.Syncing == SyncingAutoSubscribe { // latestIntC function ensures that // - receiving from the in chan is not blocked by processing inside the for loop // - the latest int value is delivered to the loop after the processing is done @@ -385,7 +418,7 @@ func (r *Registry) Run(p *network.BzzPeer) error { defer close(sp.quit) defer sp.close() - if r.doRetrieve { + if r.autoRetrieval && !p.LightNode { err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", true), nil, Top) if err != nil { return err diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index e7f79e7a12..16c74d3b34 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -765,6 +765,8 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { func TestMaxPeerServersWithUnsubscribe(t *testing.T) { var maxPeerServers = 6 tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, MaxPeerServers: maxPeerServers, }) defer teardown() diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index f2be3bef99..113807b98a 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -114,6 +114,8 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bucket.Store(bucketKeyDelivery, delivery) r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, SkipCheck: skipCheck, }) diff --git a/swarm/swarm.go b/swarm/swarm.go index 7214abbda4..1fb5443fd4 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -175,18 +175,24 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil { return nil, err } + + syncing := stream.SyncingAutoSubscribe + if !config.SyncEnabled || config.LightNodeEnabled { + syncing = stream.SyncingDisabled + } + + retrieval := stream.RetrievalEnabled + if config.LightNodeEnabled { + retrieval = stream.RetrievalClientOnly + } + registryOptions := &stream.RegistryOptions{ SkipCheck: config.DeliverySkipCheck, - DoSync: config.SyncEnabled, - DoRetrieve: true, - DoServeRetrieve: true, + Syncing: syncing, + Retrieval: retrieval, SyncUpdateDelay: config.SyncUpdateDelay, MaxPeerServers: config.MaxStreamPeerServers, } - if config.LightNodeEnabled { - registryOptions.DoSync = false - registryOptions.DoRetrieve = false - } self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage From e2640a96d4311e3ac48de467a6eb3c88b95dfb3e Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 7 Nov 2018 09:55:56 +0100 Subject: [PATCH 040/100] miner: fix miner stress test (#18039) --- miner/stress_clique.go | 23 ++++++++--------------- miner/stress_ethash.go | 23 ++++++++--------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 8961091d56..7e19975aec 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -22,7 +22,6 @@ package main import ( "bytes" "crypto/ecdsa" - "fmt" "io/ioutil" "math/big" "math/rand" @@ -40,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,11 +61,11 @@ func main() { var ( nodes []*node.Node - enodes []string + enodes []*enode.Node ) for _, sealer := range sealers { // Start the node and wait until it's up - node, err := makeSealer(genesis, enodes) + node, err := makeSealer(genesis) if err != nil { panic(err) } @@ -76,18 +75,12 @@ func main() { time.Sleep(250 * time.Millisecond) } // Connect the node to al the previous ones - for _, enode := range enodes { - enode, err := discover.ParseNode(enode) - if err != nil { - panic(err) - } - node.Server().AddPeer(enode) + for _, n := range enodes { + node.Server().AddPeer(n) } - // Start tracking the node and it's enode url + // Start tracking the node and it's enode nodes = append(nodes, node) - - enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) - enodes = append(enodes, enode) + enodes = append(enodes, node.Server().Self()) // Inject the signer key and start sealing with it store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core return genesis } -func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { +func makeSealer(genesis *core.Genesis) (*node.Node, error) { // Define the basic configurations for the Ethereum node datadir, _ := ioutil.TempDir("", "") diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 5ed11d73a5..044ca9a218 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -21,7 +21,6 @@ package main import ( "crypto/ecdsa" - "fmt" "io/ioutil" "math/big" "math/rand" @@ -41,7 +40,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,11 +61,11 @@ func main() { var ( nodes []*node.Node - enodes []string + enodes []*enode.Node ) for i := 0; i < 4; i++ { // Start the node and wait until it's up - node, err := makeMiner(genesis, enodes) + node, err := makeMiner(genesis) if err != nil { panic(err) } @@ -76,18 +75,12 @@ func main() { time.Sleep(250 * time.Millisecond) } // Connect the node to al the previous ones - for _, enode := range enodes { - enode, err := discover.ParseNode(enode) - if err != nil { - panic(err) - } - node.Server().AddPeer(enode) + for _, n := range enodes { + node.Server().AddPeer(n) } - // Start tracking the node and it's enode url + // Start tracking the node and it's enode nodes = append(nodes, node) - - enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) - enodes = append(enodes, enode) + enodes = append(enodes, node.Server().Self()) // Inject the signer key and start sealing with it store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { return genesis } -func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { +func makeMiner(genesis *core.Genesis) (*node.Node, error) { // Define the basic configurations for the Ethereum node datadir, _ := ioutil.TempDir("", "") From 80e2f3aca42583856669372cfc636cb7fd370fdb Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Wed, 7 Nov 2018 22:17:41 +1100 Subject: [PATCH 041/100] travis, appveyor: bump to Go 1.11.2 (#18031) --- .travis.yml | 2 +- appveyor.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69535b7ef3..c1cc7c4aa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,7 +148,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go diff --git a/appveyor.yml b/appveyor.yml index 11848ddb95..e5126b2529 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip - - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip + - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version From 0fe0b8f7b9904767cdd73cf1aac4d7ad543d95a7 Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Wed, 7 Nov 2018 19:22:31 +0800 Subject: [PATCH 042/100] p2p/protocols: use keyed fields for struct instantiation (#18017) --- p2p/protocols/protocol_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index 2874af48d2..a26222cd8b 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -318,7 +318,7 @@ func TestProtocolHook(t *testing.T) { <-testHook.waitC time.Sleep(100 * time.Millisecond) - err = tester.TestDisconnected(&p2ptest.Disconnect{tester.Nodes[1].ID(), testHook.err}) + err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: tester.Nodes[1].ID(), Error: testHook.err}) if err != nil { t.Fatalf("Expected a specific disconnect error, but got different one: %v", err) } From dc6648bb58fb37f7b5d9fd8e7ffbeb4cf45492fd Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 7 Nov 2018 13:18:07 +0100 Subject: [PATCH 043/100] downloader: measure successfull deliveries, not failed (#17983) * downloader: measure successfull deliveries, not failed * downloader: fix typos --- eth/downloader/statesync.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 8d33dfec74..29d5ee4ddb 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) { s.d.dropPeer(req.peer.id) } // Process all the received blobs and check for stale delivery - if err = s.process(req); err != nil { + delivered, err := s.process(req) + if err != nil { log.Warn("Node data write error", "err", err) return err } - req.peer.SetNodeDataIdle(len(req.response)) + req.peer.SetNodeDataIdle(delivered) } } return nil @@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) { // process iterates over a batch of delivered state data, injecting each item // into a running state sync, re-queuing any items that were requested but not // delivered. -func (s *stateSync) process(req *stateReq) error { +// Returns whether the peer actually managed to deliver anything of value, +// and any error that occurred +func (s *stateSync) process(req *stateReq) (int, error) { // Collect processing stats and update progress if valid data was received - duplicate, unexpected := 0, 0 + duplicate, unexpected, successful := 0, 0, 0 defer func(start time.Time) { if duplicate > 0 || unexpected > 0 { @@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error { // Iterate over all the delivered data and inject one-by-one into the trie progress := false - for _, blob := range req.response { prog, hash, err := s.processNodeData(blob) switch err { @@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error { s.numUncommitted++ s.bytesUncommitted += len(blob) progress = progress || prog + successful++ case trie.ErrNotRequested: unexpected++ case trie.ErrAlreadyProcessed: duplicate++ default: - return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) + return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } if _, ok := req.tasks[hash]; ok { delete(req.tasks, hash) @@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error { // If we've requested the node too many times already, it may be a malicious // sync where nobody has the right data. Abort. if len(task.attempts) >= npeers { - return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) } // Missing item, place into the retry queue. s.tasks[hash] = task } - return nil + return successful, nil } // processNodeData tries to inject a trie node data blob delivered from a remote From eea3ae42a3d9bcbd33474c0e482754c5196a469f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 7 Nov 2018 13:47:11 +0100 Subject: [PATCH 044/100] core, eth/downloader: fix validation flaw, fix downloader printout flaw (#17974) --- core/block_validator.go | 12 ++++++------ eth/downloader/downloader.go | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 1329f62427..3b9496fece 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { return ErrKnownBlock } - if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { - if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { - return consensus.ErrUnknownAncestor - } - return consensus.ErrPrunedAncestor - } // Header validity is known at this point, check the uncles and transactions header := block.Header() if err := v.engine.VerifyUncles(v.bc, block); err != nil { @@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) } + if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { + if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { + return consensus.ErrUnknownAncestor + } + return consensus.ErrPrunedAncestor + } return nil } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f01a8fdbd8..56c54c8ed5 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -740,6 +740,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return 0, errBadPeer } start = check + hash = h case <-timeout: p.log.Debug("Waiting for search header timed out", "elapsed", ttl) From 5b74bb6445a7fb7220da74c6c6e891c20e9a26dc Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 7 Nov 2018 13:55:54 +0100 Subject: [PATCH 045/100] signer: remove ineffectual assignments (#18049) * signer: remove ineffectual assignments * signer: remove ineffectual assignments --- signer/core/abihelper_test.go | 6 +++--- signer/rules/rules_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/signer/core/abihelper_test.go b/signer/core/abihelper_test.go index 2afeec73e6..878210be1f 100644 --- a/signer/core/abihelper_test.go +++ b/signer/core/abihelper_test.go @@ -38,13 +38,13 @@ func verify(t *testing.T, jsondata, calldata string, exp []interface{}) { cd := common.Hex2Bytes(calldata) sigdata, argdata := cd[:4], cd[4:] method, err := abispec.MethodById(sigdata) - if err != nil { t.Fatal(err) } - data, err := method.Inputs.UnpackValues(argdata) - + if err != nil { + t.Fatal(err) + } if len(data) != len(exp) { t.Fatalf("Mismatched length, expected %d, got %d", len(exp), len(data)) } diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 55614077ca..0b520a15bf 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -151,7 +151,7 @@ func TestListRequest(t *testing.T) { t.Errorf("Couldn't create evaluator %v", err) return } - resp, err := r.ApproveListing(&core.ListRequest{ + resp, _ := r.ApproveListing(&core.ListRequest{ Accounts: accs, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, }) @@ -515,7 +515,7 @@ func TestLimitWindow(t *testing.T) { r.OnApprovedTx(response) } // Fourth should fail - resp, err := r.ApproveTx(dummyTx(h)) + resp, _ := r.ApproveTx(dummyTx(h)) if resp.Approved { t.Errorf("Expected check to resolve to 'Reject'") } @@ -609,8 +609,8 @@ func TestContextIsCleared(t *testing.T) { t.Fatalf("Failed to load bootstrap js: %v", err) } tx := dummyTxWithV(0) - r1, err := r.ApproveTx(tx) - r2, err := r.ApproveTx(tx) + r1, _ := r.ApproveTx(tx) + r2, _ := r.ApproveTx(tx) if r1.Approved != r2.Approved { t.Errorf("Expected execution context to be cleared between executions") } From b35165555d737042d5f958413827c31a7a5f4805 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Wed, 7 Nov 2018 07:30:19 -0600 Subject: [PATCH 046/100] eth/downloader: remove the expired id directly (#17963) --- eth/downloader/queue.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index c6b635aff1..863cc8de14 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, } // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) + + // Remove the expired requests from the pending pool directly + delete(pendPool, id) } } - // Remove the expired requests from the pending pool - for id := range expiries { - delete(pendPool, id) - } return expiries } From 36ca85fa1c2936087b2c3d976d3576f0f5d2157e Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Wed, 7 Nov 2018 14:49:42 +0100 Subject: [PATCH 047/100] swarm/api: Fix #18007, missing signature should return HTTP 400 (#18008) --- swarm/api/http/server.go | 10 ++++++---- swarm/api/http/server_test.go | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index e9005104ee..803b789878 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -484,7 +484,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { return } - if updateRequest.IsUpdate() { + switch { + case updateRequest.IsUpdate(): // Verify that the signature is intact and that the signer is authorized // to update this feed // Check this early, to avoid creating a feed and then not being able to set its first update. @@ -497,9 +498,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { respondError(w, r, err.Error(), http.StatusInternalServerError) return } - } - - if query.Get("manifest") == "1" { + fallthrough + case query.Get("manifest") == "1": // we create a manifest so we can retrieve feed updates with bzz:// later // this manifest has a special "feed type" manifest, and saves the // feed identification used to retrieve feed updates later @@ -519,6 +519,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, string(outdata)) w.Header().Add("Content-type", "application/json") + default: + respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest) } } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1cf7ff5770..159c8a159f 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -333,15 +333,45 @@ func TestBzzFeed(t *testing.T) { } urlQuery = testUrl.Query() body = updateRequest.AppendValues(urlQuery) // this adds all query parameters + goodQueryParameters := urlQuery.Encode() // save the query parameters for a second attempt + + // create bad query parameters in which the signature is missing + urlQuery.Del("signature") testUrl.RawQuery = urlQuery.Encode() + // 1st attempt with bad query parameters in which the signature is missing resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("Update returned %s", resp.Status) + expectedCode := http.StatusBadRequest + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) + } + + // 2nd attempt with bad query parameters in which the signature is of incorrect length + urlQuery.Set("signature", "0xabcd") // should be 130 hex chars + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + expectedCode = http.StatusBadRequest + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) + } + + // 3rd attempt, with good query parameters: + testUrl.RawQuery = goodQueryParameters + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + expectedCode = http.StatusOK + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) } // get latest update through bzz-feed directly From 0bcff8f52505fa1eece6c8fbfc5cab8aa9d9ed5d Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 7 Nov 2018 15:07:43 +0100 Subject: [PATCH 048/100] eth/downloader: speed up tests by generating chain only once (#17916) * core: speed up GenerateChain Use a mock implementation of ChainReader instead of creating and destroying a BlockChain object for each generated block. * eth/downloader: speed up tests by generating chain only once This change reworks the downloader tests so they share a common test blockchain instead of generating a chain in every test. The tests are roughly twice as fast now. --- core/chain_makers.go | 44 +- eth/downloader/downloader_test.go | 816 ++++++++++-------------------- eth/downloader/testchain_test.go | 221 ++++++++ 3 files changed, 520 insertions(+), 561 deletions(-) create mode 100644 eth/downloader/testchain_test.go diff --git a/core/chain_makers.go b/core/chain_makers.go index 0bc453fdf3..e3a5537a40 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -33,12 +33,11 @@ import ( // BlockGen creates blocks for testing. // See GenerateChain for a detailed explanation. type BlockGen struct { - i int - parent *types.Block - chain []*types.Block - chainReader consensus.ChainReader - header *types.Header - statedb *state.StateDB + i int + parent *types.Block + chain []*types.Block + header *types.Header + statedb *state.StateDB gasPool *GasPool txs []*types.Transaction @@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) { // For index -1, PrevBlock returns the parent block given to GenerateChain. func (b *BlockGen) PrevBlock(index int) *types.Block { if index >= b.i { - panic("block index out of range") + panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i)) } if index == -1 { return b.parent @@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) { if b.header.Time.Cmp(b.parent.Header().Time) <= 0 { panic("block time out of range") } - b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header()) + chainreader := &fakeChainReader{config: b.config} + b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header()) } // GenerateChain creates a chain of n blocks. The first block's @@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse config = params.TestChainConfig } blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) + chainreader := &fakeChainReader{config: config} genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { - // TODO(karalabe): This is needed for clique, which depends on multiple blocks. - // It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow. - blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil) - defer blockchain.Stop() - - b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine} - b.header = makeHeader(b.chainReader, parent, statedb, b.engine) + b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} + b.header = makeHeader(chainreader, parent, statedb, b.engine) // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { @@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } if b.engine != nil { // Finalize and seal the block - block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts) + block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts) // Write state changes to db root, err := statedb.Commit(config.IsEIP158(b.header.Number)) @@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd }) return blocks } + +type fakeChainReader struct { + config *params.ChainConfig + genesis *types.Block +} + +// Config returns the chain configuration. +func (cr *fakeChainReader) Config() *params.ChainConfig { + return cr.config +} + +func (cr *fakeChainReader) CurrentHeader() *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } +func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index dad626e89a..1fe02d8849 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -25,22 +25,14 @@ import ( "testing" "time" + ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) -) - // Reduce some of the parameters to make the tester faster. func init() { MaxForkAncestry = uint64(10000) @@ -55,6 +47,7 @@ type downloadTester struct { genesis *types.Block // Genesis blocks used by the tester and peers stateDb ethdb.Database // Database used by the tester for syncing from peers peerDb ethdb.Database // Database of the peers containing all data + peers map[string]*downloadTesterPeer ownHashes []common.Hash // Hash chain belonging to the tester ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester @@ -62,129 +55,27 @@ type downloadTester struct { ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain - peerHashes map[string][]common.Hash // Hash chain belonging to different test peers - peerHeaders map[string]map[common.Hash]*types.Header // Headers belonging to different test peers - peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers - peerReceipts map[string]map[common.Hash]types.Receipts // Receipts belonging to different test peers - peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains - - peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return - lock sync.RWMutex } // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb := ethdb.NewMemDatabase() - genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) - tester := &downloadTester{ - genesis: genesis, - peerDb: testdb, - ownHashes: []common.Hash{genesis.Hash()}, - ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil}, - ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, - peerHashes: make(map[string][]common.Hash), - peerHeaders: make(map[string]map[common.Hash]*types.Header), - peerBlocks: make(map[string]map[common.Hash]*types.Block), - peerReceipts: make(map[string]map[common.Hash]types.Receipts), - peerChainTds: make(map[string]map[common.Hash]*big.Int), - peerMissingStates: make(map[string]map[common.Hash]bool), + genesis: testGenesis, + peerDb: testDB, + peers: make(map[string]*downloadTesterPeer), + ownHashes: []common.Hash{testGenesis.Hash()}, + ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, + ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, + ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, + ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, } tester.stateDb = ethdb.NewMemDatabase() - tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) - + tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) - return tester } -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts, heavy bool) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) { - // Generate the block chain - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), dl.peerDb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If a heavy chain is requested, delay blocks to raise difficulty - if heavy { - block.OffsetTime(-1) - } - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == dl.genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ - ParentHash: block.PrevBlock(i - 1).Hash(), - Number: big.NewInt(block.Number().Int64() - 1), - }) - } - }) - // Convert the block-chain into a hash-chain and header/block maps - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - - headerm := make(map[common.Hash]*types.Header, n+1) - headerm[parent.Hash()] = parent.Header() - - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - - receiptm := make(map[common.Hash]types.Receipts, n+1) - receiptm[parent.Hash()] = parentReceipts - - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - headerm[b.Hash()] = b.Header() - blockm[b.Hash()] = b - receiptm[b.Hash()] = receipts[i] - } - return hashes, headerm, blockm, receiptm -} - -// makeChainFork creates two chains of length n, such that h1[:f] and -// h2[:f] are different but have a common suffix of length n-f. -func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentReceipts types.Receipts, balanced bool) ([]common.Hash, []common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]*types.Block, map[common.Hash]types.Receipts, map[common.Hash]types.Receipts) { - // Create the common suffix - hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false) - - // Create the forks, making the second heavier if non balanced forks were requested - hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false) - hashes1 = append(hashes1, hashes[1:]...) - - heavy := false - if !balanced { - heavy = true - } - hashes2, headers2, blocks2, receipts2 := dl.makeChain(f, 2, blocks[hashes[0]], receipts[hashes[0]], heavy) - hashes2 = append(hashes2, hashes[1:]...) - - for hash, header := range headers { - headers1[hash] = header - headers2[hash] = header - } - for hash, block := range blocks { - blocks1[hash] = block - blocks2[hash] = block - } - for hash, receipt := range receipts { - receipts1[hash] = receipt - receipts2[hash] = receipt - } - return hashes1, hashes2, headers1, headers2, blocks1, blocks2, receipts1, receipts2 -} - // terminate aborts any operations on the embedded downloader and releases all // held resources. func (dl *downloadTester) terminate() { @@ -194,13 +85,10 @@ func (dl *downloadTester) terminate() { // sync starts synchronizing with a remote peer, blocking until it completes. func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { dl.lock.RLock() - hash := dl.peerHashes[id][0] + hash := dl.peers[id].chain.headBlock().Hash() // If no particular TD was requested, load from the peer's blockchain if td == nil { - td = big.NewInt(1) - if diff, ok := dl.peerChainTds[id][hash]; ok { - td = diff - } + td = dl.peers[id].chain.td(hash) } dl.lock.RUnlock() @@ -302,7 +190,7 @@ func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int { } // InsertHeaderChain injects a new batch of headers into the simulated chain. -func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (int, error) { +func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -331,7 +219,7 @@ func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq i } // InsertChain injects a new batch of blocks into the simulated chain. -func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { +func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -353,7 +241,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { } // InsertReceiptChain injects a new batch of receipts into the simulated chain. -func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (int, error) { +func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -387,60 +275,13 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error { - return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0) -} - -// newSlowPeer registers a new block download source into the downloader, with a -// specific delay time on processing the network packets sent to it, simulating -// potentially slow network IO. -func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts, delay time.Duration) error { +func (dl *downloadTester) newPeer(id string, version int, chain *testChain) error { dl.lock.Lock() defer dl.lock.Unlock() - var err = dl.downloader.RegisterPeer(id, version, &downloadTesterPeer{dl: dl, id: id, delay: delay}) - if err == nil { - // Assign the owned hashes, headers and blocks to the peer (deep copy) - dl.peerHashes[id] = make([]common.Hash, len(hashes)) - copy(dl.peerHashes[id], hashes) - - dl.peerHeaders[id] = make(map[common.Hash]*types.Header) - dl.peerBlocks[id] = make(map[common.Hash]*types.Block) - dl.peerReceipts[id] = make(map[common.Hash]types.Receipts) - dl.peerChainTds[id] = make(map[common.Hash]*big.Int) - dl.peerMissingStates[id] = make(map[common.Hash]bool) - - genesis := hashes[len(hashes)-1] - if header := headers[genesis]; header != nil { - dl.peerHeaders[id][genesis] = header - dl.peerChainTds[id][genesis] = header.Difficulty - } - if block := blocks[genesis]; block != nil { - dl.peerBlocks[id][genesis] = block - dl.peerChainTds[id][genesis] = block.Difficulty() - } - - for i := len(hashes) - 2; i >= 0; i-- { - hash := hashes[i] - - if header, ok := headers[hash]; ok { - dl.peerHeaders[id][hash] = header - if _, ok := dl.peerHeaders[id][header.ParentHash]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(header.Difficulty, dl.peerChainTds[id][header.ParentHash]) - } - } - if block, ok := blocks[hash]; ok { - dl.peerBlocks[id][hash] = block - if _, ok := dl.peerBlocks[id][block.ParentHash()]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][block.ParentHash()]) - } - } - if receipt, ok := receipts[hash]; ok { - dl.peerReceipts[id][hash] = receipt - } - } - } - return err + peer := &downloadTesterPeer{dl: dl, id: id, chain: chain} + dl.peers[id] = peer + return dl.downloader.RegisterPeer(id, version, peer) } // dropPeer simulates a hard peer removal from the connection pool. @@ -448,89 +289,48 @@ func (dl *downloadTester) dropPeer(id string) { dl.lock.Lock() defer dl.lock.Unlock() - delete(dl.peerHashes, id) - delete(dl.peerHeaders, id) - delete(dl.peerBlocks, id) - delete(dl.peerChainTds, id) - + delete(dl.peers, id) dl.downloader.UnregisterPeer(id) } type downloadTesterPeer struct { - dl *downloadTester - id string - delay time.Duration - lock sync.RWMutex -} - -// setDelay is a thread safe setter for the network delay value. -func (dlp *downloadTesterPeer) setDelay(delay time.Duration) { - dlp.lock.Lock() - defer dlp.lock.Unlock() - - dlp.delay = delay -} - -// waitDelay is a thread safe way to sleep for the configured time. -func (dlp *downloadTesterPeer) waitDelay() { - dlp.lock.RLock() - delay := dlp.delay - dlp.lock.RUnlock() - - time.Sleep(delay) + dl *downloadTester + id string + lock sync.RWMutex + chain *testChain + missingStates map[common.Hash]bool // State entries that fast sync should not return } // Head constructs a function to retrieve a peer's current head hash // and total difficulty. func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - return dlp.dl.peerHashes[dlp.id][0], nil + b := dlp.chain.headBlock() + return b.Hash(), dlp.chain.td(b.Hash()) } // RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - // Find the canonical number of the hash - dlp.dl.lock.RLock() - number := uint64(0) - for num, hash := range dlp.dl.peerHashes[dlp.id] { - if hash == origin { - number = uint64(len(dlp.dl.peerHashes[dlp.id]) - num - 1) - break - } + if reverse { + panic("reverse header requests not supported") } - dlp.dl.lock.RUnlock() - // Use the absolute header fetcher to satisfy the query - return dlp.RequestHeadersByNumber(number, amount, skip, reverse) + result := dlp.chain.headersByHash(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) + return nil } // RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - // Gather the next batch of headers - hashes := dlp.dl.peerHashes[dlp.id] - headers := dlp.dl.peerHeaders[dlp.id] - result := make([]*types.Header, 0, amount) - for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ { - if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok { - result = append(result, header) - } + if reverse { + panic("reverse header requests not supported") } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dlp.dl.downloader.DeliverHeaders(dlp.id, result) - }() + + result := dlp.chain.headersByNumber(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) return nil } @@ -538,24 +338,8 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, // peer in the download tester. The returned function can be used to retrieve // batches of block bodies from the particularly requested peer. func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - blocks := dlp.dl.peerBlocks[dlp.id] - - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - go dlp.dl.downloader.DeliverBodies(dlp.id, transactions, uncles) - + txs, uncles := dlp.chain.bodies(hashes) + go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles) return nil } @@ -563,21 +347,8 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - receipts := dlp.dl.peerReceipts[dlp.id] - - results := make([][]*types.Receipt, 0, len(hashes)) - for _, hash := range hashes { - if receipt, ok := receipts[hash]; ok { - results = append(results, receipt) - } - } - go dlp.dl.downloader.DeliverReceipts(dlp.id, results) - + receipts := dlp.chain.receipts(hashes) + go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts) return nil } @@ -585,21 +356,18 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of node state data from the particularly requested peer. func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { - dlp.waitDelay() - dlp.dl.lock.RLock() defer dlp.dl.lock.RUnlock() results := make([][]byte, 0, len(hashes)) for _, hash := range hashes { if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { - if !dlp.dl.peerMissingStates[dlp.id][hash] { + if !dlp.missingStates[hash] { results = append(results, data) } } } go dlp.dl.downloader.DeliverNodeData(dlp.id, results) - return nil } @@ -639,21 +407,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng if rs := len(tester.ownReceipts); rs != receipts { t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts) } - // Verify the state trie too for fast syncs - /*if tester.downloader.mode == FastSync { - pivot := uint64(0) - var index int - if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common { - index = pivot - } else { - index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot) - } - if index > 0 { - if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil { - t.Fatalf("state reconstruction failed: %v", err) - } - } - }*/ } // Tests that simple synchronization against a canonical chain works correctly. @@ -673,16 +426,14 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(blockCacheItems - 15) + tester.newPeer("peer", protocol, chain) // Synchronise with the peer and make sure all relevant data was retrieved if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if a large batch of blocks are being downloaded, it is throttled @@ -699,10 +450,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long block chain to download and the tester - targetBlocks := 8 * blockCacheItems - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + targetBlocks := testChainBase.len() - 1 + tester.newPeer("peer", protocol, testChainBase) // Wrap the importer to allow stepping blocked, proceed := uint32(0), make(chan struct{}) @@ -734,9 +483,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { cached = len(tester.downloader.queue.blockDonePool) if mode == FastSync { if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached { - //if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot { cached = receipts - //} } } frozen = int(atomic.LoadUint32(&blocked)) @@ -786,24 +533,22 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkLightB.shorten(testChainBase.len() + 80) + tester.newPeer("fork A", protocol, chainA) + tester.newPeer("fork B", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("fork A", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("fork B", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that synchronising against a much shorter but much heavyer fork works @@ -821,24 +566,22 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 4*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkHeavy.shorten(testChainBase.len() + 80) + tester.newPeer("light", protocol, chainA) + tester.newPeer("heavy", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("light", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("heavy", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that chain forks are contained within a certain interval of the current @@ -857,18 +600,16 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA + chainB := testChainForkLightB + tester.newPeer("original", protocol, chainA) + tester.newPeer("rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { @@ -893,17 +634,16 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit + chainA := testChainForkLightA + chainB := testChainForkHeavy + tester.newPeer("original", protocol, chainA) + tester.newPeer("heavy-rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { @@ -924,7 +664,7 @@ func TestInactiveDownloader62(t *testing.T) { t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } } @@ -962,17 +702,8 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download and the tester - targetBlocks := blockCacheItems - 15 - if targetBlocks >= MaxHashFetch { - targetBlocks = MaxHashFetch - 15 - } - if targetBlocks >= MaxHeaderFetch { - targetBlocks = MaxHeaderFetch - 15 - } - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(MaxHeaderFetch) + tester.newPeer("peer", protocol, chain) // Make sure canceling works with a pristine downloader tester.downloader.Cancel() @@ -1005,17 +736,16 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Create various peers with various parts of the chain targetPeers := 8 - targetBlocks := targetPeers*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetPeers * 100) for i := 0; i < targetPeers; i++ { id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts) + tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1))) } if err := tester.sync("peer #0", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that synchronisations behave well in multi-version protocol environments @@ -1034,24 +764,23 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Create peers of every type - tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) - tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) - tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) + tester.newPeer("peer 62", 62, chain) + tester.newPeer("peer 63", 63, chain) + tester.newPeer("peer 64", 64, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peerHashes[peer]; !ok { + if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) } } @@ -1073,10 +802,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a block chain to download - targetBlocks := 2*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase + tester.newPeer("peer", protocol, chain) // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) @@ -1090,16 +817,16 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Validate the number of block bodies that should have been requested bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range blocks { + for _, block := range chain.blockm { if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { bodiesNeeded++ } } - for _, receipt := range receipts { + for _, receipt := range chain.receiptm { if mode == FastSync && len(receipt) > 0 { receiptsNeeded++ } @@ -1127,24 +854,20 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - // Attempt a full sync with an attacker feeding gapped headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["attack"], hashes[missing]) + chain := testChainBase.shorten(blockCacheItems - 15) + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if requested headers are shifted (i.e. first is missing), the queue @@ -1162,25 +885,24 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Attempt a full sync with an attacker feeding shifted headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - delete(tester.peerHeaders["attack"], hashes[len(hashes)-2]) - delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) - delete(tester.peerReceipts["attack"], hashes[len(hashes)-2]) - + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[1]) + delete(brokenChain.blockm, brokenChain.chain[1]) + delete(brokenChain.receiptm, brokenChain.chain[1]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } + // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that upon detecting an invalid header, the recent ones are rolled back @@ -1198,13 +920,14 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Create a small enough block chain to download targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetBlocks) // Attempt to sync with an attacker that feeds junk during the fast sync phase. // This should result in the last fsHeaderSafetyNet headers being rolled back. - tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts) missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) + fastAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) + tester.newPeer("fast-attack", protocol, fastAttackChain) if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") @@ -1212,13 +935,15 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } + // Attempt to sync with an attacker that feeds junk during the block import phase. // This should result in both the last fsHeaderSafetyNet number of headers being // rolled back, and also the pivot point being reverted to a non-block status. - tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts) missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in - delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing]) + blockAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in + delete(blockAttackChain.headerm, blockAttackChain.chain[missing]) + tester.newPeer("block-attack", protocol, blockAttackChain) if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") @@ -1231,19 +956,18 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } + // Attempt to sync with an attacker that withholds promised blocks after the // fast sync pivot point. This could be a trial to leave the node with a bad // but already imported pivot block. - tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts) - missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - + withholdAttackChain := chain.shorten(chain.len()) + tester.newPeer("withhold-attack", protocol, withholdAttackChain) tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i <= len(hashes); i++ { - delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i]) + for i := missing; i < withholdAttackChain.len(); i++ { + delete(withholdAttackChain.headerm, withholdAttackChain.chain[i]) } tester.downloader.syncInitHook = nil } - if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } @@ -1255,20 +979,21 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } - // Synchronise with the valid peer and make sure sync succeeds. Since the last - // rollback should also disable fast syncing for this process, verify that we - // did a fresh full sync. Note, we can't assert anything about the receipts - // since we won't purge the database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + + // synchronise with the valid peer and make sure sync succeeds. Since the last rollback + // should also disable fast syncing for this process, verify that we did a fresh full + // sync. Note, we can't assert anything about the receipts since we won't purge the + // database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - if hs := len(tester.ownHeaders); hs != len(headers) { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers)) + if hs := len(tester.ownHeaders); hs != chain.len() { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len()) } if mode != LightSync { - if bs := len(tester.ownBlocks); bs != len(blocks) { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks)) + if bs := len(tester.ownBlocks); bs != chain.len() { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len()) } } } @@ -1288,9 +1013,8 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false) - tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts) - + chain := testChainBase.shorten(1) + tester.newPeer("attack", protocol, chain) if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) } @@ -1333,21 +1057,22 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Run the tests and check disconnection status tester := newTester() defer tester.terminate() + chain := testChainBase.shorten(1) for i, tt := range tests { // Register a new peer and ensure it's presence id := fmt.Sprintf("test %d", i) - if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil { + if err := tester.newPeer(id, protocol, chain); err != nil { t.Fatalf("test %d: failed to register new peer: %v", i, err) } - if _, ok := tester.peerHashes[id]; !ok { + if _, ok := tester.peers[id]; !ok { t.Fatalf("test %d: registered peer not found", i) } // Simulate a synchronisation and check the required result tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync) - if _, ok := tester.peerHashes[id]; !ok != tt.drop { + if _, ok := tester.peers[id]; !ok != tt.drop { t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) } } @@ -1367,10 +1092,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1380,12 +1102,10 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) + tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2)) pending := new(sync.WaitGroup) pending.Add(1) @@ -1396,16 +1116,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chain.len()/2 - 1), + }) progress <- struct{}{} pending.Wait() // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts) + tester.newPeer("peer-full", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("peer-full", nil, mode); err != nil { @@ -1413,15 +1132,29 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len()/2 - 1), + HighestBlock: uint64(chain.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) +} + +func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + t.Helper() + p := d.Progress() + p.KnownStates, p.PulledStates = 0, 0 + want.KnownStates, want.PulledStates = 0, 0 + if p != want { + t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) } } @@ -1440,10 +1173,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a forked chain to simulate origin revertal - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHashFetch) + chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHashFetch) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1453,15 +1184,12 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("fork A", protocol, chainA) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork A", nil, mode); err != nil { @@ -1469,9 +1197,10 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) - } + + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chainA.len() - 1), + }) progress <- struct{}{} pending.Wait() @@ -1479,9 +1208,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + tester.newPeer("fork B", protocol, chainB) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork B", nil, mode); err != nil { @@ -1489,16 +1217,20 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainA.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainB.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) } // Tests that if synchronisation is aborted due to some failure, then the progress @@ -1516,10 +1248,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1529,20 +1258,18 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Attempt a full sync with a faulty peer - tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["faulty"], hashes[missing]) - delete(tester.peerBlocks["faulty"], hashes[missing]) - delete(tester.peerReceipts["faulty"], hashes[missing]) + brokenChain := chain.shorten(chain.len()) + missing := brokenChain.len() / 2 + delete(brokenChain.headerm, brokenChain.chain[missing]) + delete(brokenChain.blockm, brokenChain.chain[missing]) + delete(brokenChain.receiptm, brokenChain.chain[missing]) + tester.newPeer("faulty", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("faulty", nil, mode); err == nil { @@ -1550,16 +1277,17 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress origin remind the same after a failure - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + // Synchronise with a good peer and check that the progress origin remind the same + // after a failure + tester.newPeer("valid", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("valid", nil, mode); err != nil { @@ -1567,16 +1295,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", afterFailedSync) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) } // Tests that if an attacker fakes a chain height, after the attack is detected, @@ -1593,34 +1320,27 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small block chain - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) progress := make(chan struct{}) - tester.downloader.syncInitHook = func(origin, latest uint64) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } - // Create and sync with an attacker that promises a higher chain than available - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - for i := 1; i < 3; i++ { - delete(tester.peerHeaders["attack"], hashes[i]) - delete(tester.peerBlocks["attack"], hashes[i]) - delete(tester.peerReceipts["attack"], hashes[i]) + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Create and sync with an attacker that promises a higher chain than available. + brokenChain := chain.shorten(chain.len()) + numMissing := 5 + for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- { + delete(brokenChain.headerm, brokenChain.chain[i]) } + tester.newPeer("attack", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("attack", nil, mode); err == nil { @@ -1628,14 +1348,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress height has been reduced to the true value - tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts) + // Synchronise with a good peer and check that the progress height has been reduced to + // the true value. + validChain := chain.shorten(chain.len() - numMissing) + tester.newPeer("valid", protocol, validChain) pending.Add(1) go func() { @@ -1645,23 +1368,25 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + CurrentBlock: afterFailedSync.CurrentBlock, + HighestBlock: uint64(validChain.len() - 1), + }) + + // Check final progress after successful sync. progress <- struct{}{} pending.Wait() - - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(validChain.len() - 1), + HighestBlock: uint64(validChain.len() - 1), + }) } // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -// We use data driven subtests to manage this so that it will be parallel on its own -// and not with the other tests, avoiding intermittent failures. func TestDeliverHeadersHang(t *testing.T) { + t.Parallel() + testCases := []struct { protocol int syncMode SyncMode @@ -1675,15 +1400,38 @@ func TestDeliverHeadersHang(t *testing.T) { } for _, tc := range testCases { t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) { + t.Parallel() testDeliverHeadersHang(t, tc.protocol, tc.syncMode) }) } } +func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { + master := newTester() + defer master.terminate() + chain := testChainBase.shorten(15) + + for i := 0; i < 200; i++ { + tester := newTester() + tester.peerDb = master.peerDb + tester.newPeer("peer", protocol, chain) + + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + } +} + type floodingTestPeer struct { peer Peer tester *downloadTester - pend sync.WaitGroup } func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } @@ -1702,54 +1450,32 @@ func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { deliveriesDone := make(chan struct{}, 500) - for i := 0; i < cap(deliveriesDone); i++ { + for i := 0; i < cap(deliveriesDone)-1; i++ { peer := fmt.Sprintf("fake-peer%d", i) - ftp.pend.Add(1) - go func() { ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) deliveriesDone <- struct{}{} - ftp.pend.Done() }() } - // Deliver the actual requested headers. - go ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + // None of the extra deliveries should block. timeout := time.After(60 * time.Second) + launched := false for i := 0; i < cap(deliveriesDone); i++ { select { case <-deliveriesDone: + if !launched { + // Start delivering the requested headers + // after one of the flooding responses has arrived. + go func() { + ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + deliveriesDone <- struct{}{} + }() + launched = true + } case <-timeout: panic("blocked") } } return nil } - -func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - master := newTester() - defer master.terminate() - - hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) - for i := 0; i < 200; i++ { - tester := newTester() - tester.peerDb = master.peerDb - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, - } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() - - // Flush all goroutines to prevent messing with subsequent tests - tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait() - } -} diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go new file mode 100644 index 0000000000..0b5a214257 --- /dev/null +++ b/eth/downloader/testchain_test.go @@ -0,0 +1,221 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +// Test chain parameters. +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + testDB = ethdb.NewMemDatabase() + testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000)) +) + +// The common prefix of all test chains: +var testChainBase = newTestChain(blockCacheItems+200, testGenesis) + +// Different forks on top of the base chain: +var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain + +func init() { + var forkLen = int(MaxForkAncestry + 50) + var wg sync.WaitGroup + wg.Add(3) + go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() + go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() + go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() + wg.Wait() +} + +type testChain struct { + genesis *types.Block + chain []common.Hash + headerm map[common.Hash]*types.Header + blockm map[common.Hash]*types.Block + receiptm map[common.Hash][]*types.Receipt + tdm map[common.Hash]*big.Int +} + +// newTestChain creates a blockchain of the given length. +func newTestChain(length int, genesis *types.Block) *testChain { + tc := new(testChain).copy(length) + tc.genesis = genesis + tc.chain = append(tc.chain, genesis.Hash()) + tc.headerm[tc.genesis.Hash()] = tc.genesis.Header() + tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty() + tc.blockm[tc.genesis.Hash()] = tc.genesis + tc.generate(length-1, 0, genesis, false) + return tc +} + +// makeFork creates a fork on top of the test chain. +func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { + fork := tc.copy(tc.len() + length) + fork.generate(length, seed, tc.headBlock(), heavy) + return fork +} + +// shorten creates a copy of the chain with the given length. It panics if the +// length is longer than the number of available blocks. +func (tc *testChain) shorten(length int) *testChain { + if length > tc.len() { + panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len())) + } + return tc.copy(length) +} + +func (tc *testChain) copy(newlen int) *testChain { + cpy := &testChain{ + genesis: tc.genesis, + headerm: make(map[common.Hash]*types.Header, newlen), + blockm: make(map[common.Hash]*types.Block, newlen), + receiptm: make(map[common.Hash][]*types.Receipt, newlen), + tdm: make(map[common.Hash]*big.Int, newlen), + } + for i := 0; i < len(tc.chain) && i < newlen; i++ { + hash := tc.chain[i] + cpy.chain = append(cpy.chain, tc.chain[i]) + cpy.tdm[hash] = tc.tdm[hash] + cpy.blockm[hash] = tc.blockm[hash] + cpy.headerm[hash] = tc.headerm[hash] + cpy.receiptm[hash] = tc.receiptm[hash] + } + return cpy +} + +// generate creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 22th block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { + // start := time.Now() + // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }() + + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // If a heavy chain is requested, delay blocks to raise difficulty + if heavy { + block.OffsetTime(-1) + } + // Include transactions to the miner to make blocks more interesting. + if parent == tc.genesis && i%22 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // if the block number is a multiple of 5, add a bonus uncle to the block + if i > 0 && i%5 == 0 { + block.AddUncle(&types.Header{ + ParentHash: block.PrevBlock(i - 1).Hash(), + Number: big.NewInt(block.Number().Int64() - 1), + }) + } + }) + + // Convert the block-chain into a hash-chain and header/block maps + td := new(big.Int).Set(tc.td(parent.Hash())) + for i, b := range blocks { + td := td.Add(td, b.Difficulty()) + hash := b.Hash() + tc.chain = append(tc.chain, hash) + tc.blockm[hash] = b + tc.headerm[hash] = b.Header() + tc.receiptm[hash] = receipts[i] + tc.tdm[hash] = new(big.Int).Set(td) + } +} + +// len returns the total number of blocks in the chain. +func (tc *testChain) len() int { + return len(tc.chain) +} + +// headBlock returns the head of the chain. +func (tc *testChain) headBlock() *types.Block { + return tc.blockm[tc.chain[len(tc.chain)-1]] +} + +// td returns the total difficulty of the given block. +func (tc *testChain) td(hash common.Hash) *big.Int { + return tc.tdm[hash] +} + +// headersByHash returns headers in ascending order from the given hash. +func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header { + num, _ := tc.hashToNumber(origin) + return tc.headersByNumber(num, amount, skip) +} + +// headersByNumber returns headers in ascending order from the given number. +func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header { + result := make([]*types.Header, 0, amount) + for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 { + if header, ok := tc.headerm[tc.chain[int(num)]]; ok { + result = append(result, header) + } + } + return result +} + +// receipts returns the receipts of the given block hashes. +func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt { + results := make([][]*types.Receipt, 0, len(hashes)) + for _, hash := range hashes { + if receipt, ok := tc.receiptm[hash]; ok { + results = append(results, receipt) + } + } + return results +} + +// bodies returns the block bodies of the given block hashes. +func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) { + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) + for _, hash := range hashes { + if block, ok := tc.blockm[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) + } + } + return transactions, uncles +} + +func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) { + for num, hash := range tc.chain { + if hash == target { + return uint64(num), true + } + } + return 0, false +} From 81533deae5ee4a7ec08842e2b6647f3affde5a71 Mon Sep 17 00:00:00 2001 From: Mark Vujevits Date: Wed, 7 Nov 2018 20:33:36 +0100 Subject: [PATCH 049/100] swarm/network: light nodes are not dialed, saved and requested from (#17975) * RequestFromPeers does not use peers marked as lightnode * fix warning about variable name * write tests for RequestFromPeers * lightnodes should be omitted from the addressbook * resolve pr comments regarding logging, formatting and comments * resolve pr comments regarding comments and added a missing newline * add assertions to check peers in live connections --- swarm/network/kademlia.go | 20 ++++--- swarm/network/kademlia_test.go | 58 +++++++++++++++++-- swarm/network/stream/delivery.go | 5 +- swarm/network/stream/delivery_test.go | 83 +++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 13 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 55a0c6f13d..cd94741be3 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -261,7 +261,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // found among live peers, do nothing return v }) - if ins { + if ins && !p.BzzPeer.LightNode { a := newEntry(p.BzzAddr) a.conn = p // insert new online peer into addrs @@ -329,14 +329,18 @@ func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() var del bool - k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { - // v cannot be nil, must check otherwise we overwrite entry - if v == nil { - panic(fmt.Sprintf("connected peer not found %v", p)) - } + if !p.BzzPeer.LightNode { + k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { + // v cannot be nil, must check otherwise we overwrite entry + if v == nil { + panic(fmt.Sprintf("connected peer not found %v", p)) + } + del = true + return newEntry(p.BzzAddr) + }) + } else { del = true - return newEntry(p.BzzAddr) - }) + } if del { k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val { diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 903c8dbdac..d2e051f457 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -46,19 +46,19 @@ func newTestKademlia(b string) *Kademlia { return NewKademlia(base, params) } -func newTestKadPeer(k *Kademlia, s string) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k) +func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k) } func On(k *Kademlia, ons ...string) { for _, s := range ons { - k.On(newTestKadPeer(k, s)) + k.On(newTestKadPeer(k, s, false)) } } func Off(k *Kademlia, offs ...string) { for _, s := range offs { - k.Off(newTestKadPeer(k, s)) + k.Off(newTestKadPeer(k, s, false)) } } @@ -254,6 +254,56 @@ func TestSuggestPeerFindPeers(t *testing.T) { } +// a node should stay in the address book if it's removed from the kademlia +func TestOffEffectingAddressBookNormalNode(t *testing.T) { + k := newTestKademlia("00000000") + // peer added to kademlia + k.On(newTestKadPeer(k, "01000000", false)) + // peer should be in the address book + if k.addrs.Size() != 1 { + t.Fatal("known peer addresses should contain 1 entry") + } + // peer should be among live connections + if k.conns.Size() != 1 { + t.Fatal("live peers should contain 1 entry") + } + // remove peer from kademlia + k.Off(newTestKadPeer(k, "01000000", false)) + // peer should be in the address book + if k.addrs.Size() != 1 { + t.Fatal("known peer addresses should contain 1 entry") + } + // peer should not be among live connections + if k.conns.Size() != 0 { + t.Fatal("live peers should contain 0 entry") + } +} + +// a light node should not be in the address book +func TestOffEffectingAddressBookLightNode(t *testing.T) { + k := newTestKademlia("00000000") + // light node peer added to kademlia + k.On(newTestKadPeer(k, "01000000", true)) + // peer should not be in the address book + if k.addrs.Size() != 0 { + t.Fatal("known peer addresses should contain 0 entry") + } + // peer should be among live connections + if k.conns.Size() != 1 { + t.Fatal("live peers should contain 1 entry") + } + // remove peer from kademlia + k.Off(newTestKadPeer(k, "01000000", true)) + // peer should not be in the address book + if k.addrs.Size() != 0 { + t.Fatal("known peer addresses should contain 0 entry") + } + // peer should not be among live connections + if k.conns.Size() != 0 { + t.Fatal("live peers should contain 0 entry") + } +} + func TestSuggestPeerRetries(t *testing.T) { k := newTestKademlia("00000000") k.RetryInterval = int64(300 * time.Millisecond) // cycle diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 9092ffe3ed..0109fbdef2 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -245,7 +245,10 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( } else { d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int, nn bool) bool { id := p.ID() - // TODO: skip light nodes that do not accept retrieve requests + if p.LightNode { + // skip light nodes + return true + } if req.SkipPeer(id.String()) { log.Trace("Delivery.RequestFromPeers: skip peer", "peer id", id) return true diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 29b4f2f69a..c77682e0e1 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -29,10 +29,13 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/network" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" @@ -274,6 +277,86 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } } +// if there is one peer in the Kademlia, RequestFromPeers should return it +func TestRequestFromPeers(t *testing.T) { + dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") + + addr := network.RandomAddr() + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + delivery := NewDelivery(to, nil) + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + peer := network.NewPeer(&network.BzzPeer{ + BzzAddr: network.RandomAddr(), + LightNode: false, + Peer: protocolsPeer, + }, to) + to.On(peer) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + + // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished + sp := &Peer{ + Peer: protocolsPeer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: r, + } + r.setPeer(sp) + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + ctx := context.Background() + id, _, err := delivery.RequestFromPeers(ctx, req) + + if err != nil { + t.Fatal(err) + } + if *id != dummyPeerID { + t.Fatalf("Expected an id, got %v", id) + } +} + +// RequestFromPeers should not return light nodes +func TestRequestFromPeersWithLightNode(t *testing.T) { + dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") + + addr := network.RandomAddr() + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + delivery := NewDelivery(to, nil) + + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + // setting up a lightnode + peer := network.NewPeer(&network.BzzPeer{ + BzzAddr: network.RandomAddr(), + LightNode: true, + Peer: protocolsPeer, + }, to) + to.On(peer) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished + sp := &Peer{ + Peer: protocolsPeer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: r, + } + r.setPeer(sp) + + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + + ctx := context.Background() + // making a request which should return with "no peer found" + _, _, err := delivery.RequestFromPeers(ctx, req) + + expectedError := "no peer found" + if err.Error() != expectedError { + t.Fatalf("expected '%v', got %v", expectedError, err) + } +} + func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ Retrieval: RetrievalDisabled, From cf3b187bdef59078ba6570a2f5ee046ab87bcefd Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 7 Nov 2018 20:39:08 +0100 Subject: [PATCH 050/100] swarm, cmd/swarm: address ineffectual assignments (#18048) * swarm, cmd/swarm: address ineffectual assignments * swarm/network: remove unused vars from testHandshake * swarm/storage/feed: revert cursor changes --- cmd/swarm/access.go | 6 ++++++ cmd/swarm/fs_test.go | 3 +++ cmd/swarm/upload_test.go | 3 +-- swarm/api/act.go | 3 +++ swarm/api/client/client_test.go | 3 +++ swarm/api/filesystem.go | 4 ++++ swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 5 ++++- swarm/api/manifest.go | 1 - swarm/network/hive_test.go | 6 ++++++ swarm/network/protocol_test.go | 12 +----------- swarm/network/simulation/bucket_test.go | 4 ++-- swarm/network/stream/syncer_test.go | 3 +++ swarm/pss/client/client_test.go | 6 ++++++ swarm/pss/notify/notify_test.go | 6 ++++++ swarm/pss/protocol_test.go | 6 ++++++ swarm/state/dbstore.go | 2 +- swarm/state/dbstore_test.go | 3 +++ swarm/state/inmemorystore.go | 2 +- swarm/storage/types.go | 5 +---- 20 files changed, 61 insertions(+), 24 deletions(-) diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index 629781edd8..072541b659 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -114,6 +114,9 @@ func accessNewPass(ctx *cli.Context) { utils.Fatalf("error getting session key: %v", err) } m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if err != nil { + utils.Fatalf("had an error generating the manifest: %v", err) + } if dryRun { err = printManifests(m, nil) if err != nil { @@ -147,6 +150,9 @@ func accessNewPK(ctx *cli.Context) { utils.Fatalf("error getting session key: %v", err) } m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae) + if err != nil { + utils.Fatalf("had an error generating the manifest: %v", err) + } if dryRun { err = printManifests(m, nil) if err != nil { diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index 4f38b094bb..3b722515ee 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -80,6 +80,9 @@ func TestCLISwarmFs(t *testing.T) { t.Fatal(err) } dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir") + if err != nil { + t.Fatal(err) + } dummyContent := "somerandomtestcontentthatshouldbeasserted" dirs := []string{ diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 0ac2456a51..ba4463e8bf 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -243,8 +243,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } defer os.RemoveAll(tmpDownload) bzzLocator := "bzz:/" + hash - flagss := []string{} - flagss = []string{ + flagss := []string{ "--bzzapi", cluster.Nodes[0].URL, "down", "--recursive", diff --git a/swarm/api/act.go b/swarm/api/act.go index 52d9098271..e54369f9a8 100644 --- a/swarm/api/act.go +++ b/swarm/api/act.go @@ -458,6 +458,9 @@ func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees return nil, nil, nil, err } sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt) + if err != nil { + return nil, nil, nil, err + } hasher := sha3.NewKeccak256() hasher.Write(append(sessionKey, 0)) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 03c6cbb28c..c30d699110 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -457,6 +457,9 @@ func TestClientCreateUpdateFeed(t *testing.T) { } feedManifestHash, err := client.CreateFeedWithManifest(createRequest) + if err != nil { + t.Fatal(err) + } correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882" if feedManifestHash != correctManifestAddrHex { diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 43695efc1e..266ef71bec 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -122,6 +122,10 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error var wait func(context.Context) error ctx := context.TODO() hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt) + if err != nil { + errors[i] = err + return + } if hash != nil { list[i].Hash = hash.Hex() } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 803b789878..3c6735a73e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -366,7 +366,7 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api } var size int64 - var reader io.Reader = part + var reader io.Reader if contentLength := part.Header.Get("Content-Length"); contentLength != "" { size, err = strconv.ParseInt(contentLength, 10, 64) if err != nil { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 159c8a159f..04d0e045aa 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -263,7 +263,7 @@ func TestBzzFeed(t *testing.T) { if resp.StatusCode == http.StatusOK { t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK") } - b, err = ioutil.ReadAll(resp.Body) + _, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } @@ -491,6 +491,9 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } defer resp.Body.Close() respbody, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Error while reading response body: %v", err) + } if string(respbody) != testmanifest[v] { isexpectedfailrequest := false diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 7c4cc88e47..890ed88bd4 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -557,7 +557,6 @@ func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manif if path != entry.Path { return nil, 0 } - pos = epl } } return nil, 0 diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 059c3dc96d..56adc5a8e9 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -70,6 +70,9 @@ func TestHiveStatePersistance(t *testing.T) { defer os.RemoveAll(dir) store, err := state.NewDBStore(dir) //start the hive with an empty dbstore + if err != nil { + t.Fatal(err) + } params := NewHiveParams() s, pp := newHiveTester(t, params, 5, store) @@ -90,6 +93,9 @@ func TestHiveStatePersistance(t *testing.T) { store.Close() persistedStore, err := state.NewDBStore(dir) //start the hive with an empty dbstore + if err != nil { + t.Fatal(err) + } s1, pp := newHiveTester(t, params, 1, persistedStore) diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index cdf370f359..f0d2666288 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -153,17 +153,7 @@ func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) * // should test handshakes in one exchange? parallelisation func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptest.Disconnect) error { - var peers []enode.ID - id := rhs.Addr.ID() - if len(disconnects) > 0 { - for _, d := range disconnects { - peers = append(peers, d.Peer) - } - } else { - peers = []enode.ID{id} - } - - if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, id)...); err != nil { + if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, rhs.Addr.ID())...); err != nil { return err } diff --git a/swarm/network/simulation/bucket_test.go b/swarm/network/simulation/bucket_test.go index 461d99825c..69df19bfe4 100644 --- a/swarm/network/simulation/bucket_test.go +++ b/swarm/network/simulation/bucket_test.go @@ -94,7 +94,7 @@ func TestServiceBucket(t *testing.T) { t.Fatalf("expected %q, got %q", customValue, s) } - v, ok = sim.NodeItem(id2, customKey) + _, ok = sim.NodeItem(id2, customKey) if ok { t.Fatal("bucket item should not be found") } @@ -119,7 +119,7 @@ func TestServiceBucket(t *testing.T) { t.Fatalf("expected %q, got %q", testValue+id1.String(), s) } - v, ok = items[id2] + _, ok = items[id2] if ok { t.Errorf("node 2 item should not be found") } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 113807b98a..b0e35b0db3 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -62,6 +62,9 @@ func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network params.Init(datadir) params.BaseKey = addr.Over() lstore, err = storage.NewLocalStore(params, mockStore) + if err != nil { + return nil, "", err + } return lstore, datadir, nil } diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index cfef3c7947..8f2f0e8059 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -252,7 +252,13 @@ func newServices() adapters.Services { ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) + if err != nil { + return nil, err + } privkey, err := w.GetPrivateKey(keys) + if err != nil { + return nil, err + } psparams := pss.NewPssParams().WithPrivateKey(privkey) pskad := kademlia(ctx.Config.ID) ps, err := pss.NewPss(pskad, psparams) diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index 675b41ada2..d4d383a6ba 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -223,7 +223,13 @@ func newServices(allowRaw bool) adapters.Services { ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) + if err != nil { + return nil, err + } privkey, err := w.GetPrivateKey(keys) + if err != nil { + return nil, err + } pssp := pss.NewPssParams().WithPrivateKey(privkey) pssp.MsgTTL = time.Second * 30 pssp.AllowRaw = allowRaw diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index f4209fea59..4ef3e90a04 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -93,11 +93,17 @@ func testProtocol(t *testing.T) { lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + if err != nil { + t.Fatal(err) + } defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + if err != nil { + t.Fatal(err) + } defer rsub.Unsubscribe() // set reciprocal public keys diff --git a/swarm/state/dbstore.go b/swarm/state/dbstore.go index 5e5c172b25..b0aa92e272 100644 --- a/swarm/state/dbstore.go +++ b/swarm/state/dbstore.go @@ -69,7 +69,7 @@ func (s *DBStore) Get(key string, i interface{}) (err error) { // Put stores an object that implements Binary for a specific key. func (s *DBStore) Put(key string, i interface{}) (err error) { - bytes := []byte{} + var bytes []byte marshaler, ok := i.(encoding.BinaryMarshaler) if !ok { diff --git a/swarm/state/dbstore_test.go b/swarm/state/dbstore_test.go index 6683e788f7..f7098956d0 100644 --- a/swarm/state/dbstore_test.go +++ b/swarm/state/dbstore_test.go @@ -112,6 +112,9 @@ func testPersistedStore(t *testing.T, store Store) { as := []string{} err = store.Get("key2", &as) + if err != nil { + t.Fatal(err) + } if len(as) != 3 { t.Fatalf("serialized array did not match expectation") diff --git a/swarm/state/inmemorystore.go b/swarm/state/inmemorystore.go index 1ca25404a1..3ba48592bc 100644 --- a/swarm/state/inmemorystore.go +++ b/swarm/state/inmemorystore.go @@ -59,7 +59,7 @@ func (s *InmemoryStore) Get(key string, i interface{}) (err error) { func (s *InmemoryStore) Put(key string, i interface{}) (err error) { s.mu.Lock() defer s.mu.Unlock() - bytes := []byte{} + var bytes []byte marshaler, ok := i.(encoding.BinaryMarshaler) if !ok { diff --git a/swarm/storage/types.go b/swarm/storage/types.go index ada86831fc..092843db0c 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -244,11 +244,8 @@ func GenerateRandomChunk(dataSize int64) Chunk { } func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { - if dataSize > ch.DefaultSize { - dataSize = ch.DefaultSize - } for i := 0; i < count; i++ { - ch := GenerateRandomChunk(ch.DefaultSize) + ch := GenerateRandomChunk(dataSize) chunks = append(chunks, ch) } return chunks From 503993c819f0a4e7e53f6739e90497106ebdfe09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Thu, 8 Nov 2018 13:11:20 +0200 Subject: [PATCH 051/100] p2p: use enode.ID type in metered connection (#17933) Change the type of the metered connection's id field from string to enode.ID. --- p2p/metrics.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/p2p/metrics.go b/p2p/metrics.go index 6a7c0bad33..d7873f39a5 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -74,7 +74,7 @@ const ( type MeteredPeerEvent struct { Type MeteredPeerEventType // Type of peer event IP net.IP // IP address of the peer - ID string // NodeID of the peer + ID enode.ID // NodeID of the peer Elapsed time.Duration // Time elapsed between the connection and the handshake/disconnection Ingress uint64 // Ingress count at the moment of the event Egress uint64 // Egress count at the moment of the event @@ -93,7 +93,7 @@ type meteredConn struct { connected time.Time // Connection time of the peer ip net.IP // IP address of the peer - id string // NodeID of the peer + id enode.ID // NodeID of the peer // trafficMetered denotes if the peer is registered in the traffic registries. // Its value is true if the metered peer count doesn't reach the limit in the @@ -160,8 +160,7 @@ func (c *meteredConn) Write(b []byte) (n int, err error) { // handshakeDone is called when a peer handshake is done. Registers the peer to // the ingress and the egress traffic registries using the peer's IP and node ID, // also emits connect event. -func (c *meteredConn) handshakeDone(nodeID enode.ID) { - id := nodeID.String() +func (c *meteredConn) handshakeDone(id enode.ID) { if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit { // Don't register the peer in the traffic registries. atomic.AddInt32(&meteredPeerCount, -1) @@ -170,7 +169,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) { c.lock.Unlock() log.Warn("Metered peer count reached the limit") } else { - key := fmt.Sprintf("%s/%s", c.ip, id) + key := fmt.Sprintf("%s/%s", c.ip, id.String()) c.lock.Lock() c.id, c.trafficMetered = id, true c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry) @@ -190,7 +189,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) { func (c *meteredConn) Close() error { err := c.Conn.Close() c.lock.RLock() - if c.id == "" { + if c.id == (enode.ID{}) { // If the peer disconnects before the handshake. c.lock.RUnlock() meteredPeerFeed.Send(MeteredPeerEvent{ From 968f6019d00e349e988f720cdaa065def85ea72f Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 8 Nov 2018 19:17:01 +0800 Subject: [PATCH 052/100] event, event/filter: minor code cleanup (#18061) --- event/event_test.go | 8 ++++---- event/filter/generic_filter.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/event/event_test.go b/event/event_test.go index a12945a471..2be357ba2d 100644 --- a/event/event_test.go +++ b/event/event_test.go @@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) { } } -func emptySubscriber(mux *TypeMux, types ...interface{}) { +func emptySubscriber(mux *TypeMux) { s := mux.Subscribe(testEvent(0)) go func() { for range s.Chan() { @@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) { func BenchmarkPostConcurrent(b *testing.B) { var mux = new(TypeMux) defer mux.Stop() - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) + emptySubscriber(mux) + emptySubscriber(mux) + emptySubscriber(mux) var wg sync.WaitGroup poster := func() { diff --git a/event/filter/generic_filter.go b/event/filter/generic_filter.go index d679b8bfa8..467bf01bec 100644 --- a/event/filter/generic_filter.go +++ b/event/filter/generic_filter.go @@ -25,7 +25,7 @@ type Generic struct { // self = registered, f = incoming func (self Generic) Compare(f Filter) bool { - var strMatch, dataMatch = true, true + var strMatch = true filter := f.(Generic) if (len(self.Str1) > 0 && filter.Str1 != self.Str1) || @@ -40,7 +40,7 @@ func (self Generic) Compare(f Filter) bool { } } - return strMatch && dataMatch + return strMatch } func (self Generic) Trigger(data interface{}) { From c71e4fc4d5e5f97f57f1c52dced187da41b59aaf Mon Sep 17 00:00:00 2001 From: Liang Ma Date: Thu, 8 Nov 2018 11:22:28 +0000 Subject: [PATCH 053/100] p2p: fix comment typo (#18027) --- p2p/rlpx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/rlpx.go b/p2p/rlpx.go index a105720a44..22a27dd96a 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -151,7 +151,7 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake, } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the - // spec and we send it ourself if the posthanshake checks fail. + // spec and we send it ourself if the post-handshake checks fail. // We can't return the reason directly, though, because it is echoed // back otherwise. Wrap it in a string instead. var reason [1]DiscReason From 212bf266c55b3737da37e6588142d0c9f7c6e10f Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 8 Nov 2018 19:25:14 +0800 Subject: [PATCH 054/100] eth, p2p: fix comment typos (#18014) --- eth/config.go | 2 +- p2p/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/config.go b/eth/config.go index efbaafb6a2..e32c01a734 100644 --- a/eth/config.go +++ b/eth/config.go @@ -122,7 +122,7 @@ type Config struct { // Miscellaneous options DocRoot string `toml:"-"` - // Type of the EWASM interpreter ("" for detault) + // Type of the EWASM interpreter ("" for default) EWASMInterpreter string // Type of the EVM interpreter ("" for default) EVMInterpreter string diff --git a/p2p/server.go b/p2p/server.go index 38a881f7bd..6678608634 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -228,7 +228,7 @@ type transport interface { MsgReadWriter // transports must provide Close because we use MsgPipe in some of // the tests. Closing the actual network connection doesn't do - // anything in those tests because NsgPipe doesn't use it. + // anything in those tests because MsgPipe doesn't use it. close(err error) } From a5dc087845c90dad126bf203640ac6e876d7bbcb Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 8 Nov 2018 20:07:15 +0800 Subject: [PATCH 055/100] core/vm, eth/tracers: use pointer receiver for GetRefund (#18018) --- core/vm/logger_test.go | 2 +- eth/tracers/tracer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index cba7c7a0ef..2ea7535a79 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -46,7 +46,7 @@ type dummyStatedb struct { state.StateDB } -func (dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } func TestStoreCapture(t *testing.T) { var ( diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 52f29c83fc..a45a121159 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -48,7 +48,7 @@ type dummyStatedb struct { state.StateDB } -func (dummyStatedb) GetRefund() uint64 { return 1337 } +func (*dummyStatedb) GetRefund() uint64 { return 1337 } func runTrace(tracer *Tracer) (json.RawMessage, error) { env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) From 1064e3283db6e388db4fc353c8bd17b57d57fa42 Mon Sep 17 00:00:00 2001 From: JoranHonig Date: Thu, 8 Nov 2018 13:09:25 +0100 Subject: [PATCH 056/100] common/compiler: capture runtime code and source maps (#18020) --- common/compiler/solidity.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index f6e8d2e42a..b7c8ec5638 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -31,14 +31,15 @@ import ( var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) -// Contract contains information about a compiled contract, alongside its code. +// Contract contains information about a compiled contract, alongside its code and runtime code. type Contract struct { - Code string `json:"code"` - Info ContractInfo `json:"info"` + Code string `json:"code"` + RuntimeCode string `json:"runtime-code"` + Info ContractInfo `json:"info"` } // ContractInfo contains information about a compiled contract, including access -// to the ABI definition, user and developer docs, and metadata. +// to the ABI definition, source mapping, user and developer docs, and metadata. // // Depending on the source, language version, compiler version, and compiler // options will provide information about how the contract was compiled. @@ -48,6 +49,8 @@ type ContractInfo struct { LanguageVersion string `json:"languageVersion"` CompilerVersion string `json:"compilerVersion"` CompilerOptions string `json:"compilerOptions"` + SrcMap string `json:"srcMap"` + SrcMapRuntime string `json:"srcMapRuntime"` AbiDefinition interface{} `json:"abiDefinition"` UserDoc interface{} `json:"userDoc"` DeveloperDoc interface{} `json:"developerDoc"` @@ -63,14 +66,16 @@ type Solidity struct { // --combined-output format type solcOutput struct { Contracts map[string]struct { - Bin, Abi, Devdoc, Userdoc, Metadata string + BinRuntime string `json:"bin-runtime"` + SrcMapRuntime string `json:"srcmap-runtime"` + Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string } Version string } func (s *Solidity) makeArgs() []string { p := []string{ - "--combined-json", "bin,abi,userdoc,devdoc", + "--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc", "--optimize", // code optimizer switched on } if s.Major > 0 || s.Minor > 4 || s.Patch > 6 { @@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro // provided source, language and compiler version, and compiler options are all // passed through into the Contract structs. // -// The solc output is expected to contain ABI, user docs, and dev docs. +// The solc output is expected to contain ABI, source mapping, user docs, and dev docs. // // Returns an error if the JSON is malformed or missing data, or if the JSON // embedded within the JSON is malformed. @@ -184,13 +189,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin return nil, fmt.Errorf("solc: error reading dev doc: %v", err) } contracts[name] = &Contract{ - Code: "0x" + info.Bin, + Code: "0x" + info.Bin, + RuntimeCode: "0x" + info.BinRuntime, Info: ContractInfo{ Source: source, Language: "Solidity", LanguageVersion: languageVersion, CompilerVersion: compilerVersion, CompilerOptions: compilerOptions, + SrcMap: info.SrcMap, + SrcMapRuntime: info.SrcMapRuntime, AbiDefinition: abi, UserDoc: userdoc, DeveloperDoc: devdoc, From bd519ab8aec7c82b28fb2a7e7edc428d8a38e781 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Thu, 8 Nov 2018 04:18:38 -0800 Subject: [PATCH 057/100] internal/web3ext: add eth.getProof (#18052) --- internal/web3ext/web3ext.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index addf3c7660..a5f3196535 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -481,6 +481,12 @@ web3._extend({ params: 2, inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex] }), + new web3._extend.Method({ + name: 'getProof', + call: 'eth_getProof', + params: 3, + inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter] + }), ], properties: [ new web3._extend.Property({ From 9313fa63f959f8a5c3609c187120711a484a4c57 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 8 Nov 2018 13:26:29 +0100 Subject: [PATCH 058/100] event/filter: delete unused package (#18063) --- event/filter/filter.go | 95 ---------------------------------- event/filter/filter_test.go | 60 --------------------- event/filter/generic_filter.go | 48 ----------------- 3 files changed, 203 deletions(-) delete mode 100644 event/filter/filter.go delete mode 100644 event/filter/filter_test.go delete mode 100644 event/filter/generic_filter.go diff --git a/event/filter/filter.go b/event/filter/filter.go deleted file mode 100644 index a6fe46d6a0..0000000000 --- a/event/filter/filter.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package filter implements event filters. -package filter - -import "reflect" - -type Filter interface { - Compare(Filter) bool - Trigger(data interface{}) -} - -type FilterEvent struct { - filter Filter - data interface{} -} - -type Filters struct { - id int - watchers map[int]Filter - ch chan FilterEvent - - quit chan struct{} -} - -func New() *Filters { - return &Filters{ - ch: make(chan FilterEvent), - watchers: make(map[int]Filter), - quit: make(chan struct{}), - } -} - -func (f *Filters) Start() { - go f.loop() -} - -func (f *Filters) Stop() { - close(f.quit) -} - -func (f *Filters) Notify(filter Filter, data interface{}) { - f.ch <- FilterEvent{filter, data} -} - -func (f *Filters) Install(watcher Filter) int { - f.watchers[f.id] = watcher - f.id++ - - return f.id - 1 -} - -func (f *Filters) Uninstall(id int) { - delete(f.watchers, id) -} - -func (f *Filters) loop() { -out: - for { - select { - case <-f.quit: - break out - case event := <-f.ch: - for _, watcher := range f.watchers { - if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) { - if watcher.Compare(event.filter) { - watcher.Trigger(event.data) - } - } - } - } - } -} - -func (f *Filters) Match(a, b Filter) bool { - return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b) -} - -func (f *Filters) Get(i int) Filter { - return f.watchers[i] -} diff --git a/event/filter/filter_test.go b/event/filter/filter_test.go deleted file mode 100644 index dcc9112453..0000000000 --- a/event/filter/filter_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filter - -import ( - "testing" - "time" -) - -// Simple test to check if baseline matching/mismatching filtering works. -func TestFilters(t *testing.T) { - fm := New() - fm.Start() - - // Register two filters to catch posted data - first := make(chan struct{}) - fm.Install(Generic{ - Str1: "hello", - Fn: func(data interface{}) { - first <- struct{}{} - }, - }) - second := make(chan struct{}) - fm.Install(Generic{ - Str1: "hello1", - Str2: "hello", - Fn: func(data interface{}) { - second <- struct{}{} - }, - }) - // Post an event that should only match the first filter - fm.Notify(Generic{Str1: "hello"}, true) - fm.Stop() - - // Ensure only the mathcing filters fire - select { - case <-first: - case <-time.After(100 * time.Millisecond): - t.Error("matching filter timed out") - } - select { - case <-second: - t.Error("mismatching filter fired") - case <-time.After(100 * time.Millisecond): - } -} diff --git a/event/filter/generic_filter.go b/event/filter/generic_filter.go deleted file mode 100644 index 467bf01bec..0000000000 --- a/event/filter/generic_filter.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filter - -type Generic struct { - Str1, Str2, Str3 string - Data map[string]struct{} - - Fn func(data interface{}) -} - -// self = registered, f = incoming -func (self Generic) Compare(f Filter) bool { - var strMatch = true - - filter := f.(Generic) - if (len(self.Str1) > 0 && filter.Str1 != self.Str1) || - (len(self.Str2) > 0 && filter.Str2 != self.Str2) || - (len(self.Str3) > 0 && filter.Str3 != self.Str3) { - strMatch = false - } - - for k := range self.Data { - if _, ok := filter.Data[k]; !ok { - return false - } - } - - return strMatch -} - -func (self Generic) Trigger(data interface{}) { - self.Fn(data) -} From b16cc501a88eb7c36cfd8ee9be1c0c985ef4a7d9 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 8 Nov 2018 13:26:47 +0100 Subject: [PATCH 059/100] ethclient: include block hash from FilterQuery (#17996) ethereum/go-ethereum#16734 introduced BlockHash to the FilterQuery struct. However, ethclient was not updated to include BlockHash in the actual RPC request. --- ethclient/ethclient.go | 36 ++++++++--- ethclient/ethclient_test.go | 120 +++++++++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 11 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index b40837c8cc..f3163e19b3 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb // FilterLogs executes a filter query. func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { var result []types.Log - err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q)) + arg, err := toFilterArg(q) + if err != nil { + return nil, err + } + err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg) return result, err } // SubscribeFilterLogs subscribes to the results of a streaming filter query. func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q)) + arg, err := toFilterArg(q) + if err != nil { + return nil, err + } + return ec.c.EthSubscribe(ctx, ch, "logs", arg) } -func toFilterArg(q ethereum.FilterQuery) interface{} { +func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { arg := map[string]interface{}{ - "fromBlock": toBlockNumArg(q.FromBlock), - "toBlock": toBlockNumArg(q.ToBlock), - "address": q.Addresses, - "topics": q.Topics, + "address": q.Addresses, + "topics": q.Topics, } - if q.FromBlock == nil { - arg["fromBlock"] = "0x0" + if q.BlockHash != nil { + arg["blockHash"] = *q.BlockHash + if q.FromBlock != nil || q.ToBlock != nil { + return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + } + } else { + if q.FromBlock == nil { + arg["fromBlock"] = "0x0" + } else { + arg["fromBlock"] = toBlockNumArg(q.FromBlock) + } + arg["toBlock"] = toBlockNumArg(q.ToBlock) } - return arg + return arg, nil } // Pending State diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 178eb2be98..3e8bf974c2 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -16,7 +16,15 @@ package ethclient -import "github.com/ethereum/go-ethereum" +import ( + "fmt" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" +) // Verify that Client implements the ethereum interfaces. var ( @@ -32,3 +40,113 @@ var ( // _ = ethereum.PendingStateEventer(&Client{}) _ = ethereum.PendingContractCaller(&Client{}) ) + +func TestToFilterArg(t *testing.T) { + blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + addresses := []common.Address{ + common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"), + } + blockHash := common.HexToHash( + "0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb", + ) + + for _, testCase := range []struct { + name string + input ethereum.FilterQuery + output interface{} + err error + }{ + { + "without BlockHash", + ethereum.FilterQuery{ + Addresses: addresses, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "fromBlock": "0x1", + "toBlock": "0x2", + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with nil fromBlock and nil toBlock", + ethereum.FilterQuery{ + Addresses: addresses, + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "fromBlock": "0x0", + "toBlock": "latest", + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with blockhash", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "blockHash": blockHash, + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with blockhash and from block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + FromBlock: big.NewInt(1), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + { + "with blockhash and to block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + ToBlock: big.NewInt(1), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + { + "with blockhash and both from / to block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + output, err := toFilterArg(testCase.input) + if (testCase.err == nil) != (err == nil) { + t.Fatalf("expected error %v but got %v", testCase.err, err) + } + if testCase.err != nil { + if testCase.err.Error() != err.Error() { + t.Fatalf("expected error %v but got %v", testCase.err, err) + } + } else if !reflect.DeepEqual(testCase.output, output) { + t.Fatalf("expected filter arg %v but got %v", testCase.output, output) + } + }) + } +} From 144c1c6c52456808428e2b69dbe5c4ebfc3606ca Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 8 Nov 2018 16:08:57 +0100 Subject: [PATCH 060/100] consensus: extend getWork API with block number (#18038) --- consensus/ethash/api.go | 11 ++++++----- consensus/ethash/ethash.go | 2 +- consensus/ethash/ethash_test.go | 2 +- consensus/ethash/sealer.go | 5 ++++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go index a04ea235d9..4d8eed4161 100644 --- a/consensus/ethash/api.go +++ b/consensus/ethash/api.go @@ -37,27 +37,28 @@ type API struct { // result[0] - 32 bytes hex encoded current block header pow-hash // result[1] - 32 bytes hex encoded seed hash used for DAG // result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty -func (api *API) GetWork() ([3]string, error) { +// result[3] - hex encoded block number +func (api *API) GetWork() ([4]string, error) { if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { - return [3]string{}, errors.New("not supported") + return [4]string{}, errors.New("not supported") } var ( - workCh = make(chan [3]string, 1) + workCh = make(chan [4]string, 1) errc = make(chan error, 1) ) select { case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}: case <-api.ethash.exitCh: - return [3]string{}, errEthashStopped + return [4]string{}, errEthashStopped } select { case work := <-workCh: return work, nil case err := <-errc: - return [3]string{}, err + return [4]string{}, err } } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d124cb1e20..78892e1da8 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -432,7 +432,7 @@ type hashrate struct { // sealWork wraps a seal work package for remote sealer. type sealWork struct { errc chan error - res chan [3]string + res chan [4]string } // Ethash is a consensus engine based on proof-of-work implementing the ethash diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 8eded2ca81..90cb6470f7 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) { ethash.Seal(nil, block, results, nil) var ( - work [3]string + work [4]string err error ) if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 06c98a7811..3a0919ca99 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -30,6 +30,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { results chan<- *types.Block currentBlock *types.Block - currentWork [3]string + currentWork [4]string notifyTransport = &http.Transport{} notifyClient = &http.Client{ @@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { // result[0], 32 bytes hex encoded current block header pow-hash // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty + // result[3], hex encoded block number makeWork := func(block *types.Block) { hash := ethash.SealHash(block.Header()) currentWork[0] = hash.Hex() currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() + currentWork[3] = hexutil.EncodeBig(block.Number()) // Trace the seal work fetched by remote sealer. currentBlock = block From 870efeef01ad45f1e06bbe4479e5afa9986f3518 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 8 Nov 2018 21:37:19 +0100 Subject: [PATCH 061/100] core/state: remove lock (#18065) The lock in StateDB is useless. It's only held in Copy, but Copy is safe for concurrent use because all it does is read. --- core/state/statedb.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index f0d7cdb6eb..76e67d839d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -22,7 +22,6 @@ import ( "fmt" "math/big" "sort" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -87,8 +86,6 @@ type StateDB struct { journal *journal validRevisions []revision nextRevisionId int - - lock sync.Mutex } // Create a new state from a given trie. @@ -496,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common // Copy creates a deep, independent copy of the state. // Snapshots of the copied state cannot be applied to the copy. func (self *StateDB) Copy() *StateDB { - self.lock.Lock() - defer self.lock.Unlock() - // Copy all the basic fields, initialize the memory ones state := &StateDB{ db: self.db, From f574c4e74b12d275d24b209aac8dd98ed9b9bb52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Fri, 9 Nov 2018 11:20:51 +0200 Subject: [PATCH 062/100] metrics, p2p: add ephemeral registry (#18067) * metrics, p2p: add ephemeral registry * metrics: fix linter issue --- metrics/registry.go | 5 ++++- p2p/metrics.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metrics/registry.go b/metrics/registry.go index cc34c9dfd2..c1cf7906ce 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() { r.underlying.UnregisterAll() } -var DefaultRegistry Registry = NewRegistry() +var ( + DefaultRegistry = NewRegistry() + EphemeralRegistry = NewRegistry() +) // Call the given function for each registered metric. func Each(f func(string, interface{})) { diff --git a/p2p/metrics.go b/p2p/metrics.go index d7873f39a5..8df82bb074 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -47,8 +47,8 @@ var ( egressConnectMeter = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections egressTrafficMeter = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil) // Meter metering the cumulative egress traffic - PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress - PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress + PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress + PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress meteredPeerFeed event.Feed // Event feed for peer metrics meteredPeerCount int32 // Actually stored peer connection count From 1ff152f3a43e4adf030ac61eb5d8da345554fc5a Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Fri, 9 Nov 2018 18:51:07 +0800 Subject: [PATCH 063/100] rawdb: remove unused parameter for WritePreimages func (#18059) * rawdb: remove unused parameter for WritePreimages func and modify a spelling mistake * rawdb: update the doc for function WritePreimages --- cmd/utils/cmd.go | 4 ++-- core/blockchain.go | 2 +- core/rawdb/accessors_metadata.go | 5 ++--- core/rawdb/schema.go | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 58d72f32ba..f23aa5775b 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { // Accumulate the preimages and flush when enough ws gathered preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob) if len(preimages) > 1024 { - rawdb.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, preimages) preimages = make(map[common.Hash][]byte) } } // Flush the last batch preimage data if len(preimages) > 0 { - rawdb.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, preimages) } return nil } diff --git a/core/blockchain.go b/core/blockchain.go index f4a818f4c0..1b0c26ae25 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1002,7 +1002,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Write the positional metadata for transaction/receipt lookups and preimages rawdb.WriteTxLookupEntries(batch, block) - rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()) + rawdb.WritePreimages(batch, state.Preimages()) status = CanonStatTy } else { diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 514328e87b..3b6e6548d3 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { return data } -// WritePreimages writes the provided set of preimages to the database. `number` is the -// current block number, and is used for debug messages only. -func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { +// WritePreimages writes the provided set of preimages to the database. +func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) { for hash, preimage := range preimages { if err := db.Put(preimageKey(hash), preimage); err != nil { log.Crit("Failed to store trie preimage", "err", err) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index ef597ef309..8a9921ef40 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -35,7 +35,7 @@ var ( // headBlockKey tracks the latest know full block's hash. headBlockKey = []byte("LastBlock") - // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync. + // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") // fastTrieProgressKey tracks the number of trie entries imported during fast sync. From a0876f7433f63276a3d8d4e099b261fd16aada40 Mon Sep 17 00:00:00 2001 From: Andrew Chiw Date: Mon, 12 Nov 2018 13:04:13 +0100 Subject: [PATCH 064/100] Imply that SwarmApiFlag is the API endpoint to connect to, not to listen on (#18071) --- cmd/swarm/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 7e891f3022..0dedca674b 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -88,7 +88,7 @@ var ( } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", - Usage: "Swarm HTTP endpoint", + Usage: "Specifies the Swarm HTTP endpoint to connect to", Value: "http://127.0.0.1:8500", } SwarmRecursiveFlag = cli.BoolFlag{ From 201a0bf18181da8d783f6e7adf3ceaccd159eb73 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 12 Nov 2018 14:57:17 +0100 Subject: [PATCH 065/100] p2p/simulations, swarm/network: Custom services in snapshot (#17991) * p2p/simulations: Add custom services to simnodes + remove sim down conn objs * p2p/simulation, swarm/network: Add selective services to discovery sim * p2p/simulations, swarm/network: Remove useless comments * p2p/simulations, swarm/network: Clean up mess from rebase * p2p/simulation: Add sleep to prevent connect flakiness in http test * p2p/simulations: added concurrent goroutines to prevent sleeps on simulation connect/disconnect * p2p/simulations, swarm/network/simulations: address pr comments * reinstated dummy service * fixed http snapshot test --- p2p/simulations/http_test.go | 55 ++++++++++++++++++- p2p/simulations/network.go | 44 ++++++++++++++- .../simulations/discovery/discovery_test.go | 31 +++++++++-- 3 files changed, 118 insertions(+), 12 deletions(-) diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 7bdbad1b53..d9513caaa0 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -18,6 +18,7 @@ package simulations import ( "context" + "flag" "fmt" "math/rand" "net/http/httptest" @@ -28,13 +29,26 @@ import ( "time" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" + colorable "github.com/mattn/go-colorable" ) +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + // testService implements the node.Service interface and provides protocols // and APIs which are useful for testing nodes in a simulation network type testService struct { @@ -584,9 +598,26 @@ func TestHTTPNodeRPC(t *testing.T) { // TestHTTPSnapshot tests creating and loading network snapshots func TestHTTPSnapshot(t *testing.T) { // start the server - _, s := testHTTPServer(t) + network, s := testHTTPServer(t) defer s.Close() + var eventsDone = make(chan struct{}) + count := 1 + eventsDoneChan := make(chan *Event) + eventSub := network.Events().Subscribe(eventsDoneChan) + go func() { + defer eventSub.Unsubscribe() + for event := range eventsDoneChan { + if event.Type == EventTypeConn && !event.Control { + count-- + if count == 0 { + eventsDone <- struct{}{} + return + } + } + } + }() + // create a two-node network client := NewClient(s.URL) nodeCount := 2 @@ -620,7 +651,7 @@ func TestHTTPSnapshot(t *testing.T) { } states[i] = state } - + <-eventsDone // create a snapshot snap, err := client.CreateSnapshot() if err != nil { @@ -634,9 +665,23 @@ func TestHTTPSnapshot(t *testing.T) { } // create another network - _, s = testHTTPServer(t) + network2, s := testHTTPServer(t) defer s.Close() client = NewClient(s.URL) + count = 1 + eventSub = network2.Events().Subscribe(eventsDoneChan) + go func() { + defer eventSub.Unsubscribe() + for event := range eventsDoneChan { + if event.Type == EventTypeConn && !event.Control { + count-- + if count == 0 { + eventsDone <- struct{}{} + return + } + } + } + }() // subscribe to events so we can check them later events := make(chan *Event, 100) @@ -651,6 +696,7 @@ func TestHTTPSnapshot(t *testing.T) { if err := client.LoadSnapshot(snap); err != nil { t.Fatalf("error loading snapshot: %s", err) } + <-eventsDone // check the nodes and connection exists net, err := client.GetNetwork() @@ -676,6 +722,9 @@ func TestHTTPSnapshot(t *testing.T) { if conn.Other.String() != nodes[1].ID { t.Fatalf("expected connection to have other=%q, got other=%q", nodes[1].ID, conn.Other) } + if !conn.Up { + t.Fatal("should be up") + } // check the node states were restored for i, node := range nodes { diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 200015ff39..92ccfde817 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -644,11 +644,18 @@ type NodeSnapshot struct { // Snapshot creates a network snapshot func (net *Network) Snapshot() (*Snapshot, error) { + return net.snapshot(nil, nil) +} + +func (net *Network) SnapshotWithServices(addServices []string, removeServices []string) (*Snapshot, error) { + return net.snapshot(addServices, removeServices) +} + +func (net *Network) snapshot(addServices []string, removeServices []string) (*Snapshot, error) { net.lock.Lock() defer net.lock.Unlock() snap := &Snapshot{ Nodes: make([]NodeSnapshot, len(net.Nodes)), - Conns: make([]Conn, len(net.Conns)), } for i, node := range net.Nodes { snap.Nodes[i] = NodeSnapshot{Node: *node} @@ -660,9 +667,40 @@ func (net *Network) Snapshot() (*Snapshot, error) { return nil, err } snap.Nodes[i].Snapshots = snapshots + for _, addSvc := range addServices { + haveSvc := false + for _, svc := range snap.Nodes[i].Node.Config.Services { + if svc == addSvc { + haveSvc = true + break + } + } + if !haveSvc { + snap.Nodes[i].Node.Config.Services = append(snap.Nodes[i].Node.Config.Services, addSvc) + } + } + if len(removeServices) > 0 { + var cleanedServices []string + for _, svc := range snap.Nodes[i].Node.Config.Services { + haveSvc := false + for _, rmSvc := range removeServices { + if rmSvc == svc { + haveSvc = true + break + } + } + if !haveSvc { + cleanedServices = append(cleanedServices, svc) + } + + } + snap.Nodes[i].Node.Config.Services = cleanedServices + } } - for i, conn := range net.Conns { - snap.Conns[i] = *conn + for _, conn := range net.Conns { + if conn.Up { + snap.Conns = append(snap.Conns, *conn) + } } return snap, nil } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 3c3affe58c..cd5456b73e 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -85,11 +85,12 @@ func getDbStore(nodeID string) (*state.DBStore, error) { } var ( - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") - initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") - snapshotFile = flag.String("snapshot", "", "create snapshot") - loglevel = flag.Int("loglevel", 3, "verbosity of logs") - rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") + nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") + snapshotFile = flag.String("snapshot", "", "path to create snapshot file in") + loglevel = flag.Int("loglevel", 3, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") + serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery") ) func init() { @@ -306,7 +307,25 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul } if *snapshotFile != "" { - snap, err := net.Snapshot() + var err error + var snap *simulations.Snapshot + if len(*serviceOverride) > 0 { + var addServices []string + var removeServices []string + for _, osvc := range strings.Split(*serviceOverride, ",") { + if strings.Index(osvc, "+") == 0 { + addServices = append(addServices, osvc[1:]) + } else if strings.Index(osvc, "-") == 0 { + removeServices = append(removeServices, osvc[1:]) + } else { + panic("stick to the rules, you know what they are") + } + } + snap, err = net.SnapshotWithServices(addServices, removeServices) + } else { + snap, err = net.Snapshot() + } + if err != nil { return nil, errors.New("no shapshot dude") } From 1212c7b844e3ef13cbb5476b088eae27782535b4 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 13 Nov 2018 00:06:34 +0800 Subject: [PATCH 066/100] core: fix default trie cache limit (#17860) --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 1b0c26ae25..26ac75b8cf 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -140,7 +140,7 @@ type BlockChain struct { func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, + TrieNodeLimit: 256, TrieTimeLimit: 5 * time.Minute, } } From 8080265f3f591d33e127a924724a8bfe5ced6475 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Tue, 13 Nov 2018 07:41:01 +0100 Subject: [PATCH 067/100] swarm/storage: fix access count on dbstore after cache hit (#17978) Access count was not incremented when chunk was retrieved from cache. So the garbage collector might have deleted the most frequently accessed chunk from disk. Co-authored-by: Ferenc Szabo --- swarm/storage/ldbstore.go | 48 +++++++++++++++-------- swarm/storage/ldbstore_test.go | 41 +++++++++++++++++++- swarm/storage/localstore.go | 1 + swarm/storage/localstore_test.go | 65 ++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 17 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 9feb687413..46e0402501 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -186,6 +186,20 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { return s, nil } +// MarkAccessed increments the access counter as a best effort for a chunk, so +// the chunk won't get garbage collected. +func (s *LDBStore) MarkAccessed(addr Address) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.closed { + return + } + + proximity := s.po(addr) + s.tryAccessIdx(addr, proximity) +} + // initialize and set values for processing of gc round func (s *LDBStore) startGC(c int) { @@ -349,6 +363,7 @@ func (s *LDBStore) collectGarbage() error { s.delete(s.gc.batch.Batch, index, keyIdx, po) singleIterationCount++ s.gc.count++ + log.Trace("garbage collect enqueued chunk for deletion", "key", hash) // break if target is not on max garbage batch boundary if s.gc.count >= s.gc.target { @@ -685,12 +700,7 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { idata, err := s.db.Get(ikey) if err != nil { s.doPut(chunk, &index, po) - } else { - log.Debug("ldbstore.put: chunk already exists, only update access", "key", chunk.Address(), "po", po) - decodeIndex(idata, &index) } - index.Access = s.accessCnt - s.accessCnt++ idata = encodeIndex(&index) s.batch.Put(ikey, idata) @@ -723,7 +733,8 @@ func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { s.entryCnt++ dbEntryCount.Inc(1) s.dataIdx++ - + index.Access = s.accessCnt + s.accessCnt++ cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po @@ -796,18 +807,23 @@ func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { } } -// try to find index; if found, update access cnt and return true -func (s *LDBStore) tryAccessIdx(ikey []byte, po uint8, index *dpaDBIndex) bool { +// tryAccessIdx tries to find index entry. If found then increments the access +// count for garbage collection and returns the index entry and true for found, +// otherwise returns nil and false. +func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) { + ikey := getIndexKey(addr) idata, err := s.db.Get(ikey) if err != nil { - return false + return nil, false } + + index := new(dpaDBIndex) decodeIndex(idata, index) oldGCIdxKey := getGCIdxKey(index) s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) - s.accessCnt++ index.Access = s.accessCnt idata = encodeIndex(index) + s.accessCnt++ s.batch.Put(ikey, idata) newGCIdxKey := getGCIdxKey(index) newGCIdxData := getGCIdxValue(index, po, ikey) @@ -817,7 +833,7 @@ func (s *LDBStore) tryAccessIdx(ikey []byte, po uint8, index *dpaDBIndex) bool { case s.batchesC <- struct{}{}: default: } - return true + return index, true } // GetSchema is returning the current named schema of the datastore as read from LevelDB @@ -858,12 +874,12 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) // TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { - var indx dpaDBIndex if s.closed { return nil, ErrDBClosed } proximity := s.po(addr) - if s.tryAccessIdx(getIndexKey(addr), proximity, &indx) { + index, found := s.tryAccessIdx(addr, proximity) + if found { var data []byte if s.getDataFunc != nil { // if getDataFunc is defined, use it to retrieve the chunk data @@ -874,12 +890,12 @@ func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { } } else { // default DbStore functionality to retrieve chunk data - datakey := getDataKey(indx.Idx, proximity) + datakey := getDataKey(index.Idx, proximity) data, err = s.db.Get(datakey) - log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", indx.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) + log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", index.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) if err != nil { log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err) - s.deleteNow(&indx, getIndexKey(addr), s.po(addr)) + s.deleteNow(index, getIndexKey(addr), s.po(addr)) return } } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 48af8c57c0..22213b12d4 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -31,7 +31,6 @@ import ( ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" - ldberrors "github.com/syndtr/goleveldb/leveldb/errors" ) @@ -105,6 +104,46 @@ func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) { testStoreCorrect(db, n, chunksize, t) } +func TestMarkAccessed(t *testing.T) { + db, cleanup, err := newTestDbStore(false, true) + defer cleanup() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + + h := GenerateRandomChunk(ch.DefaultSize) + + db.Put(context.Background(), h) + + var index dpaDBIndex + addr := h.Address() + idxk := getIndexKey(addr) + + idata, err := db.db.Get(idxk) + if err != nil { + t.Fatal(err) + } + decodeIndex(idata, &index) + + if index.Access != 0 { + t.Fatalf("Expected the access index to be %d, but it is %d", 0, index.Access) + } + + db.MarkAccessed(addr) + db.writeCurrentBatch() + + idata, err = db.db.Get(idxk) + if err != nil { + t.Fatal(err) + } + decodeIndex(idata, &index) + + if index.Access != 1 { + t.Fatalf("Expected the access index to be %d, but it is %d", 1, index.Access) + } + +} + func TestDbStoreRandom_1(t *testing.T) { testDbStoreRandom(1, 0, false, t) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 4fa6fb2f60..6971d759e3 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -153,6 +153,7 @@ func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err e if err == nil { metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) + go ls.DbStore.MarkAccessed(addr) return chunk, nil } diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 10f43f30f7..7a07726d1c 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "testing" + "time" ch "github.com/ethereum/go-ethereum/swarm/chunk" ) @@ -144,3 +145,67 @@ func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs [] } return hs, errs } + +// TestGetFrequentlyAccessedChunkWontGetGarbageCollected tests that the most +// frequently accessed chunk is not garbage collected from LDBStore, i.e., +// from disk when we are at the capacity and garbage collector runs. For that +// we start putting random chunks into the DB while continuously accessing the +// chunk we care about then check if we can still retrieve it from disk. +func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) { + ldbCap := defaultGCRatio + store, cleanup := setupLocalStore(t, ldbCap) + defer cleanup() + + var chunks []Chunk + for i := 0; i < ldbCap; i++ { + chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize)) + } + + mostAccessed := chunks[0].Address() + for _, chunk := range chunks { + if err := store.Put(context.Background(), chunk); err != nil { + t.Fatal(err) + } + + if _, err := store.Get(context.Background(), mostAccessed); err != nil { + t.Fatal(err) + } + // Add time for MarkAccessed() to be able to finish in a separate Goroutine + time.Sleep(1 * time.Millisecond) + } + + store.DbStore.collectGarbage() + if _, err := store.DbStore.Get(context.Background(), mostAccessed); err != nil { + t.Logf("most frequntly accessed chunk not found on disk (key: %v)", mostAccessed) + t.Fatal(err) + } + +} + +func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) { + t.Helper() + + var err error + datadir, err := ioutil.TempDir("", "storage") + if err != nil { + t.Fatal(err) + } + + params := &LocalStoreParams{ + StoreParams: NewStoreParams(uint64(ldbCap), uint(ldbCap), nil, nil), + } + params.Init(datadir) + + store, err := NewLocalStore(params, nil) + if err != nil { + _ = os.RemoveAll(datadir) + t.Fatal(err) + } + + cleanup = func() { + store.Close() + _ = os.RemoveAll(datadir) + } + + return store, cleanup +} From 588aa88121db007b59142ed37e74800c09038a7a Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Tue, 13 Nov 2018 17:02:04 +0800 Subject: [PATCH 068/100] github: format code owners file (#18090) replace tabs by spaces in the code owners file --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f7f6916c54..9a61d39325 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,9 +9,9 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi -p2p/simulations @lmars -p2p/protocols @zelig -swarm/api/http @justelad +p2p/simulations @lmars +p2p/protocols @zelig +swarm/api/http @justelad swarm/bmt @zelig swarm/dev @lmars swarm/fuse @jmozah @holisticode From 4fecc7a3b1b9c51efad47ea128abcb7259158487 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Tue, 13 Nov 2018 17:57:46 +0800 Subject: [PATCH 069/100] eth: fix minor grammar issue in comment (#18091) --- eth/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index 1f62d820ec..bd227a84e0 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -653,7 +653,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trueHead = request.Block.ParentHash() trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty()) ) - // Update the peers total difficulty if better than the previous + // Update the peer's total difficulty if better than the previous if _, td := p.Head(); trueTD.Cmp(td) > 0 { p.SetHead(trueHead, trueTD) From c41e1bd1ebf8a625dd22a07c31bcd6837705e0d7 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Tue, 13 Nov 2018 15:22:53 +0100 Subject: [PATCH 070/100] swarm/storage: fix garbage collector index skew (#18080) On file access LDBStore's tryAccessIdx() function created a faulty GC Index Data entry, because not indexing the ikey correctly. That caused the chunk addresses/hashes to start with '00' and the last two digits were dropped. => Incorrect chunk address. Besides the fix, the commit also contains a schema change which will run the CleanGCIndex() function to clean the GC index from erroneous entries. Note: CleanGCIndex() rebuilds the index from scratch which can take a really-really long time with a huge DB (possibly an hour). --- swarm/storage/ldbstore.go | 122 +++++++++++++++++++--------- swarm/storage/ldbstore_test.go | 140 +++++++++++++++++++++++++++++++++ swarm/storage/localstore.go | 53 ++++++++----- swarm/storage/schema.go | 13 ++- 4 files changed, 271 insertions(+), 57 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 46e0402501..fbae59facf 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -57,7 +57,6 @@ var ( var ( keyIndex = byte(0) - keyOldData = byte(1) keyAccessCnt = []byte{2} keyEntryCnt = []byte{3} keyDataIdx = []byte{4} @@ -285,6 +284,10 @@ func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { return val } +func parseGCIdxKey(key []byte) (byte, []byte) { + return key[0], key[1:] +} + func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) { index = &dpaDBIndex{ Idx: binary.BigEndian.Uint64(val[1:]), @@ -504,7 +507,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } } -//Cleanup iterates over the database and deletes chunks if they pass the `f` condition +// Cleanup iterates over the database and deletes chunks if they pass the `f` condition func (s *LDBStore) Cleanup(f func(*chunk) bool) { var errorsFound, removed, total int @@ -569,47 +572,90 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) { log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) } -func (s *LDBStore) ReIndex() { - //Iterates over the database and checks that there are no faulty chunks +// CleanGCIndex rebuilds the garbage collector index from scratch, while +// removing inconsistent elements, e.g., indices with missing data chunks. +// WARN: it's a pretty heavy, long running function. +func (s *LDBStore) CleanGCIndex() error { + s.lock.Lock() + defer s.lock.Unlock() + + batch := leveldb.Batch{} + + var okEntryCount uint64 + var totalEntryCount uint64 + + // throw out all gc indices, we will rebuild from cleaned index it := s.db.NewIterator() - startPosition := []byte{keyOldData} - it.Seek(startPosition) - var key []byte - var errorsFound, total int + it.Seek([]byte{keyGCIdx}) + var gcDeletes int for it.Valid() { - key = it.Key() - if (key == nil) || (key[0] != keyOldData) { + rowType, _ := parseGCIdxKey(it.Key()) + if rowType != keyGCIdx { break } - data := it.Value() - hasher := s.hashfunc() - hasher.Write(data) - hash := hasher.Sum(nil) - - newKey := make([]byte, 10) - oldCntKey := make([]byte, 2) - newCntKey := make([]byte, 2) - oldCntKey[0] = keyDistanceCnt - newCntKey[0] = keyDistanceCnt - key[0] = keyData - key[1] = s.po(Address(key[1:])) - oldCntKey[1] = key[1] - newCntKey[1] = s.po(Address(newKey[1:])) - copy(newKey[2:], key[1:]) - newValue := append(hash, data...) - - batch := new(leveldb.Batch) - batch.Delete(key) - s.bucketCnt[oldCntKey[1]]-- - batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]])) - batch.Put(newKey, newValue) - s.bucketCnt[newCntKey[1]]++ - batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]])) - s.db.Write(batch) + batch.Delete(it.Key()) + gcDeletes++ it.Next() } + log.Debug("gc", "deletes", gcDeletes) + if err := s.db.Write(&batch); err != nil { + return err + } + + it.Seek([]byte{keyIndex}) + var idx dpaDBIndex + var poPtrs [256]uint64 + for it.Valid() { + rowType, chunkHash := parseGCIdxKey(it.Key()) + if rowType != keyIndex { + break + } + err := decodeIndex(it.Value(), &idx) + if err != nil { + return fmt.Errorf("corrupt index: %v", err) + } + po := s.po(chunkHash) + + // if we don't find the data key, remove the entry + dataKey := getDataKey(idx.Idx, po) + _, err = s.db.Get(dataKey) + if err != nil { + log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) + batch.Delete(it.Key()) + } else { + gcIdxKey := getGCIdxKey(&idx) + gcIdxData := getGCIdxValue(&idx, po, chunkHash) + batch.Put(gcIdxKey, gcIdxData) + log.Trace("clean ok", "key", chunkHash, "gcKey", gcIdxKey, "gcData", gcIdxData) + okEntryCount++ + if idx.Idx > poPtrs[po] { + poPtrs[po] = idx.Idx + } + } + totalEntryCount++ + it.Next() + } + it.Release() - log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) + log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len()) + + var entryCount [8]byte + binary.BigEndian.PutUint64(entryCount[:], okEntryCount) + batch.Put(keyEntryCnt, entryCount[:]) + var poKey [2]byte + poKey[0] = keyDistanceCnt + for i, poPtr := range poPtrs { + poKey[1] = uint8(i) + if poPtr == 0 { + batch.Delete(poKey[:]) + } else { + var idxCount [8]byte + binary.BigEndian.PutUint64(idxCount[:], poPtr) + batch.Put(poKey[:], idxCount[:]) + } + } + + return s.db.Write(&batch) } // Delete is removes a chunk and updates indices. @@ -826,7 +872,7 @@ func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) { s.accessCnt++ s.batch.Put(ikey, idata) newGCIdxKey := getGCIdxKey(index) - newGCIdxData := getGCIdxValue(index, po, ikey) + newGCIdxData := getGCIdxValue(index, po, ikey[1:]) s.batch.Delete(oldGCIdxKey) s.batch.Put(newGCIdxKey, newGCIdxData) select { @@ -844,7 +890,7 @@ func (s *LDBStore) GetSchema() (string, error) { data, err := s.db.Get(keySchema) if err != nil { if err == leveldb.ErrNotFound { - return "", nil + return DbSchemaNone, nil } return "", err } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 22213b12d4..07557980c6 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -19,6 +19,7 @@ package storage import ( "bytes" "context" + "encoding/binary" "fmt" "io/ioutil" "os" @@ -623,6 +624,145 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) } +func TestCleanIndex(t *testing.T) { + capacity := 5000 + n := 3 + + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(uint64(capacity)) + defer cleanup() + + chunks, err := mputRandomChunks(ldb, n, 4096) + if err != nil { + t.Fatal(err) + } + + // remove the data of the first chunk + po := ldb.po(chunks[0].Address()[:]) + dataKey := make([]byte, 10) + dataKey[0] = keyData + dataKey[1] = byte(po) + // dataKey[2:10] = first chunk has storageIdx 0 on [2:10] + if _, err := ldb.db.Get(dataKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Delete(dataKey); err != nil { + t.Fatal(err) + } + + // remove the gc index row for the first chunk + gcFirstCorrectKey := make([]byte, 9) + gcFirstCorrectKey[0] = keyGCIdx + if err := ldb.db.Delete(gcFirstCorrectKey); err != nil { + t.Fatal(err) + } + + // warp the gc data of the second chunk + // this data should be correct again after the clean + gcSecondCorrectKey := make([]byte, 9) + gcSecondCorrectKey[0] = keyGCIdx + binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(1)) + gcSecondCorrectVal, err := ldb.db.Get(gcSecondCorrectKey) + if err != nil { + t.Fatal(err) + } + warpedGCVal := make([]byte, len(gcSecondCorrectVal)+1) + copy(warpedGCVal[1:], gcSecondCorrectVal) + if err := ldb.db.Delete(gcSecondCorrectKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Put(gcSecondCorrectKey, warpedGCVal); err != nil { + t.Fatal(err) + } + + if err := ldb.CleanGCIndex(); err != nil { + t.Fatal(err) + } + + // the index without corresponding data should have been deleted + idxKey := make([]byte, 33) + idxKey[0] = keyIndex + copy(idxKey[1:], chunks[0].Address()) + if _, err := ldb.db.Get(idxKey); err == nil { + t.Fatalf("expected chunk 0 idx to be pruned: %v", idxKey) + } + + // the two other indices should be present + copy(idxKey[1:], chunks[1].Address()) + if _, err := ldb.db.Get(idxKey); err != nil { + t.Fatalf("expected chunk 1 idx to be present: %v", idxKey) + } + + copy(idxKey[1:], chunks[2].Address()) + if _, err := ldb.db.Get(idxKey); err != nil { + t.Fatalf("expected chunk 2 idx to be present: %v", idxKey) + } + + // first gc index should still be gone + if _, err := ldb.db.Get(gcFirstCorrectKey); err == nil { + t.Fatalf("expected gc 0 idx to be pruned: %v", idxKey) + } + + // second gc index should still be fixed + if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { + t.Fatalf("expected gc 1 idx to be present: %v", idxKey) + } + + // third gc index should be unchanged + binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(2)) + if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + + c, err := ldb.db.Get(keyEntryCnt) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + + // entrycount should now be one less + entryCount := binary.BigEndian.Uint64(c) + if entryCount != 2 { + t.Fatalf("expected entrycnt to be 2, was %d", c) + } + + // the chunks might accidentally be in the same bin + // if so that bin counter will now be 2 - the highest added index. + // if not, the total of them will be 3 + poBins := []uint8{ldb.po(chunks[1].Address()), ldb.po(chunks[2].Address())} + if poBins[0] == poBins[1] { + poBins = poBins[:1] + } + + var binTotal uint64 + var currentBin [2]byte + currentBin[0] = keyDistanceCnt + if len(poBins) == 1 { + currentBin[1] = poBins[0] + c, err := ldb.db.Get(currentBin[:]) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + binCount := binary.BigEndian.Uint64(c) + if binCount != 2 { + t.Fatalf("expected entrycnt to be 2, was %d", binCount) + } + } else { + for _, bin := range poBins { + currentBin[1] = bin + c, err := ldb.db.Get(currentBin[:]) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + binCount := binary.BigEndian.Uint64(c) + binTotal += binCount + + } + if binTotal != 3 { + t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal) + } + } +} + func waitGc(ctx context.Context, ldb *LDBStore) { <-ldb.gc.runC ldb.gc.runC <- struct{}{} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 6971d759e3..fa98848ddc 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -196,31 +196,48 @@ func (ls *LocalStore) Close() { // Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match func (ls *LocalStore) Migrate() error { - schema, err := ls.DbStore.GetSchema() + actualDbSchema, err := ls.DbStore.GetSchema() if err != nil { log.Error(err.Error()) return err } - log.Debug("found schema", "schema", schema, "runtime-schema", CurrentDbSchema) - if schema != CurrentDbSchema { - // run migrations + log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) - if schema == "" { - log.Debug("running migrations for", "schema", schema, "runtime-schema", CurrentDbSchema) - - // delete chunks that are not valid, i.e. chunks that do not pass any of the ls.Validators - ls.DbStore.Cleanup(func(c *chunk) bool { - return !ls.isValid(c) - }) - - err := ls.DbStore.PutSchema(DbSchemaPurity) - if err != nil { - log.Error(err.Error()) - return err - } - } + if actualDbSchema == CurrentDbSchema { + return nil } + if actualDbSchema == DbSchemaNone { + ls.migrateFromNoneToPurity() + actualDbSchema = DbSchemaPurity + } + + if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { + return err + } + + if actualDbSchema == DbSchemaPurity { + if err := ls.migrateFromPurityToHalloween(); err != nil { + return err + } + actualDbSchema = DbSchemaHalloween + } + + if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { + return err + } return nil } + +func (ls *LocalStore) migrateFromNoneToPurity() { + // delete chunks that are not valid, i.e. chunks that do not pass + // any of the ls.Validators + ls.DbStore.Cleanup(func(c *chunk) bool { + return !ls.isValid(c) + }) +} + +func (ls *LocalStore) migrateFromPurityToHalloween() error { + return ls.DbStore.CleanGCIndex() +} diff --git a/swarm/storage/schema.go b/swarm/storage/schema.go index fb8498a293..91847ca0f9 100644 --- a/swarm/storage/schema.go +++ b/swarm/storage/schema.go @@ -1,6 +1,17 @@ package storage +// The DB schema we want to use. The actual/current DB schema might differ +// until migrations are run. +const CurrentDbSchema = DbSchemaHalloween + +// There was a time when we had no schema at all. +const DbSchemaNone = "" + // "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5 const DbSchemaPurity = "purity" -const CurrentDbSchema = DbSchemaPurity +// "halloween" is here because we had a screw in the garbage collector index. +// Because of that we had to rebuild the GC index to get rid of erroneous +// entries and that takes a long time. This schema is used for bookkeeping, +// so rebuild index will run just once. +const DbSchemaHalloween = "halloween" From cef7ed53bdec8d3b43b30ac33e8dfc2a92d3ea3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Nov 2018 10:16:28 +0200 Subject: [PATCH 071/100] params: update CHTs --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index ba719b873b..007e4a66d8 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 195, - SectionHead: common.HexToHash("0x1cdd2a84cf6c1261ffccc88f6bcefb513abd7934a96c1e909fbf74767560f16b"), - CHTRoot: common.HexToHash("0xe453333c20391d16b91b6fe11c104704f62c8dba15f69db73b4cdf7e100105eb"), - BloomRoot: common.HexToHash("0x47f30069473072e00d2cdca146dce40f0aad243dfc8221bf810822c091674efe"), + SectionIndex: 203, + SectionHead: common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"), + CHTRoot: common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"), + BloomRoot: common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -73,10 +73,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 126, - SectionHead: common.HexToHash("0x48f7dd4c9c60be04bf15fd4d0bcac46ddd8caf6b01d6fb8f8e1f7955cdd1337a"), - CHTRoot: common.HexToHash("0x6e54cb80a1884881ea1a114243af9012c95e0296b47f103b5ab124313968508e"), - BloomRoot: common.HexToHash("0xb55accf6dce6455b47db8510d15eff38d0ed7378829f3036d26b48e7d15da3f6"), + SectionIndex: 134, + SectionHead: common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"), + CHTRoot: common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"), + BloomRoot: common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 93, - SectionHead: common.HexToHash("0xdefb94aa217ab38f2919f7318d1d5476bd2aabf1ec9148047fe03e555615e0b4"), - CHTRoot: common.HexToHash("0x52c98c2fe508a8332c27dc10538f3fead43306e2b22b597587763c2fe6586da6"), - BloomRoot: common.HexToHash("0x93d83be0c1b12f732b1a027ecdfb16f39b0d020b8c10bfb90e76f3b01adfc5b6"), + SectionIndex: 100, + SectionHead: common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"), + CHTRoot: common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"), + BloomRoot: common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced From eb8fa3cc89ae3a3247c649486839b1c250554d2d Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Wed, 14 Nov 2018 15:21:14 +0700 Subject: [PATCH 072/100] cmd/swarm, swarm/api/http, swarm/bmt, swarm/fuse, swarm/network/stream, swarm/storage, swarm/storage/encryption, swarm/testutil: use pseudo-random instead of crypto-random for test files content generation (#18083) - Replace "crypto/rand" to "math/rand" for files content generation - Remove swarm/network_test.go.Shuffle and swarm/btm/btm_test.go.Shuffle - because go1.9 support dropped (see https://github.com/ethereum/go-ethereum/pull/17807 and comments to swarm/network_test.go.Shuffle) --- cmd/swarm/access_test.go | 3 +- cmd/swarm/export_test.go | 36 +--- cmd/swarm/feeds_test.go | 37 ++-- cmd/swarm/manifest_test.go | 6 +- cmd/swarm/run_test.go | 3 +- cmd/swarm/upload_test.go | 23 +-- swarm/api/client/client_test.go | 17 +- swarm/api/http/response_test.go | 12 +- swarm/api/http/server_test.go | 32 ++-- .../http.go => api/http/test_server.go} | 2 +- swarm/bmt/bmt_test.go | 71 ++------ swarm/fuse/swarmfs_test.go | 163 +++++++----------- swarm/network/stream/common_test.go | 9 +- swarm/network/stream/delivery_test.go | 7 +- swarm/network/stream/intervals_test.go | 6 +- swarm/network/stream/snapshot_sync_test.go | 5 +- swarm/network/stream/syncer_test.go | 5 +- swarm/network_test.go | 33 +--- swarm/storage/chunker_test.go | 34 ++-- swarm/storage/common_test.go | 7 +- swarm/storage/encryption/encryption_test.go | 5 +- swarm/storage/filestore_test.go | 16 +- swarm/storage/types.go | 11 -- swarm/testutil/file.go | 21 +++ 24 files changed, 202 insertions(+), 362 deletions(-) rename swarm/{testutil/http.go => api/http/test_server.go} (99%) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index b4d2e1dbc5..e812cd8fd1 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -54,7 +55,7 @@ var DefaultCurve = crypto.S256() // is then fetched through 2nd node. since the tested code is not key-aware - we can just // fetch from the 2nd node using HTTP BasicAuth func TestAccessPassword(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dataFilename := testutil.TempFileWithContent(t, data) diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go index c533511af0..f1bc2f2658 100644 --- a/cmd/swarm/export_test.go +++ b/cmd/swarm/export_test.go @@ -19,9 +19,7 @@ package main import ( "bytes" "crypto/md5" - "crypto/rand" "io" - "io/ioutil" "net/http" "os" "runtime" @@ -29,6 +27,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/swarm" + "github.com/ethereum/go-ethereum/swarm/testutil" ) // TestCLISwarmExportImport perform the following test: @@ -45,11 +44,12 @@ func TestCLISwarmExportImport(t *testing.T) { cluster := newTestCluster(t, 1) // generate random 10mb file - f, cleanup := generateRandomFile(t, 10000000) - defer cleanup() + content := testutil.RandomBytes(1, 10000000) + fileName := testutil.TempFileWithContent(t, string(content)) + defer os.Remove(fileName) // upload the file with 'swarm up' and expect a hash - up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name()) + up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName) _, matches := up.ExpectRegexp(`[a-f\d]{64}`) up.ExpectExit() hash := matches[0] @@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) { } // compare downloaded file with the generated random file - mustEqualFiles(t, f, res.Body) + mustEqualFiles(t, bytes.NewReader(content), res.Body) } func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { @@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen) } } - -func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) { - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - - // callback for tmp file cleanup - teardown = func() { - tmp.Close() - os.Remove(tmp.Name()) - } - - // write 10mb random data to file - buf := make([]byte, 10000000) - _, err = rand.Read(buf) - if err != nil { - t.Fatal(err) - } - ioutil.WriteFile(tmp.Name(), buf, 0755) - - return tmp, teardown -} diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go index 46727c21d3..fc3f72ab15 100644 --- a/cmd/swarm/feeds_test.go +++ b/cmd/swarm/feeds_test.go @@ -20,49 +20,37 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" "os" "testing" - "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" - "github.com/ethereum/go-ethereum/swarm/testutil" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestCLIFeedUpdate(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer { + srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") }, nil) log.Info("starting a test swarm server") defer srv.Close() // create a private key file for signing - pkfile, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer pkfile.Close() - defer os.Remove(pkfile.Name()) privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979" privKey, _ := crypto.HexToECDSA(privkeyHex) address := crypto.PubkeyToAddress(privKey.PublicKey) - // save the private key to a file - _, err = io.WriteString(pkfile, privkeyHex) - if err != nil { - t.Fatal(err) - } + pkFileName := testutil.TempFileWithContent(t, privkeyHex) + defer os.Remove(pkFileName) // compose a topic. We'll be doing quotes about Miguel de Cervantes var topic feed.Topic @@ -76,7 +64,7 @@ func TestCLIFeedUpdate(t *testing.T) { flags := []string{ "--bzzapi", srv.URL, - "--bzzaccount", pkfile.Name(), + "--bzzaccount", pkFileName, "feed", "update", "--topic", topic.Hex(), "--name", name, @@ -89,13 +77,10 @@ func TestCLIFeedUpdate(t *testing.T) { // now try to get the update using the client client := swarm.NewClient(srv.URL) - if err != nil { - t.Fatal(err) - } // build the same topic as before, this time // we use NewTopic to create a topic automatically. - topic, err = feed.NewTopic(name, subject) + topic, err := feed.NewTopic(name, subject) if err != nil { t.Fatal(err) } @@ -153,7 +138,7 @@ func TestCLIFeedUpdate(t *testing.T) { // test publishing a manifest flags = []string{ "--bzzapi", srv.URL, - "--bzzaccount", pkfile.Name(), + "--bzzaccount", pkFileName, "feed", "create", "--topic", topic.Hex(), } diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go index 7ea4e0c45b..01d982aa7a 100644 --- a/cmd/swarm/manifest_test.go +++ b/cmd/swarm/manifest_test.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/testutil" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" ) // TestManifestChange tests manifest add, update and remove @@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) { // Argument encrypt controls whether to use encryption or not. func testManifestChange(t *testing.T, encrypt bool) { t.Parallel() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-manifest-test") @@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { t.Parallel() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-manifest-test") diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 55199e955e..416fa7a503 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -42,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/swarm" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/testutil" ) var loglevel = flag.Int("loglevel", 3, "verbosity of logs") @@ -58,7 +57,7 @@ func init() { }) } -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } func TestMain(m *testing.M) { diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index ba4463e8bf..5f9844950e 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/mattn/go-colorable" ) @@ -77,33 +78,22 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) { cluster := newTestCluster(t, 3) defer cluster.Shutdown() - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer tmp.Close() - defer os.Remove(tmp.Name()) + tmpFileName := testutil.TempFileWithContent(t, data) + defer os.Remove(tmpFileName) // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - hashRegexp := `[a-f\d]{64}` flags := []string{ "--bzzapi", cluster.Nodes[0].URL, "up", - tmp.Name()} + tmpFileName} if toEncrypt { hashRegexp = `[a-f\d]{128}` flags = []string{ "--bzzapi", cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()} + tmpFileName} } // upload the file with 'swarm up' and expect a hash log.Info(fmt.Sprintf("uploading file with 'swarm up'")) @@ -203,7 +193,6 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } defer os.RemoveAll(tmpUploadDir) // create tmp files - data := "notsorandomdata" for _, path := range []string{"tmp1", "tmp2"} { if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil { t.Fatal(err) @@ -298,7 +287,7 @@ func TestCLISwarmUpDefaultPath(t *testing.T) { } func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index c30d699110..76b3493978 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -33,10 +33,9 @@ import ( swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/ethereum/go-ethereum/swarm/testutil" ) -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } @@ -49,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) { } func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -90,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) { } func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -188,7 +187,7 @@ func newTestDirectory(t *testing.T) string { // TestClientUploadDownloadDirectory tests uploading and downloading a // directory of files to a swarm manifest func TestClientUploadDownloadDirectory(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -254,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) { } func testClientFileList(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -312,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { // TestClientMultipartUpload tests uploading files to swarm using a multipart // upload func TestClientMultipartUpload(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // define an uploader which uploads testDirFiles with some data @@ -378,7 +377,7 @@ func TestClientCreateFeedMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() @@ -440,7 +439,7 @@ func TestClientCreateUpdateFeed(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() diff --git a/swarm/api/http/response_test.go b/swarm/api/http/response_test.go index 50a704be6d..486c19ab0e 100644 --- a/swarm/api/http/response_test.go +++ b/swarm/api/http/response_test.go @@ -24,12 +24,10 @@ import ( "testing" "golang.org/x/net/html" - - "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestError(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -55,7 +53,7 @@ func TestError(t *testing.T) { } func Test404Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -81,7 +79,7 @@ func Test404Page(t *testing.T) { } func Test500Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -106,7 +104,7 @@ func Test500Page(t *testing.T) { } } func Test500PageWith0xHashPrefix(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) { } func TestJsonResponse(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 04d0e045aa..1ef3deece2 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -20,7 +20,6 @@ import ( "archive/tar" "bytes" "context" - "crypto/rand" "encoding/json" "errors" "flag" @@ -58,7 +57,7 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) TestServer { return NewServer(api, "") } @@ -79,7 +78,7 @@ func TestBzzFeedMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // add the data our multihash aliased manifest will point to @@ -167,26 +166,19 @@ func TestBzzFeedMultihash(t *testing.T) { // Test Swarm feeds using the raw update methods func TestBzzFeed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) signer, _ := newTestSigner() defer srv.Close() // data of update 1 - update1Data := make([]byte, 666) + update1Data := testutil.RandomBytes(1, 666) update1Timestamp := srv.CurrentTime - _, err := rand.Read(update1Data) - if err != nil { - t.Fatal(err) - } //data for update 2 update2Data := []byte("foo") topic, _ := feed.NewTopic("foo.eth", nil) updateRequest := feed.NewFirstRequest(topic) - if err != nil { - t.Fatal(err) - } updateRequest.SetData(update1Data) if err := updateRequest.Sign(signer); err != nil { @@ -450,7 +442,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { addr := [3]storage.Address{} - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for i, mf := range testmanifest { @@ -688,7 +680,7 @@ func TestBzzTar(t *testing.T) { } func testBzzTar(encrypted bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} @@ -823,7 +815,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) { } func testBzzRootRedirect(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // create a manifest with some data at the root path @@ -878,7 +870,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { } func TestMethodsNotAllowed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() databytes := "bar" for _, c := range []struct { @@ -937,7 +929,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string } func TestGet(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for _, testCase := range []struct { @@ -1020,7 +1012,7 @@ func TestGet(t *testing.T) { } func TestModify(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() swarmClient := swarm.NewClient(srv.URL) @@ -1121,7 +1113,7 @@ func TestMultiPartUpload(t *testing.T) { // POST /bzz:/ Content-Type: multipart/form-data verbose := false // Setup Swarm - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() url := fmt.Sprintf("%s/bzz:/", srv.URL) @@ -1152,7 +1144,7 @@ func TestMultiPartUpload(t *testing.T) { // TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver func TestBzzGetFileWithResolver(t *testing.T) { resolver := newTestResolveValidator("") - srv := testutil.NewTestSwarmServer(t, serverFunc, resolver) + srv := NewTestSwarmServer(t, serverFunc, resolver) defer srv.Close() fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} diff --git a/swarm/testutil/http.go b/swarm/api/http/test_server.go similarity index 99% rename from swarm/testutil/http.go rename to swarm/api/http/test_server.go index cdf9239d06..9245c9c5b8 100644 --- a/swarm/testutil/http.go +++ b/swarm/api/http/test_server.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package testutil +package http import ( "io/ioutil" diff --git a/swarm/bmt/bmt_test.go b/swarm/bmt/bmt_test.go index 760aa11d8b..683ba4f5b2 100644 --- a/swarm/bmt/bmt_test.go +++ b/swarm/bmt/bmt_test.go @@ -18,10 +18,8 @@ package bmt import ( "bytes" - crand "crypto/rand" "encoding/binary" "fmt" - "io" "math/rand" "sync" "sync/atomic" @@ -29,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) // the actual data length generated (could be longer than max datalength of the BMT) @@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) { }) // run the tests - for _, x := range tests { + for i, x := range tests { for segmentCount := x.from; segmentCount <= x.to; segmentCount++ { for length := 1; length <= segmentCount*32; length++ { t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) { - data := make([]byte, length) - if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF { - t.Fatal(err) - } + data := testutil.RandomBytes(i, length) expected := x.expected(data) actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data) if !bytes.Equal(actual, expected) { @@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) { // tests sequential write with entire max size written in one go func TestSyncHasherCorrectness(t *testing.T) { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) hasher := sha3.NewKeccak256 size := hasher().Size() @@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) { // tests order-neutral concurrent writes with entire max size written in one go func TestAsyncCorrectness(t *testing.T) { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) hasher := sha3.NewKeccak256 size := hasher().Size() whs := []whenHash{first, last, random} @@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) { bmt := New(pool) for i := 0; i < 100; i++ { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) n := rand.Intn(bmt.Size()) err := testHasherCorrectness(bmt, hasher, data, n, segmentCount) if err != nil { @@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) { for i := 0; i < cycles; i++ { go func() { bmt := New(pool) - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) n := rand.Intn(bmt.Size()) errc <- testHasherCorrectness(bmt, hasher, data, n, 128) }() @@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) { defer pool.Drain(0) n := count * 32 bmt := New(pool) - data := newData(n) + data := testutil.RandomBytes(1, n) rbmt := NewRefHasher(hasher, count) refHash := rbmt.Hash(data) expHash := syncHash(bmt, nil, data) @@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) { // benchmarks simple sha3 hash on chunks func benchmarkSHA3(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 h := hasher() @@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) { func benchmarkBMTBaseline(t *testing.B, n int) { hasher := sha3.NewKeccak256 hashSize := hasher().Size() - data := newData(hashSize) + data := testutil.RandomBytes(1, hashSize) t.ReportAllocs() t.ResetTimer() @@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) { // benchmarks BMT Hasher func benchmarkBMT(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool) @@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) { // benchmarks BMT hasher with asynchronous concurrent segment/section writes func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool).NewAsyncWriter(double) idxs, segments := splitAndShuffle(bmt.SectionSize(), data) - shuffle(len(idxs), func(i int, j int) { + rand.Shuffle(len(idxs), func(i int, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] }) @@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { // benchmarks 100 concurrent bmt hashes with pool capacity func benchmarkPool(t *testing.B, poolsize, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, poolsize) cycles := 100 @@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) { // benchmarks the reference hasher func benchmarkRefHasher(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 rbmt := NewRefHasher(hasher, 128) @@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) { } } -func newData(bufferSize int) []byte { - data := make([]byte, bufferSize) - _, err := io.ReadFull(crand.Reader, data) - if err != nil { - panic(err.Error()) - } - return data -} - // Hash hashes the data and the span using the bmt hasher func syncHash(h *Hasher, span, data []byte) []byte { h.ResetWithLength(span) @@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) { section := data[i*secsize : end] segments = append(segments, section) } - shuffle(n, func(i int, j int) { + rand.Shuffle(n, func(i int, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] }) return idxs, segments @@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s } return <-c } - -// this is also in swarm/network_test.go -// shuffle pseudo-randomizes the order of elements. -// n is the number of elements. Shuffle panics if n < 0. -// swap swaps the elements with indexes i and j. -func shuffle(n int, swap func(i, j int)) { - if n < 0 { - panic("invalid argument to Shuffle") - } - - // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - // Shuffle really ought not be called with n that doesn't fit in 32 bits. - // Not only will it take a very long time, but with 2³¹! possible permutations, - // there's no way that any PRNG can have a big enough internal state to - // generate even a minuscule percentage of the possible permutations. - // Nevertheless, the right API signature accepts an int n, so handle it as best we can. - i := n - 1 - for ; i > 1<<31-1-1; i-- { - j := int(rand.Int63n(int64(i + 1))) - swap(i, j) - } - for ; i > 0; i-- { - j := int(rand.Int31n(int32(i + 1))) - swap(i, j) - } -} diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 87d9185505..460e31c4e9 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -20,20 +20,19 @@ package fuse import ( "bytes" - "crypto/rand" "flag" "fmt" "io" "io/ioutil" + "math/rand" "os" "path/filepath" "testing" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" - - "github.com/ethereum/go-ethereum/log" - + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { } } -func getRandomBytes(size int) []byte { - contents := make([]byte, size) - rand.Read(contents) - return contents -} - func isDirEmpty(name string) bool { f, err := os.Open(name) if err != nil { @@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "testMountDir") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)} - dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)} - dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)} - dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)} - dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)} - dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)} - dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)} - dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)} - dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)} - dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)} - dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)} + dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)} + dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)} + dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)} + dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)} + dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)} + dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)} + dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)} + dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)} + dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)} + dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)} + dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)} + dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)} + dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1") dat.testMountDir = filepath.Join(dat.testDir, "max-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2") dat.testMountDir = filepath.Join(dat.testDir, "max-mount2") - dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3") dat.testMountDir = filepath.Join(dat.testDir, "max-mount3") - dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4") dat.testMountDir = filepath.Join(dat.testDir, "max-mount4") - dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5") dat.testMountDir = filepath.Join(dat.testDir, "max-mount5") - dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { if err != nil { t.Fatalf("Couldn't create upload dir 6: %v", err) } - dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} testMountDir6 := filepath.Join(dat.testDir, "max-mount6") err = os.MkdirAll(testMountDir6, 0777) if err != nil { @@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { } // mount a different hash in already mounted point - dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2") if err3 != nil { t.Fatalf("Error creating second upload dir: %v", err3) @@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { //we need to manually close the file before mount for this test //but let's defer too in case of errors defer d.Close() - _, err = d.Write(getRandomBytes(10)) + _, err = d.Write(testutil.RandomBytes(1, 10)) if err != nil { t.Fatalf("Couldn't write to file: %v", err) } @@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1") dat.testMountDir = filepath.Join(dat.testDir, "seek-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1") dat.testMountDir = filepath.Join(dat.testDir, "create-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("Opened file") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Could not rand read contents %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload") dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("File opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error filling random bytes into byte array %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("Content read") _, err = d.Write(contents) if err != nil { @@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) } defer d.Close() log.Debug("File opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload") dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("file opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload") dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} + dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)} + dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)} + dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount") dat.files = make(map[string]fileInfo) - line1 := make([]byte, 10) - _, err = rand.Read(line1) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + line1 := testutil.RandomBytes(1, 10) dat.files["1.txt"] = fileInfo{0700, 333, 444, line1} @@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { } defer fd.Close() log.Debug("file opened") - line2 := make([]byte, 5) - _, err = rand.Read(line2) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + line2 := testutil.RandomBytes(1, 5) log.Debug("line read") _, err = fd.Seek(int64(len(line1)), 0) if err != nil { diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 72fdb2bd96..721b873b78 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -18,7 +18,6 @@ package stream import ( "context" - crand "crypto/rand" "errors" "flag" "fmt" @@ -40,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -230,12 +230,7 @@ func generateRandomFile() (string, error) { //generate a random file size between minFileSize and maxFileSize fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize)) - b := make([]byte, fileSize*1024) - _, err := crand.Read(b) - if err != nil { - log.Error("Error generating random file.", "err", err) - return "", err - } + b := testutil.RandomBytes(1, fileSize*1024) return string(b), nil } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index c77682e0e1..c9a530115c 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -19,9 +19,7 @@ package stream import ( "bytes" "context" - crand "crypto/rand" "fmt" - "io" "os" "sync" "testing" @@ -39,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) //Tests initializing a retrieve request @@ -530,7 +529,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck //now we can actually upload a (random) file to the round-robin store size := chunkCount * chunkSize log.Debug("Storing data to file store") - fileHash, wait, err := roundRobinFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) // wait until all chunks stored if err != nil { return err @@ -719,7 +718,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip for i := 0; i < chunkCount; i++ { // create actual size real chunks ctx := context.TODO() - hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false) + hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false) if err != nil { b.Fatalf("expected no error. got %v", err) } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 0c95fabb70..037984f220 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -18,10 +18,8 @@ package stream import ( "context" - crand "crypto/rand" "encoding/binary" "fmt" - "io" "os" "sync" "testing" @@ -36,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestIntervalsLive(t *testing.T) { @@ -130,7 +129,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { fileStore := item.(*storage.FileStore) size := chunkCount * chunkSize - _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + + _, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) if err != nil { log.Error("Store error: %v", "err", err) t.Fatal(err) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 2ddbed9368..4bd7f38f5b 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -17,9 +17,7 @@ package stream import ( "context" - crand "crypto/rand" "fmt" - "io" "os" "runtime" "sync" @@ -39,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" ) const MaxTimeout = 600 @@ -603,7 +602,7 @@ func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, lstore *storage.Lo size := chunkSize var rootAddrs []storage.Address for i := 0; i < chunkCount; i++ { - rk, wait, err := fileStore.Store(context.TODO(), io.LimitReader(crand.Reader, int64(size)), int64(size), false) + rk, wait, err := fileStore.Store(context.TODO(), testutil.RandomReader(i, size), int64(size), false) if err != nil { return nil, err } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index b0e35b0db3..a543cae058 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -18,9 +18,7 @@ package stream import ( "context" - crand "crypto/rand" "fmt" - "io" "io/ioutil" "math" "os" @@ -39,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" ) const dataChunkCount = 200 @@ -183,7 +182,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } fileStore := item.(*storage.FileStore) size := chunkCount * chunkSize - _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + _, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false) if err != nil { t.Fatal(err.Error()) } diff --git a/swarm/network_test.go b/swarm/network_test.go index aab1c05099..d84f28147f 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -335,7 +335,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() - shuffle(len(nodeIDs), func(i, j int) { + rand.Shuffle(len(nodeIDs), func(i, j int) { nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i] }) for _, id := range nodeIDs { @@ -404,7 +404,7 @@ func retrieve( nodeStatusM *sync.Map, totalFoundCount *uint64, ) (missing uint64) { - shuffle(len(files), func(i, j int) { + rand.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) @@ -499,32 +499,3 @@ func retrieve( return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount) } - -// Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333 -// -// Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped. -// -// shuffle pseudo-randomizes the order of elements. -// n is the number of elements. Shuffle panics if n < 0. -// swap swaps the elements with indexes i and j. -func shuffle(n int, swap func(i, j int)) { - if n < 0 { - panic("invalid argument to Shuffle") - } - - // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - // Shuffle really ought not be called with n that doesn't fit in 32 bits. - // Not only will it take a very long time, but with 2³¹! possible permutations, - // there's no way that any PRNG can have a big enough internal state to - // generate even a minuscule percentage of the possible permutations. - // Nevertheless, the right API signature accepts an int n, so handle it as best we can. - i := n - 1 - for ; i > 1<<31-1-1; i-- { - j := int(rand.Int63n(int64(i + 1))) - swap(i, j) - } - for ; i > 0; i-- { - j := int(rand.Int31n(int32(i + 1))) - swap(i, j) - } -} diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 6172d8a092..1f847edcb9 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -19,13 +19,13 @@ package storage import ( "bytes" "context" - "crypto/rand" "encoding/binary" "fmt" "io" "testing" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) /* @@ -47,7 +47,7 @@ func newTestHasherStore(store ChunkStore, hash string) *hasherStore { } func testRandomBrokenData(n int, tester *chunkerTester) { - data := io.LimitReader(rand.Reader, int64(n)) + data := testutil.RandomReader(1, n) brokendata := brokenLimitReader(data, n, n/2) buf := make([]byte, n) @@ -56,7 +56,7 @@ func testRandomBrokenData(n int, tester *chunkerTester) { tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) } - data = io.LimitReader(rand.Reader, int64(n)) + data = testutil.RandomReader(2, n) brokendata = brokenLimitReader(data, n, n/2) putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) @@ -77,7 +77,8 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = GenerateRandomData(n) + input = testutil.RandomBytes(1, n) + data = bytes.NewReader(input) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -118,14 +119,13 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) // testing partial read for i := 1; i < n; i += 10000 { readableLength := n - i - output := make([]byte, readableLength) r, err := reader.ReadAt(output, int64(i)) if r != readableLength || err != io.EOF { tester.t.Fatalf("readAt error with offset %v read: %v n = %v err = %v\n", i, r, readableLength, err) } if input != nil { - if !bytes.Equal(output, input[i:]) { - tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output) + if !bytes.Equal(output[:readableLength], input[i:]) { + tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output[:readableLength]) } } } @@ -177,7 +177,8 @@ func TestDataAppend(t *testing.T) { input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = GenerateRandomData(n) + input = testutil.RandomBytes(i, n) + data = bytes.NewReader(input) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -199,7 +200,8 @@ func TestDataAppend(t *testing.T) { appendInput, found := tester.inputs[uint64(m)] var appendData io.Reader if !found { - appendData, appendInput = GenerateRandomData(m) + appendInput = testutil.RandomBytes(i, m) + appendData = bytes.NewReader(appendInput) tester.inputs[uint64(m)] = appendInput } else { appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) @@ -272,7 +274,7 @@ func benchReadAll(reader LazySectionReader) { func benchmarkSplitJoin(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) ctx := context.TODO() @@ -292,7 +294,7 @@ func benchmarkSplitJoin(n int, t *testing.B) { func benchmarkSplitTreeSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() @@ -311,7 +313,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { func benchmarkSplitTreeBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() @@ -329,7 +331,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { func benchmarkSplitPyramidBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() @@ -347,7 +349,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { func benchmarkSplitPyramidSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() @@ -365,8 +367,8 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) - data1 := testDataReader(m) + data := testutil.RandomReader(i, n) + data1 := testutil.RandomReader(t.N+i, m) store := NewMapChunkStore() putGetter := newTestHasherStore(store, SHA3Hash) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 600be164a9..af104a5aeb 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -19,7 +19,6 @@ package storage import ( "bytes" "context" - "crypto/rand" "flag" "fmt" "io" @@ -31,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" ch "github.com/ethereum/go-ethereum/swarm/chunk" - colorable "github.com/mattn/go-colorable" + "github.com/mattn/go-colorable" ) var ( @@ -151,10 +150,6 @@ func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error) return err } -func testDataReader(l int) (r io.Reader) { - return io.LimitReader(rand.Reader, int64(l)) -} - func (r *brokenLimitedReader) Read(buf []byte) (int, error) { if r.off+len(buf) > r.errAt { return 0, fmt.Errorf("Broken reader") diff --git a/swarm/storage/encryption/encryption_test.go b/swarm/storage/encryption/encryption_test.go index c3abefdcec..0c0d0508c3 100644 --- a/swarm/storage/encryption/encryption_test.go +++ b/swarm/storage/encryption/encryption_test.go @@ -18,12 +18,12 @@ package encryption import ( "bytes" - "crypto/rand" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) var expectedTransformedHex = "352187af3a843decc63ceca6cb01ea39dbcf77caf0a8f705f5c30d557044ceec9392b94a79376f1e5c10cd0c0f2a98e5353bf22b3ea4fdac6677ee553dec192e3db64e179d0474e96088fb4abd2babd67de123fb398bdf84d818f7bda2c1ab60b3ea0e0569ae54aa969658eb4844e6960d2ff44d7c087ee3aaffa1c0ee5df7e50b615f7ad90190f022934ad5300c7d1809bfe71a11cc04cece5274eb97a5f20350630522c1dbb7cebaf4f97f84e03f5cfd88f2b48880b25d12f4d5e75c150f704ef6b46c72e07db2b705ac3644569dccd22fd8f964f6ef787fda63c46759af334e6f665f70eac775a7017acea49f3c7696151cb1b9434fa4ac27fb803921ffb5ec58dafa168098d7d5b97e384be3384cf5bc235c3d887fef89fe76c0065f9b8d6ad837b442340d9e797b46ef5709ea3358bc415df11e4830de986ef0f1c418ffdcc80e9a3cda9bea0ab5676c0d4240465c43ba527e3b4ea50b4f6255b510e5d25774a75449b0bd71e56c537ade4fcf0f4d63c99ae1dbb5a844971e2c19941b8facfcfc8ee3056e7cb3c7114c5357e845b52f7103cb6e00d2308c37b12baa5b769e1cc7b00fc06f2d16e70cc27a82cb9c1a4e40cb0d43907f73df2c9db44f1b51a6b0bc6d09f77ac3be14041fae3f9df2da42df43ae110904f9ecee278030185254d7c6e918a5512024d047f77a992088cb3190a6587aa54d0c7231c1cd2e455e0d4c07f74bece68e29cd8ba0190c0bcfb26d24634af5d91a81ef5d4dd3d614836ce942ddbf7bb1399317f4c03faa675f325f18324bf9433844bfe5c4cc04130c8d5c329562b7cd66e72f7355de8f5375a72202971613c32bd7f3fcdcd51080758cd1d0a46dbe8f0374381dbc359f5864250c63dde8131cbd7c98ae2b0147d6ea4bf65d1443d511b18e6d608bbb46ac036353b4c51df306a10a6f6939c38629a5c18aaf89cac04bd3ad5156e6b92011c88341cb08551bab0a89e6a46538f5af33b86121dba17e3a434c273f385cd2e8cb90bdd32747d8425d929ccbd9b0815c73325988855549a8489dfd047daf777aaa3099e54cf997175a5d9e1edfe363e3b68c70e02f6bf4fcde6a0f3f7d0e7e98bde1a72ae8b6cd27b32990680cc4a04fc467f41c5adcaddabfc71928a3f6872c360c1d765260690dd28b269864c8e380d9c92ef6b89b0094c8f9bb22608b4156381b19b920e9583c9616ce5693b4d2a6c689f02e6a91584a8e501e107403d2689dd0045269dd9946c0e969fb656a3b39d84a798831f5f9290f163eb2f97d3ae25071324e95e2256d9c1e56eb83c26397855323edc202d56ad05894333b7f0ed3c1e4734782eb8bd5477242fd80d7a89b12866f85cfae476322f032465d6b1253993033fccd4723530630ab97a1566460af9c90c9da843c229406e65f3fa578bd6bf04dee9b6153807ddadb8ceefc5c601a8ab26023c67b1ab1e8e0f29ce94c78c308005a781853e7a2e0e51738939a657c987b5e611f32f47b5ff461c52e63e0ea390515a8e1f5393dae54ea526934b5f310b76e3fa050e40718cb4c8a20e58946d6ee1879f08c52764422fe542b3240e75eccb7aa75b1f8a651e37a3bc56b0932cdae0e985948468db1f98eb4b77b82081ea25d8a762db00f7898864984bd80e2f3f35f236bf57291dec28f550769943bcfb6f884b7687589b673642ef7fe5d7d5a87d3eca5017f83ccb9a3310520474479464cb3f433440e7e2f1e28c0aef700a45848573409e7ab66e0cfd4fe5d2147ace81bc65fd8891f6245cd69246bbf5c27830e5ab882dd1d02aba34ff6ca9af88df00fd602892f02fedbdc65dedec203faf3f8ff4a97314e0ddb58b9ab756a61a562597f4088b445fcc3b28a708ca7b1485dcd791b779fbf2b3ef1ec5c6205f595fbe45a02105034147e5a146089c200a49dae33ae051a08ea5f974a21540aaeffa7f9d9e3d35478016fb27b871036eb27217a5b834b461f535752fb5f1c8dded3ae14ce3a2ef6639e2fe41939e3509e46e347a95d50b2080f1ba42c804b290ddc912c952d1cec3f2661369f738feacc0dbf1ea27429c644e45f9e26f30c341acd34c7519b2a1663e334621691e810767e9918c2c547b2e23cce915f97d26aac8d0d2fcd3edb7986ad4e2b8a852edebad534cb6c0e9f0797d3563e5409d7e068e48356c67ce519246cd9c560e881453df97cbba562018811e6cf8c327f399d1d1253ab47a19f4a0ccc7c6d86a9603e0551da310ea595d71305c4aad96819120a92cdbaf1f77ec8df9cc7c838c0d4de1e8692dd81da38268d1d71324bcffdafbe5122e4b81828e021e936d83ae8021eac592aa52cd296b5ce392c7173d622f8e07d18f59bb1b08ba15211af6703463b09b593af3c37735296816d9f2e7a369354a5374ea3955e14ca8ac56d5bfe4aef7a21bd825d6ae85530bee5d2aaaa4914981b3dfdb2e92ec2a27c83d74b59e84ff5c056f7d8945745f2efc3dcf28f288c6cd8383700fb2312f7001f24dd40015e436ae23e052fe9070ea9535b9c989898a9bda3d5382cf10e432fae6ccf0c825b3e6436edd3a9f8846e5606f8563931b5f29ba407c5236e5730225dda211a8504ec1817bc935e1fd9a532b648c502df302ed2063aed008fd5676131ac9e95998e9447b02bd29d77e38fcfd2959f2de929b31970335eb2a74348cc6918bc35b9bf749eab0fe304c946cd9e1ca284e6853c42646e60b6b39e0d3fb3c260abfc5c1b4ca3c3770f344118ca7c7f5c1ad1f123f8f369cd60afc3cdb3e9e81968c5c9fa7c8b014ffe0508dd4f0a2a976d5d1ca8fc9ad7a237d92cfe7b41413d934d6e142824b252699397e48e4bac4e91ebc10602720684bd0863773c548f9a2f9724245e47b129ecf65afd7252aac48c8a8d6fd3d888af592a01fb02dc71ed7538a700d3d16243e4621e0fcf9f8ed2b4e11c9fa9a95338bb1dac74a7d9bc4eb8cbf900b634a2a56469c00f5994e4f0934bdb947640e6d67e47d0b621aacd632bfd3c800bd7d93bd329f494a90e06ed51535831bd6e07ac1b4b11434ef3918fa9511813a002913f33f836454798b8d1787fea9a4c4743ba091ed192ed92f4d33e43a226bf9503e1a83a16dd340b3cbbf38af6db0d99201da8de529b4225f3d2fa2aad6621afc6c79ef3537720591edfc681ae6d00ede53ed724fc71b23b90d2e9b7158aaee98d626a4fe029107df2cb5f90147e07ebe423b1519d848af18af365c71bfd0665db46be493bbe99b79a188de0cf3594aef2299f0324075bdce9eb0b87bc29d62401ba4fd6ae48b1ba33261b5b845279becf38ee03e3dc5c45303321c5fac96fd02a3ad8c9e3b02127b320501333c9e6360440d1ad5e64a6239501502dde1a49c9abe33b66098458eee3d611bb06ffcd234a1b9aef4af5021cd61f0de6789f822ee116b5078aae8c129e8391d8987500d322b58edd1595dc570b57341f2df221b94a96ab7fbcf32a8ca9684196455694024623d7ed49f7d66e8dd453c0bae50e0d8b34377b22d0ece059e2c385dfc70b9089fcd27577c51f4d870b5738ee2b68c361a67809c105c7848b68860a829f29930857a9f9d40b14fd2384ac43bafdf43c0661103794c4bd07d1cfdd4681b6aeaefad53d4c1473359bcc5a83b09189352e5bb9a7498dd0effb89c35aad26954551f8b0621374b449bf515630bd3974dca982279733470fdd059aa9c3df403d8f22b38c4709c82d8f12b888e22990350490e16179caf406293cc9e65f116bafcbe96af132f679877061107a2f690a82a8cb46eea57a90abd23798c5937c6fe6b17be3f9bfa01ce117d2c268181b9095bf49f395fea07ca03838de0588c5e2db633e836d64488c1421e653ea52d810d096048c092d0da6e02fa6613890219f51a76148c8588c2487b171a28f17b7a299204874af0131725d793481333be5f08e86ca837a226850b0c1060891603bfecf9e55cddd22c0dbb28d495342d9cc3de8409f72e52a0115141cffe755c74f061c1a770428ccb0ae59536ee6fc074fbfc6cacb51a549d327527e20f8407477e60355863f1153f9ce95641198663c968874e7fdb29407bd771d94fdda8180cbb0358f5874738db705924b8cbe0cd5e1484aeb64542fe8f38667b7c34baf818c63b1e18440e9fba575254d063fd49f24ef26432f4eb323f3836972dca87473e3e9bb26dc3be236c3aae6bc8a6da567442309da0e8450e242fc9db836e2964f2c76a3b80a2c677979882dda7d7ebf62c93664018bcf4ec431fe6b403d49b3b36618b9c07c2d0d4569cb8d52223903debc72ec113955b206c34f1ae5300990ccfc0180f47d91afdb542b6312d12aeff7e19c645dc0b9fe6e3288e9539f6d5870f99882df187bfa6d24d179dfd1dac22212c8b5339f7171a3efc15b760fed8f68538bc5cbd845c2d1ab41f3a6c692820653eaef7930c02fbe6061d93805d73decdbb945572a7c44ed0241982a6e4d2d730898f82b3d9877cb7bca41cc6dcee67aa0c3d6db76f0b0a708ace0031113e48429de5d886c10e9200f68f32263a2fbf44a5992c2459fda7b8796ba796e3a0804fc25992ed2c9a5fe0580a6b809200ecde6caa0364b58be11564dcb9a616766dd7906db5636ee708b0204f38d309466d8d4a162965dd727e29f5a6c133e9b4ed5bafe803e479f9b2a7640c942c4a40b14ac7dc9828546052761a070f6404008f1ec3605836339c3da95a00b4fd81b2cabf88b51d2087d5b83e8c5b69bf96d8c72cbd278dad3bbb42b404b436f84ad688a22948adf60a81090f1e904291503c16e9f54b05fc76c881a5f95f0e732949e95d3f1bae2d3652a14fe0dda2d68879604657171856ef72637def2a96ac47d7b3fe86eb3198f5e0e626f06be86232305f2ae79ffcd2725e48208f9d8d63523f81915acc957563ab627cd6bc68c2a37d59fb0ed77a90aa9d085d6914a8ebada22a2c2d471b5163aeddd799d90fbb10ed6851ace2c4af504b7d572686700a59d6db46d5e42bb83f8e0c0ffe1dfa6582cc0b34c921ff6e85e83188d24906d5c08bb90069639e713051b3102b53e6f703e8210017878add5df68e6f2b108de279c5490e9eef5590185c4a1c744d4e00d244e1245a8805bd30407b1bc488db44870ccfd75a8af104df78efa2fb7ba31f048a263efdb3b63271fff4922bece9a71187108f65744a24f4947dc556b7440cb4fa45d296bb7f724588d1f245125b21ea063500029bd49650237f53899daf1312809552c81c5827341263cc807a29fe84746170cdfa1ff3838399a5645319bcaff674bb70efccdd88b3d3bb2f2d98111413585dc5d5bd5168f43b3f55e58972a5b2b9b3733febf02f931bd436648cb617c3794841aab961fe41277ab07812e1d3bc4ff6f4350a3e615bfba08c3b9480ef57904d3a16f7e916345202e3f93d11f7a7305170cb8c4eb9ac88ace8bbd1f377bdd5855d3162d6723d4435e84ce529b8f276a8927915ac759a0d04e5ca4a9d3da6291f0333b475df527e99fe38f7a4082662e8125936640c26dd1d17cf284ce6e2b17777a05aa0574f7793a6a062cc6f7263f7ab126b4528a17becfdec49ac0f7d8705aa1704af97fb861faa8a466161b2b5c08a5bacc79fe8500b913d65c8d3c52d1fd52d2ab2c9f52196e712455619c1cd3e0f391b274487944240e2ed8858dd0823c801094310024ae3fe4dd1cf5a2b6487b42cc5937bbafb193ee331d87e378258963d49b9da90899bbb4b88e79f78e866b0213f4719f67da7bcc2fce073c01e87c62ea3cdbcd589cfc41281f2f4c757c742d6d1e" @@ -125,8 +125,7 @@ func testEncryptDecryptIsIdentity(t *testing.T, padding int, initCtr uint32, dat key := GenerateRandomKey(keyLength) enc := New(key, padding, initCtr, hashFunc) - data := make([]byte, dataLength) - rand.Read(data) + data := testutil.RandomBytes(1, dataLength) encrypted, err := enc.Encrypt(data) if err != nil { diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index 9fe58f60c8..fb0f761a4a 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -23,6 +23,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" ) const testDataSize = 0x0001000 @@ -49,9 +51,9 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { fileStore := NewFileStore(localStore, NewFileStoreParams()) defer os.RemoveAll("/tmp/bzz") - reader, slice := GenerateRandomData(testDataSize) + slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() - key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) + key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) if err != nil { t.Fatalf("Store error: %v", err) } @@ -63,13 +65,13 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { if isEncrypted != toEncrypt { t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) } - resultSlice := make([]byte, len(slice)) + resultSlice := make([]byte, testDataSize) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { t.Fatalf("Retrieve error: %v", err) } - if n != len(slice) { - t.Fatalf("Slice size error got %d, expected %d.", n, len(slice)) + if n != testDataSize { + t.Fatalf("Slice size error got %d, expected %d.", n, testDataSize) } if !bytes.Equal(slice, resultSlice) { t.Fatalf("Comparison error.") @@ -114,9 +116,9 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { DbStore: db, } fileStore := NewFileStore(localStore, NewFileStoreParams()) - reader, slice := GenerateRandomData(testDataSize) + slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() - key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) + key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) if err != nil { t.Errorf("Store error: %v", err) } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 092843db0c..42557766ef 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -25,7 +25,6 @@ import ( "fmt" "hash" "io" - "io/ioutil" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" @@ -251,16 +250,6 @@ func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { return chunks } -func GenerateRandomData(l int) (r io.Reader, slice []byte) { - slice, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(l))) - if err != nil { - panic("rand error") - } - // log.Warn("generate random data", "len", len(slice), "data", common.Bytes2Hex(slice)) - r = io.LimitReader(bytes.NewReader(slice), int64(l)) - return r, slice -} - // Size, Seek, Read, ReadAt type LazySectionReader interface { Context() context.Context diff --git a/swarm/testutil/file.go b/swarm/testutil/file.go index ecb0d971ed..70732aa92e 100644 --- a/swarm/testutil/file.go +++ b/swarm/testutil/file.go @@ -17,8 +17,10 @@ package testutil import ( + "bytes" "io" "io/ioutil" + "math/rand" "os" "strings" "testing" @@ -42,3 +44,22 @@ func TempFileWithContent(t *testing.T, content string) string { } return tempFile.Name() } + +// RandomBytes returns pseudo-random deterministic result +// because test fails must be reproducible +func RandomBytes(seed, length int) []byte { + b := make([]byte, length) + reader := rand.New(rand.NewSource(int64(seed))) + for n := 0; n < length; { + read, err := reader.Read(b[n:]) + if err != nil { + panic(err) + } + n += read + } + return b +} + +func RandomReader(seed, length int) *bytes.Reader { + return bytes.NewReader(RandomBytes(seed, length)) +} From 58632d44021bf095b43a1bb2443e6e3690a94739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Nov 2018 10:22:10 +0200 Subject: [PATCH 073/100] params, swarm: release Geth v1.8.18 and Swarm v0.3.6 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 9e2f540c32..1abbd1a748 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 18 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 18 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index c70c30943d..0f5a453daa 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 6 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 48b4e8069cc1505a68bf69e8209dc17a9ecbfc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Nov 2018 10:27:30 +0200 Subject: [PATCH 074/100] params, swarm: begin Geth v1.8.19 and Swarm v0.3.7 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 1abbd1a748..b9dcc2a842 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 18 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 19 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 0f5a453daa..17ef34f5f4 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 698843b45f0dddd6f862c0733838272e6b4aa7f8 Mon Sep 17 00:00:00 2001 From: Kenso Trabing Date: Wed, 14 Nov 2018 05:21:10 -0500 Subject: [PATCH 075/100] rpc: fix example typo (#18100) whishes --> wishes --- rpc/client_example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go index 9c21c12d5d..3bb8717b80 100644 --- a/rpc/client_example_test.go +++ b/rpc/client_example_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// In this example, our client whishes to track the latest 'block number' +// In this example, our client wishes to track the latest 'block number' // known to the server. The server supports two methods: // // eth_getBlockByNumber("latest", {}) From 23de6197f9104a17ece1dc370c862bd23dcae273 Mon Sep 17 00:00:00 2001 From: Kenso Trabing Date: Wed, 14 Nov 2018 05:21:52 -0500 Subject: [PATCH 076/100] rpc: fix package doc typo (#18101) Changed "send" to "send," in two places --- rpc/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/doc.go b/rpc/doc.go index 9a6c4abbce..c60381b5ae 100644 --- a/rpc/doc.go +++ b/rpc/doc.go @@ -32,7 +32,7 @@ An example method: func (s *CalcService) Add(a, b int) (int, error) When the returned error isn't nil the returned integer is ignored and the error is -send back to the client. Otherwise the returned integer is send back to the client. +sent back to the client. Otherwise the returned integer is sent back to the client. Optional arguments are supported by accepting pointer values as arguments. E.g. if we want to do the addition in an optional finite field we can accept a mod From 9a000601c6c4e4f8134caedba1957ffe28d2b659 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Wed, 14 Nov 2018 20:50:30 +0800 Subject: [PATCH 077/100] consensus/clique: fix comment typo (#18103) --- consensus/clique/clique.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index eae09f91df..0cb72c35c7 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash { return sigHash(header) } -// Close implements consensus.Engine. It's a noop for clique as there is are no background threads. +// Close implements consensus.Engine. It's a noop for clique as there are no background threads. func (c *Clique) Close() error { return nil } From b8a2ac3fcf4c8249bd997969821feb4f9d0a428a Mon Sep 17 00:00:00 2001 From: Sheldon <11510383@mail.sustc.edu.cn> Date: Thu, 15 Nov 2018 17:10:45 +0800 Subject: [PATCH 078/100] les: fix pubkey index typo (#18093) --- les/serverpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/les/serverpool.go b/les/serverpool.go index 0fe6e49b61..52b54b371f 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -683,7 +683,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error { } func encodePubkey64(pub *ecdsa.PublicKey) []byte { - return crypto.FromECDSAPub(pub)[:1] + return crypto.FromECDSAPub(pub)[1:] } func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) { From 14346e4ef97ca812fb9f1d5d2cd87021c0155cf6 Mon Sep 17 00:00:00 2001 From: Kenso Trabing Date: Thu, 15 Nov 2018 04:11:14 -0500 Subject: [PATCH 079/100] internal: fix typo in comments (#18106) Changed "signTransactions" to "signTransaction" --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0c751c3282..43a33e9921 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { return fetchKeystore(s.am).Lock(addr) == nil } -// signTransactions sets defaults and signs the given transaction +// signTransaction sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) { From 434dd5bc0067cdf604d84426df9086015721dd36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 12 Nov 2018 18:47:34 +0200 Subject: [PATCH 080/100] cmd, core, eth, light, trie: add trie read caching layer --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 24 +- core/blockchain.go | 21 +- core/state/database.go | 14 +- eth/api.go | 6 +- eth/api_tracer.go | 4 +- eth/backend.go | 2 +- eth/config.go | 22 +- eth/gen_config.go | 28 ++- light/postprocess.go | 4 +- tests/block_test_util.go | 2 +- trie/database.go | 205 +++++++++------ trie/iterator_test.go | 14 +- trie/trie_test.go | 9 +- vendor/github.com/allegro/bigcache/LICENSE | 201 +++++++++++++++ vendor/github.com/allegro/bigcache/README.md | 150 +++++++++++ .../github.com/allegro/bigcache/bigcache.go | 202 +++++++++++++++ vendor/github.com/allegro/bigcache/bytes.go | 14 ++ .../allegro/bigcache/bytes_appengine.go | 7 + vendor/github.com/allegro/bigcache/clock.go | 14 ++ vendor/github.com/allegro/bigcache/config.go | 86 +++++++ .../github.com/allegro/bigcache/encoding.go | 62 +++++ .../allegro/bigcache/entry_not_found_error.go | 17 ++ vendor/github.com/allegro/bigcache/fnv.go | 28 +++ vendor/github.com/allegro/bigcache/hash.go | 8 + .../github.com/allegro/bigcache/iterator.go | 122 +++++++++ vendor/github.com/allegro/bigcache/logger.go | 30 +++ .../allegro/bigcache/queue/bytes_queue.go | 210 ++++++++++++++++ vendor/github.com/allegro/bigcache/shard.go | 236 ++++++++++++++++++ vendor/github.com/allegro/bigcache/stats.go | 15 ++ vendor/github.com/allegro/bigcache/utils.go | 16 ++ vendor/vendor.json | 12 + 33 files changed, 1660 insertions(+), 127 deletions(-) create mode 100644 vendor/github.com/allegro/bigcache/LICENSE create mode 100644 vendor/github.com/allegro/bigcache/README.md create mode 100644 vendor/github.com/allegro/bigcache/bigcache.go create mode 100644 vendor/github.com/allegro/bigcache/bytes.go create mode 100644 vendor/github.com/allegro/bigcache/bytes_appengine.go create mode 100644 vendor/github.com/allegro/bigcache/clock.go create mode 100644 vendor/github.com/allegro/bigcache/config.go create mode 100644 vendor/github.com/allegro/bigcache/encoding.go create mode 100644 vendor/github.com/allegro/bigcache/entry_not_found_error.go create mode 100644 vendor/github.com/allegro/bigcache/fnv.go create mode 100644 vendor/github.com/allegro/bigcache/hash.go create mode 100644 vendor/github.com/allegro/bigcache/iterator.go create mode 100644 vendor/github.com/allegro/bigcache/logger.go create mode 100644 vendor/github.com/allegro/bigcache/queue/bytes_queue.go create mode 100644 vendor/github.com/allegro/bigcache/shard.go create mode 100644 vendor/github.com/allegro/bigcache/stats.go create mode 100644 vendor/github.com/allegro/bigcache/utils.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0288b33806..69802a48a7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -89,6 +89,7 @@ var ( utils.LightKDFFlag, utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, utils.TrieCacheGenFlag, utils.ListenPortFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 8b0491ce3c..82f17e0ee8 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, utils.TrieCacheGenFlag, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 429c2bbb9b..d7b698c7e2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -295,7 +295,12 @@ var ( CacheDatabaseFlag = cli.IntFlag{ Name: "cache.database", Usage: "Percentage of cache memory allowance to use for database io", - Value: 75, + Value: 50, + } + CacheTrieFlag = cli.IntFlag{ + Name: "cache.trie", + Usage: "Percentage of cache memory allowance to use for trie caching", + Value: 25, } CacheGCFlag = cli.IntFlag{ Name: "cache.gc", @@ -1157,8 +1162,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { + cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") @@ -1393,12 +1401,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cache := &core.CacheConfig{ - Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive", - TrieNodeLimit: eth.DefaultConfig.TrieCache, - TrieTimeLimit: eth.DefaultConfig.TrieTimeout, + Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive", + TrieCleanLimit: eth.DefaultConfig.TrieCleanCache, + TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache, + TrieTimeLimit: eth.DefaultConfig.TrieTimeout, + } + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { + cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)} chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil) diff --git a/core/blockchain.go b/core/blockchain.go index 26ac75b8cf..22f130ce67 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -68,9 +68,10 @@ const ( // CacheConfig contains the configuration values for the trie caching/pruning // that's resident in a blockchain. type CacheConfig struct { - Disabled bool // Whether to disable trie write caching (archive node) - TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + Disabled bool // Whether to disable trie write caching (archive node) + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk } // BlockChain represents the canonical chain given a database with a genesis @@ -140,8 +141,9 @@ type BlockChain struct { func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ - TrieNodeLimit: 256, - TrieTimeLimit: 5 * time.Minute, + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, } } bodyCache, _ := lru.New(bodyCacheLimit) @@ -156,7 +158,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par cacheConfig: cacheConfig, db: db, triegc: prque.New(nil), - stateCache: state.NewDatabase(db), + stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit), quit: make(chan struct{}), shouldPreserve: shouldPreserve, bodyCache: bodyCache, @@ -393,6 +395,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { return state.New(root, bc.stateCache) } +// StateCache returns the caching database underpinning the blockchain instance. +func (bc *BlockChain) StateCache() state.Database { + return bc.stateCache +} + // Reset purges the entire blockchain, restoring it to its genesis state. func (bc *BlockChain) Reset() error { return bc.ResetWithGenesisBlock(bc.genesisBlock) @@ -938,7 +945,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() - limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024 + limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 ) if nodes > limit || imgs > 4*1024*1024 { triedb.Cap(limit - ethdb.IdealBatchSize) diff --git a/core/state/database.go b/core/state/database.go index c1b630991c..f6ea144b9b 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -72,13 +72,19 @@ type Trie interface { } // NewDatabase creates a backing store for state. The returned database is safe for -// concurrent use and retains cached trie nodes in memory. The pool is an optional -// intermediate trie-node memory pool between the low level storage layer and the -// high level trie abstraction. +// concurrent use and retains a few recent expanded trie nodes in memory. To keep +// more historical state in memory, use the NewDatabaseWithCache constructor. func NewDatabase(db ethdb.Database) Database { + return NewDatabaseWithCache(db, 0) +} + +// NewDatabase creates a backing store for state. The returned database is safe for +// concurrent use and retains both a few recent expanded trie nodes in memory, as +// well as a lot of collapsed RLP trie nodes in a large memory cache. +func NewDatabaseWithCache(db ethdb.Database, cache int) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ - db: trie.NewDatabase(db), + db: trie.NewDatabaseWithCache(db, cache), codeSizeCache: csc, } } diff --git a/eth/api.go b/eth/api.go index 3ec3afb81e..816b9cd335 100644 --- a/eth/api.go +++ b/eth/api.go @@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } + triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) + oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) + newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0) if err != nil { return nil, err } - diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{})) iter := trie.NewIterator(diff) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 80552ada8c..2ebbcc5fd9 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Ensure we have a valid starting state before doing any work origin := start.NumberU64() - database := state.NewDatabase(api.eth.ChainDb()) + database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis if number := start.NumberU64(); number > 0 { start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) @@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } // Otherwise try to reexec blocks until we find a state or reach our limit origin := block.NumberU64() - database := state.NewDatabase(api.eth.ChainDb()) + database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) for i := uint64(0); i < reexec; i++ { block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/eth/backend.go b/eth/backend.go index b555b064ad..4721408425 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EWASMInterpreter: config.EWASMInterpreter, EVMInterpreter: config.EVMInterpreter, } - cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout} + cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout} ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) if err != nil { diff --git a/eth/config.go b/eth/config.go index e32c01a734..601f4735e3 100644 --- a/eth/config.go +++ b/eth/config.go @@ -43,15 +43,16 @@ var DefaultConfig = Config{ DatasetsInMem: 1, DatasetsOnDisk: 2, }, - NetworkId: 1, - LightPeers: 100, - DatabaseCache: 768, - TrieCache: 256, - TrieTimeout: 60 * time.Minute, - MinerGasFloor: 8000000, - MinerGasCeil: 8000000, - MinerGasPrice: big.NewInt(params.GWei), - MinerRecommit: 3 * time.Second, + NetworkId: 1, + LightPeers: 100, + DatabaseCache: 512, + TrieCleanCache: 256, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + MinerGasFloor: 8000000, + MinerGasCeil: 8000000, + MinerGasPrice: big.NewInt(params.GWei), + MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ @@ -94,7 +95,8 @@ type Config struct { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCache int + TrieCleanCache int + TrieDirtyCache int TrieTimeout time.Duration // Mining-related options diff --git a/eth/gen_config.go b/eth/gen_config.go index d401a917d2..2777aa9e83 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -28,7 +28,8 @@ func (c Config) MarshalTOML() (interface{}, error) { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCache int + TrieCleanCache int + TrieDirtyCache int TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` @@ -43,6 +44,8 @@ func (c Config) MarshalTOML() (interface{}, error) { GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` + EWASMInterpreter string + EVMInterpreter string } var enc Config enc.Genesis = c.Genesis @@ -54,7 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache - enc.TrieCache = c.TrieCache + enc.TrieCleanCache = c.TrieCleanCache + enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase enc.MinerNotify = c.MinerNotify @@ -69,6 +73,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.GPO = c.GPO enc.EnablePreimageRecording = c.EnablePreimageRecording enc.DocRoot = c.DocRoot + enc.EWASMInterpreter = c.EWASMInterpreter + enc.EVMInterpreter = c.EVMInterpreter return &enc, nil } @@ -84,7 +90,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int - TrieCache *int + TrieCleanCache *int + TrieDirtyCache *int TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` @@ -99,6 +106,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` + EWASMInterpreter *string + EVMInterpreter *string } var dec Config if err := unmarshal(&dec); err != nil { @@ -131,8 +140,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseCache != nil { c.DatabaseCache = *dec.DatabaseCache } - if dec.TrieCache != nil { - c.TrieCache = *dec.TrieCache + if dec.TrieCleanCache != nil { + c.TrieCleanCache = *dec.TrieCleanCache + } + if dec.TrieDirtyCache != nil { + c.TrieDirtyCache = *dec.TrieDirtyCache } if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout @@ -176,5 +188,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DocRoot != nil { c.DocRoot = *dec.DocRoot } + if dec.EWASMInterpreter != nil { + c.EWASMInterpreter = *dec.EWASMInterpreter + } + if dec.EVMInterpreter != nil { + c.EVMInterpreter = *dec.EVMInterpreter + } return nil } diff --git a/light/postprocess.go b/light/postprocess.go index 1cfd7535e7..dd1b74a7be 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down sectionSize: size, } return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") @@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down parentSize: parentSize, size: size, } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 12cba3244f..9fa69bf4ed 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -118,7 +118,7 @@ func (t *BlockTest) Run() error { } else { engine = ethash.NewShared() } - chain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil) + chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieCleanLimit: 0}, config, engine, vm.Config{}, nil) if err != nil { return err } diff --git a/trie/database.go b/trie/database.go index d0691b637e..71190b3f3f 100644 --- a/trie/database.go +++ b/trie/database.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/allegro/bigcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -30,6 +31,11 @@ import ( ) var ( + memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) + memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) + memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) + memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) @@ -64,9 +70,10 @@ type DatabaseReader interface { type Database struct { diskdb ethdb.Database // Persistent storage for matured trie nodes - nodes map[common.Hash]*cachedNode // Data and references relationships of a node - oldest common.Hash // Oldest tracked node, flush-list head - newest common.Hash // Newest tracked node, flush-list tail + cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs + dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes + oldest common.Hash // Oldest tracked node, flush-list head + newest common.Hash // Newest tracked node, flush-list tail preimages map[common.Hash][]byte // Preimages of nodes from the secure trie seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys @@ -79,7 +86,7 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - nodesSize common.StorageSize // Storage size of the nodes cache (exc. flushlist) + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. flushlist) preimagesSize common.StorageSize // Storage size of the preimages cache lock sync.RWMutex @@ -262,11 +269,30 @@ func expandNode(hash hashNode, n node, cachegen uint16) node { } // NewDatabase creates a new trie database to store ephemeral trie content before -// its written out to disk or garbage collected. +// its written out to disk or garbage collected. No read cache is created, so all +// data retrievals will hit the underlying disk database. func NewDatabase(diskdb ethdb.Database) *Database { + return NewDatabaseWithCache(diskdb, 0) +} + +// NewDatabaseWithCache creates a new trie database to store ephemeral trie content +// before its written out to disk or garbage collected. It also acts as a read cache +// for nodes loaded from disk. +func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { + var cleans *bigcache.BigCache + if cache > 0 { + cleans, _ = bigcache.NewBigCache(bigcache.Config{ + Shards: 1024, + LifeWindow: time.Hour, + MaxEntriesInWindow: cache * 1024, + MaxEntrySize: 512, + HardMaxCacheSize: cache, + }) + } return &Database{ diskdb: diskdb, - nodes: map[common.Hash]*cachedNode{{}: {}}, + cleans: cleans, + dirties: map[common.Hash]*cachedNode{{}: {}}, preimages: make(map[common.Hash][]byte), } } @@ -293,7 +319,7 @@ func (db *Database) InsertBlob(hash common.Hash, blob []byte) { // size tracking. func (db *Database) insert(hash common.Hash, blob []byte, node node) { // If the node's already cached, skip - if _, ok := db.nodes[hash]; ok { + if _, ok := db.dirties[hash]; ok { return } // Create the cached entry for this node @@ -303,19 +329,19 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) { flushPrev: db.newest, } for _, child := range entry.childs() { - if c := db.nodes[child]; c != nil { + if c := db.dirties[child]; c != nil { c.parents++ } } - db.nodes[hash] = entry + db.dirties[hash] = entry // Update the flush-list endpoints if db.oldest == (common.Hash{}) { db.oldest, db.newest = hash, hash } else { - db.nodes[db.newest].flushNext, db.newest = hash, hash + db.dirties[db.newest].flushNext, db.newest = hash, hash } - db.nodesSize += common.StorageSize(common.HashLength + entry.size) + db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } // insertPreimage writes a new trie node pre-image to the memory database if it's @@ -333,35 +359,64 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash, cachegen uint16) node { - // Retrieve the node from cache if available + // Retrieve the node from the clean cache if available + if db.cleans != nil { + if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return mustDecodeNode(hash[:], enc, cachegen) + } + } + // Retrieve the node from the dirty cache if available db.lock.RLock() - node := db.nodes[hash] + dirty := db.dirties[hash] db.lock.RUnlock() - if node != nil { - return node.obj(hash, cachegen) + if dirty != nil { + return dirty.obj(hash, cachegen) } // Content unavailable in memory, attempt to retrieve from disk enc, err := db.diskdb.Get(hash[:]) if err != nil || enc == nil { return nil } + if db.cleans != nil { + db.cleans.Set(string(hash[:]), enc) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(enc))) + } return mustDecodeNode(hash[:], enc, cachegen) } // Node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. func (db *Database) Node(hash common.Hash) ([]byte, error) { - // Retrieve the node from cache if available + // Retrieve the node from the clean cache if available + if db.cleans != nil { + if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return enc, nil + } + } + // Retrieve the node from the dirty cache if available db.lock.RLock() - node := db.nodes[hash] + dirty := db.dirties[hash] db.lock.RUnlock() - if node != nil { - return node.rlp(), nil + if dirty != nil { + return dirty.rlp(), nil } // Content unavailable in memory, attempt to retrieve from disk - return db.diskdb.Get(hash[:]) + enc, err := db.diskdb.Get(hash[:]) + if err == nil && enc != nil { + if db.cleans != nil { + db.cleans.Set(string(hash[:]), enc) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(enc))) + } + } + return enc, err } // preimage retrieves a cached trie node pre-image from memory. If it cannot be @@ -395,8 +450,8 @@ func (db *Database) Nodes() []common.Hash { db.lock.RLock() defer db.lock.RUnlock() - var hashes = make([]common.Hash, 0, len(db.nodes)) - for hash := range db.nodes { + var hashes = make([]common.Hash, 0, len(db.dirties)) + for hash := range db.dirties { if hash != (common.Hash{}) { // Special case for "root" references/nodes hashes = append(hashes, hash) } @@ -415,18 +470,18 @@ func (db *Database) Reference(child common.Hash, parent common.Hash) { // reference is the private locked version of Reference. func (db *Database) reference(child common.Hash, parent common.Hash) { // If the node does not exist, it's a node pulled from disk, skip - node, ok := db.nodes[child] + node, ok := db.dirties[child] if !ok { return } // If the reference already exists, only duplicate for roots - if db.nodes[parent].children == nil { - db.nodes[parent].children = make(map[common.Hash]uint16) - } else if _, ok = db.nodes[parent].children[child]; ok && parent != (common.Hash{}) { + if db.dirties[parent].children == nil { + db.dirties[parent].children = make(map[common.Hash]uint16) + } else if _, ok = db.dirties[parent].children[child]; ok && parent != (common.Hash{}) { return } node.parents++ - db.nodes[parent].children[child]++ + db.dirties[parent].children[child]++ } // Dereference removes an existing reference from a root node. @@ -439,25 +494,25 @@ func (db *Database) Dereference(root common.Hash) { db.lock.Lock() defer db.lock.Unlock() - nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() db.dereference(root, common.Hash{}) - db.gcnodes += uint64(nodes - len(db.nodes)) - db.gcsize += storage - db.nodesSize + db.gcnodes += uint64(nodes - len(db.dirties)) + db.gcsize += storage - db.dirtiesSize db.gctime += time.Since(start) memcacheGCTimeTimer.Update(time.Since(start)) - memcacheGCSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheGCNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheGCSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheGCNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) } // dereference is the private locked version of Dereference. func (db *Database) dereference(child common.Hash, parent common.Hash) { // Dereference the parent-child - node := db.nodes[parent] + node := db.dirties[parent] if node.children != nil && node.children[child] > 0 { node.children[child]-- @@ -466,7 +521,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { } } // If the child does not exist, it's a previously committed node. - node, ok := db.nodes[child] + node, ok := db.dirties[child] if !ok { return } @@ -483,20 +538,20 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { switch child { case db.oldest: db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = common.Hash{} case db.newest: db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = common.Hash{} default: - db.nodes[node.flushPrev].flushNext = node.flushNext - db.nodes[node.flushNext].flushPrev = node.flushPrev + db.dirties[node.flushPrev].flushNext = node.flushNext + db.dirties[node.flushNext].flushPrev = node.flushPrev } // Dereference all children and delete the node for _, hash := range node.childs() { db.dereference(hash, child) } - delete(db.nodes, child) - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, child) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } } @@ -509,13 +564,13 @@ func (db *Database) Cap(limit common.StorageSize) error { // by only uncaching existing data when the database write finalizes. db.lock.RLock() - nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() batch := db.diskdb.NewBatch() - // db.nodesSize only contains the useful data in the cache, but when reporting + // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength) + size := db.dirtiesSize + common.StorageSize((len(db.dirties)-1)*2*common.HashLength) // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. @@ -540,7 +595,7 @@ func (db *Database) Cap(limit common.StorageSize) error { oldest := db.oldest for size > limit && oldest != (common.Hash{}) { // Fetch the oldest referenced node and push into the batch - node := db.nodes[oldest] + node := db.dirties[oldest] if err := batch.Put(oldest[:], node.rlp()); err != nil { db.lock.RUnlock() return err @@ -578,25 +633,25 @@ func (db *Database) Cap(limit common.StorageSize) error { db.preimagesSize = 0 } for db.oldest != oldest { - node := db.nodes[db.oldest] - delete(db.nodes, db.oldest) + node := db.dirties[db.oldest] + delete(db.dirties, db.oldest) db.oldest = node.flushNext - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } if db.oldest != (common.Hash{}) { - db.nodes[db.oldest].flushPrev = common.Hash{} + db.dirties[db.oldest].flushPrev = common.Hash{} } - db.flushnodes += uint64(nodes - len(db.nodes)) - db.flushsize += storage - db.nodesSize + db.flushnodes += uint64(nodes - len(db.dirties)) + db.flushsize += storage - db.dirtiesSize db.flushtime += time.Since(start) memcacheFlushTimeTimer.Update(time.Since(start)) - memcacheFlushSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheFlushNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheFlushSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheFlushNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), - "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), + "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) return nil } @@ -630,7 +685,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { } } // Move the trie itself into the batch, flushing if enough data is accumulated - nodes, storage := len(db.nodes), db.nodesSize + nodes, storage := len(db.dirties), db.dirtiesSize if err := db.commit(node, batch); err != nil { log.Error("Failed to commit trie from trie database", "err", err) db.lock.RUnlock() @@ -654,15 +709,15 @@ func (db *Database) Commit(node common.Hash, report bool) error { db.uncache(node) memcacheCommitTimeTimer.Update(time.Since(start)) - memcacheCommitSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheCommitNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) logger := log.Info if !report { logger = log.Debug } - logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes)+int(db.flushnodes), "size", storage-db.nodesSize+db.flushsize, "time", time.Since(start)+db.flushtime, - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + logger("Persisted trie from memory database", "nodes", nodes-len(db.dirties)+int(db.flushnodes), "size", storage-db.dirtiesSize+db.flushsize, "time", time.Since(start)+db.flushtime, + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) // Reset the garbage collection statistics db.gcnodes, db.gcsize, db.gctime = 0, 0, 0 @@ -674,7 +729,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { // commit is the private locked version of Commit. func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // If the node does not exist, it's a previously committed node - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return nil } @@ -702,7 +757,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // to disk. func (db *Database) uncache(hash common.Hash) { // If the node does not exist, we're done on this path - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return } @@ -710,20 +765,20 @@ func (db *Database) uncache(hash common.Hash) { switch hash { case db.oldest: db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = common.Hash{} case db.newest: db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = common.Hash{} default: - db.nodes[node.flushPrev].flushNext = node.flushNext - db.nodes[node.flushNext].flushPrev = node.flushPrev + db.dirties[node.flushPrev].flushNext = node.flushNext + db.dirties[node.flushNext].flushPrev = node.flushPrev } // Uncache the node's subtries and remove the node itself too for _, child := range node.childs() { db.uncache(child) } - delete(db.nodes, hash) - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, hash) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } // Size returns the current storage size of the memory cache in front of the @@ -732,11 +787,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { db.lock.RLock() defer db.lock.RUnlock() - // db.nodesSize only contains the useful data in the cache, but when reporting + // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength) - return db.nodesSize + flushlistSize, db.preimagesSize + var flushlistSize = common.StorageSize((len(db.dirties) - 1) * 2 * common.HashLength) + return db.dirtiesSize + flushlistSize, db.preimagesSize } // verifyIntegrity is a debug method to iterate over the entire trie stored in @@ -749,12 +804,12 @@ func (db *Database) verifyIntegrity() { // Iterate over all the cached nodes and accumulate them into a set reachable := map[common.Hash]struct{}{{}: {}} - for child := range db.nodes[common.Hash{}].children { + for child := range db.dirties[common.Hash{}].children { db.accumulate(child, reachable) } // Find any unreachable but cached nodes unreachable := []string{} - for hash, node := range db.nodes { + for hash, node := range db.dirties { if _, ok := reachable[hash]; !ok { unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}", hash, node.node, node.parents, node.flushPrev, node.flushNext)) @@ -769,7 +824,7 @@ func (db *Database) verifyIntegrity() { // cached children found in memory. func (db *Database) accumulate(hash common.Hash, reachable map[common.Hash]struct{}) { // Mark the node reachable if present in the memory cache - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 2a510b1c2d..4f633b195f 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -113,7 +113,7 @@ func TestNodeIteratorCoverage(t *testing.T) { t.Errorf("failed to retrieve reported node %x: %v", hash, err) } } - for hash, obj := range db.nodes { + for hash, obj := range db.dirties { if obj != nil && hash != (common.Hash{}) { if _, ok := hashes[hash]; !ok { t.Errorf("state entry not reported %x", hash) @@ -333,8 +333,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } } if memonly { - robj = triedb.nodes[rkey] - delete(triedb.nodes, rkey) + robj = triedb.dirties[rkey] + delete(triedb.dirties, rkey) } else { rval, _ = diskdb.Get(rkey[:]) diskdb.Delete(rkey[:]) @@ -350,7 +350,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { // Add the node back and continue iteration. if memonly { - triedb.nodes[rkey] = robj + triedb.dirties[rkey] = robj } else { diskdb.Put(rkey[:], rval) } @@ -393,8 +393,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { barNodeObj *cachedNode ) if memonly { - barNodeObj = triedb.nodes[barNodeHash] - delete(triedb.nodes, barNodeHash) + barNodeObj = triedb.dirties[barNodeHash] + delete(triedb.dirties, barNodeHash) } else { barNodeBlob, _ = diskdb.Get(barNodeHash[:]) diskdb.Delete(barNodeHash[:]) @@ -411,7 +411,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Reinsert the missing node. if memonly { - triedb.nodes[barNodeHash] = barNodeObj + triedb.dirties[barNodeHash] = barNodeObj } else { diskdb.Put(barNodeHash[:], barNodeBlob) } diff --git a/trie/trie_test.go b/trie/trie_test.go index f8e5fd12a1..f9d6029c99 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -119,7 +119,7 @@ func testMissingNode(t *testing.T, memonly bool) { hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9") if memonly { - delete(triedb.nodes, hash) + delete(triedb.dirties, hash) } else { diskdb.Delete(hash[:]) } @@ -342,15 +342,16 @@ func TestCacheUnload(t *testing.T) { // Commit the trie repeatedly and access key1. // The branch containing it is loaded from DB exactly two times: // in the 0th and 6th iteration. - db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} - trie, _ = New(root, NewDatabase(db)) + diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} + triedb := NewDatabase(diskdb) + trie, _ = New(root, triedb) trie.SetCacheLimit(5) for i := 0; i < 12; i++ { getString(trie, key1) trie.Commit(nil) } // Check that it got loaded two times. - for dbkey, count := range db.gets { + for dbkey, count := range diskdb.gets { if count != 2 { t.Errorf("db key %x loaded %d times, want %d times", []byte(dbkey), count, 2) } diff --git a/vendor/github.com/allegro/bigcache/LICENSE b/vendor/github.com/allegro/bigcache/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/vendor/github.com/allegro/bigcache/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/allegro/bigcache/README.md b/vendor/github.com/allegro/bigcache/README.md new file mode 100644 index 0000000000..c23f7f36ce --- /dev/null +++ b/vendor/github.com/allegro/bigcache/README.md @@ -0,0 +1,150 @@ +# BigCache [![Build Status](https://travis-ci.org/allegro/bigcache.svg?branch=master)](https://travis-ci.org/allegro/bigcache) [![Coverage Status](https://coveralls.io/repos/github/allegro/bigcache/badge.svg?branch=master)](https://coveralls.io/github/allegro/bigcache?branch=master) [![GoDoc](https://godoc.org/github.com/allegro/bigcache?status.svg)](https://godoc.org/github.com/allegro/bigcache) [![Go Report Card](https://goreportcard.com/badge/github.com/allegro/bigcache)](https://goreportcard.com/report/github.com/allegro/bigcache) + +Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance. +BigCache keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place, +therefore entries (de)serialization in front of the cache will be needed in most use cases. + +## Usage + +### Simple initialization + +```go +import "github.com/allegro/bigcache" + +cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute)) + +cache.Set("my-unique-key", []byte("value")) + +entry, _ := cache.Get("my-unique-key") +fmt.Println(string(entry)) +``` + +### Custom initialization + +When cache load can be predicted in advance then it is better to use custom initialization because additional memory +allocation can be avoided in that way. + +```go +import ( + "log" + + "github.com/allegro/bigcache" +) + +config := bigcache.Config { + // number of shards (must be a power of 2) + Shards: 1024, + // time after which entry can be evicted + LifeWindow: 10 * time.Minute, + // rps * lifeWindow, used only in initial memory allocation + MaxEntriesInWindow: 1000 * 10 * 60, + // max entry size in bytes, used only in initial memory allocation + MaxEntrySize: 500, + // prints information about additional memory allocation + Verbose: true, + // cache will not allocate more memory than this limit, value in MB + // if value is reached then the oldest entries can be overridden for the new ones + // 0 value means no size limit + HardMaxCacheSize: 8192, + // callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A bitmask representing the reason will be returned. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + OnRemove: nil, + // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // Ignored if OnRemove is specified. + OnRemoveWithReason: nil, + } + +cache, initErr := bigcache.NewBigCache(config) +if initErr != nil { + log.Fatal(initErr) +} + +cache.Set("my-unique-key", []byte("value")) + +if entry, err := cache.Get("my-unique-key"); err == nil { + fmt.Println(string(entry)) +} +``` + +## Benchmarks + +Three caches were compared: bigcache, [freecache](https://github.com/coocood/freecache) and map. +Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10. + +### Writes and reads + +```bash +cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m + +BenchmarkMapSet-8 3000000 569 ns/op 202 B/op 3 allocs/op +BenchmarkConcurrentMapSet-8 1000000 1592 ns/op 347 B/op 8 allocs/op +BenchmarkFreeCacheSet-8 3000000 775 ns/op 355 B/op 2 allocs/op +BenchmarkBigCacheSet-8 3000000 640 ns/op 303 B/op 2 allocs/op +BenchmarkMapGet-8 5000000 407 ns/op 24 B/op 1 allocs/op +BenchmarkConcurrentMapGet-8 3000000 558 ns/op 24 B/op 2 allocs/op +BenchmarkFreeCacheGet-8 2000000 682 ns/op 136 B/op 2 allocs/op +BenchmarkBigCacheGet-8 3000000 512 ns/op 152 B/op 4 allocs/op +BenchmarkBigCacheSetParallel-8 10000000 225 ns/op 313 B/op 3 allocs/op +BenchmarkFreeCacheSetParallel-8 10000000 218 ns/op 341 B/op 3 allocs/op +BenchmarkConcurrentMapSetParallel-8 5000000 318 ns/op 200 B/op 6 allocs/op +BenchmarkBigCacheGetParallel-8 20000000 178 ns/op 152 B/op 4 allocs/op +BenchmarkFreeCacheGetParallel-8 20000000 295 ns/op 136 B/op 3 allocs/op +BenchmarkConcurrentMapGetParallel-8 10000000 237 ns/op 24 B/op 2 allocs/op +``` + +Writes and reads in bigcache are faster than in freecache. +Writes to map are the slowest. + +### GC pause time + +```bash +cd caches_bench; go run caches_gc_overhead_comparison.go + +Number of entries: 20000000 +GC pause for bigcache: 5.8658ms +GC pause for freecache: 32.4341ms +GC pause for map: 52.9661ms +``` + +Test shows how long are the GC pauses for caches filled with 20mln of entries. +Bigcache and freecache have very similar GC pause time. +It is clear that both reduce GC overhead in contrast to map +which GC pause time took more than 10 seconds. + +## How it works + +BigCache relies on optimization presented in 1.5 version of Go ([issue-9477](https://github.com/golang/go/issues/9477)). +This optimization states that if map without pointers in keys and values is used then GC will omit its content. +Therefore BigCache uses `map[uint64]uint32` where keys are hashed and values are offsets of entries. + +Entries are kept in bytes array, to omit GC again. +Bytes array size can grow to gigabytes without impact on performance +because GC will only see single pointer to it. + +## Bigcache vs Freecache + +Both caches provide the same core features but they reduce GC overhead in different ways. +Bigcache relies on `map[uint64]uint32`, freecache implements its own mapping built on +slices to reduce number of pointers. + +Results from benchmark tests are presented above. +One of the advantage of bigcache over freecache is that you don’t need to know +the size of the cache in advance, because when bigcache is full, +it can allocate additional memory for new entries instead of +overwriting existing ones as freecache does currently. +However hard max size in bigcache also can be set, check [HardMaxCacheSize](https://godoc.org/github.com/allegro/bigcache#Config). + +## HTTP Server + +This package also includes an easily deployable HTTP implementation of BigCache, which can be found in the [server](/server) package. + +## More + +Bigcache genesis is described in allegro.tech blog post: [writing a very fast cache service in Go](http://allegro.tech/2016/03/writing-fast-cache-service-in-go.html) + +## License + +BigCache is released under the Apache 2.0 license (see [LICENSE](LICENSE)) diff --git a/vendor/github.com/allegro/bigcache/bigcache.go b/vendor/github.com/allegro/bigcache/bigcache.go new file mode 100644 index 0000000000..e6aee8663b --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bigcache.go @@ -0,0 +1,202 @@ +package bigcache + +import ( + "fmt" + "time" +) + +const ( + minimumEntriesInShard = 10 // Minimum number of entries in single shard +) + +// BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance. +// It keeps entries on heap but omits GC for them. To achieve that, operations take place on byte arrays, +// therefore entries (de)serialization in front of the cache will be needed in most use cases. +type BigCache struct { + shards []*cacheShard + lifeWindow uint64 + clock clock + hash Hasher + config Config + shardMask uint64 + maxShardSize uint32 + close chan struct{} +} + +// RemoveReason is a value used to signal to the user why a particular key was removed in the OnRemove callback. +type RemoveReason uint32 + +const ( + // Expired means the key is past its LifeWindow. + Expired RemoveReason = iota + // NoSpace means the key is the oldest and the cache size was at its maximum when Set was called, or the + // entry exceeded the maximum shard size. + NoSpace + // Deleted means Delete was called and this key was removed as a result. + Deleted +) + +// NewBigCache initialize new instance of BigCache +func NewBigCache(config Config) (*BigCache, error) { + return newBigCache(config, &systemClock{}) +} + +func newBigCache(config Config, clock clock) (*BigCache, error) { + + if !isPowerOfTwo(config.Shards) { + return nil, fmt.Errorf("Shards number must be power of two") + } + + if config.Hasher == nil { + config.Hasher = newDefaultHasher() + } + + cache := &BigCache{ + shards: make([]*cacheShard, config.Shards), + lifeWindow: uint64(config.LifeWindow.Seconds()), + clock: clock, + hash: config.Hasher, + config: config, + shardMask: uint64(config.Shards - 1), + maxShardSize: uint32(config.maximumShardSize()), + close: make(chan struct{}), + } + + var onRemove func(wrappedEntry []byte, reason RemoveReason) + if config.OnRemove != nil { + onRemove = cache.providedOnRemove + } else if config.OnRemoveWithReason != nil { + onRemove = cache.providedOnRemoveWithReason + } else { + onRemove = cache.notProvidedOnRemove + } + + for i := 0; i < config.Shards; i++ { + cache.shards[i] = initNewShard(config, onRemove, clock) + } + + if config.CleanWindow > 0 { + go func() { + ticker := time.NewTicker(config.CleanWindow) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + cache.cleanUp(uint64(t.Unix())) + case <-cache.close: + return + } + } + }() + } + + return cache, nil +} + +// Close is used to signal a shutdown of the cache when you are done with it. +// This allows the cleaning goroutines to exit and ensures references are not +// kept to the cache preventing GC of the entire cache. +func (c *BigCache) Close() error { + close(c.close) + return nil +} + +// Get reads entry for the key. +// It returns an EntryNotFoundError when +// no entry exists for the given key. +func (c *BigCache) Get(key string) ([]byte, error) { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.get(key, hashedKey) +} + +// Set saves entry under the key +func (c *BigCache) Set(key string, entry []byte) error { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.set(key, hashedKey, entry) +} + +// Delete removes the key +func (c *BigCache) Delete(key string) error { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.del(key, hashedKey) +} + +// Reset empties all cache shards +func (c *BigCache) Reset() error { + for _, shard := range c.shards { + shard.reset(c.config) + } + return nil +} + +// Len computes number of entries in cache +func (c *BigCache) Len() int { + var len int + for _, shard := range c.shards { + len += shard.len() + } + return len +} + +// Capacity returns amount of bytes store in the cache. +func (c *BigCache) Capacity() int { + var len int + for _, shard := range c.shards { + len += shard.capacity() + } + return len +} + +// Stats returns cache's statistics +func (c *BigCache) Stats() Stats { + var s Stats + for _, shard := range c.shards { + tmp := shard.getStats() + s.Hits += tmp.Hits + s.Misses += tmp.Misses + s.DelHits += tmp.DelHits + s.DelMisses += tmp.DelMisses + s.Collisions += tmp.Collisions + } + return s +} + +// Iterator returns iterator function to iterate over EntryInfo's from whole cache. +func (c *BigCache) Iterator() *EntryInfoIterator { + return newIterator(c) +} + +func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool { + oldestTimestamp := readTimestampFromEntry(oldestEntry) + if currentTimestamp-oldestTimestamp > c.lifeWindow { + evict(Expired) + return true + } + return false +} + +func (c *BigCache) cleanUp(currentTimestamp uint64) { + for _, shard := range c.shards { + shard.cleanUp(currentTimestamp) + } +} + +func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) { + return c.shards[hashedKey&c.shardMask] +} + +func (c *BigCache) providedOnRemove(wrappedEntry []byte, reason RemoveReason) { + c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry)) +} + +func (c *BigCache) providedOnRemoveWithReason(wrappedEntry []byte, reason RemoveReason) { + if c.config.onRemoveFilter == 0 || (1< 0 { + c.config.OnRemoveWithReason(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry), reason) + } +} + +func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte, reason RemoveReason) { +} diff --git a/vendor/github.com/allegro/bigcache/bytes.go b/vendor/github.com/allegro/bigcache/bytes.go new file mode 100644 index 0000000000..3944bfe135 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bytes.go @@ -0,0 +1,14 @@ +// +build !appengine + +package bigcache + +import ( + "reflect" + "unsafe" +) + +func bytesToString(b []byte) string { + bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len} + return *(*string)(unsafe.Pointer(&strHeader)) +} diff --git a/vendor/github.com/allegro/bigcache/bytes_appengine.go b/vendor/github.com/allegro/bigcache/bytes_appengine.go new file mode 100644 index 0000000000..3892f3b54f --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bytes_appengine.go @@ -0,0 +1,7 @@ +// +build appengine + +package bigcache + +func bytesToString(b []byte) string { + return string(b) +} diff --git a/vendor/github.com/allegro/bigcache/clock.go b/vendor/github.com/allegro/bigcache/clock.go new file mode 100644 index 0000000000..f8b535e133 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/clock.go @@ -0,0 +1,14 @@ +package bigcache + +import "time" + +type clock interface { + epoch() int64 +} + +type systemClock struct { +} + +func (c systemClock) epoch() int64 { + return time.Now().Unix() +} diff --git a/vendor/github.com/allegro/bigcache/config.go b/vendor/github.com/allegro/bigcache/config.go new file mode 100644 index 0000000000..9654143ab1 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/config.go @@ -0,0 +1,86 @@ +package bigcache + +import "time" + +// Config for BigCache +type Config struct { + // Number of cache shards, value must be a power of two + Shards int + // Time after which entry can be evicted + LifeWindow time.Duration + // Interval between removing expired entries (clean up). + // If set to <= 0 then no action is performed. Setting to < 1 second is counterproductive — bigcache has a one second resolution. + CleanWindow time.Duration + // Max number of entries in life window. Used only to calculate initial size for cache shards. + // When proper value is set then additional memory allocation does not occur. + MaxEntriesInWindow int + // Max size of entry in bytes. Used only to calculate initial size for cache shards. + MaxEntrySize int + // Verbose mode prints information about new memory allocation + Verbose bool + // Hasher used to map between string keys and unsigned 64bit integers, by default fnv64 hashing is used. + Hasher Hasher + // HardMaxCacheSize is a limit for cache size in MB. Cache will not allocate more memory than this limit. + // It can protect application from consuming all available memory on machine, therefore from running OOM Killer. + // Default value is 0 which means unlimited size. When the limit is higher than 0 and reached then + // the oldest entries are overridden for the new ones. + HardMaxCacheSize int + // OnRemove is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + OnRemove func(key string, entry []byte) + // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // Ignored if OnRemove is specified. + OnRemoveWithReason func(key string, entry []byte, reason RemoveReason) + + onRemoveFilter int + + // Logger is a logging interface and used in combination with `Verbose` + // Defaults to `DefaultLogger()` + Logger Logger +} + +// DefaultConfig initializes config with default values. +// When load for BigCache can be predicted in advance then it is better to use custom config. +func DefaultConfig(eviction time.Duration) Config { + return Config{ + Shards: 1024, + LifeWindow: eviction, + CleanWindow: 0, + MaxEntriesInWindow: 1000 * 10 * 60, + MaxEntrySize: 500, + Verbose: true, + Hasher: newDefaultHasher(), + HardMaxCacheSize: 0, + Logger: DefaultLogger(), + } +} + +// initialShardSize computes initial shard size +func (c Config) initialShardSize() int { + return max(c.MaxEntriesInWindow/c.Shards, minimumEntriesInShard) +} + +// maximumShardSize computes maximum shard size +func (c Config) maximumShardSize() int { + maxShardSize := 0 + + if c.HardMaxCacheSize > 0 { + maxShardSize = convertMBToBytes(c.HardMaxCacheSize) / c.Shards + } + + return maxShardSize +} + +// OnRemoveFilterSet sets which remove reasons will trigger a call to OnRemoveWithReason. +// Filtering out reasons prevents bigcache from unwrapping them, which saves cpu. +func (c Config) OnRemoveFilterSet(reasons ...RemoveReason) Config { + c.onRemoveFilter = 0 + for i := range reasons { + c.onRemoveFilter |= 1 << uint(reasons[i]) + } + + return c +} diff --git a/vendor/github.com/allegro/bigcache/encoding.go b/vendor/github.com/allegro/bigcache/encoding.go new file mode 100644 index 0000000000..4d434e5dce --- /dev/null +++ b/vendor/github.com/allegro/bigcache/encoding.go @@ -0,0 +1,62 @@ +package bigcache + +import ( + "encoding/binary" +) + +const ( + timestampSizeInBytes = 8 // Number of bytes used for timestamp + hashSizeInBytes = 8 // Number of bytes used for hash + keySizeInBytes = 2 // Number of bytes used for size of entry key + headersSizeInBytes = timestampSizeInBytes + hashSizeInBytes + keySizeInBytes // Number of bytes used for all headers +) + +func wrapEntry(timestamp uint64, hash uint64, key string, entry []byte, buffer *[]byte) []byte { + keyLength := len(key) + blobLength := len(entry) + headersSizeInBytes + keyLength + + if blobLength > len(*buffer) { + *buffer = make([]byte, blobLength) + } + blob := *buffer + + binary.LittleEndian.PutUint64(blob, timestamp) + binary.LittleEndian.PutUint64(blob[timestampSizeInBytes:], hash) + binary.LittleEndian.PutUint16(blob[timestampSizeInBytes+hashSizeInBytes:], uint16(keyLength)) + copy(blob[headersSizeInBytes:], key) + copy(blob[headersSizeInBytes+keyLength:], entry) + + return blob[:blobLength] +} + +func readEntry(data []byte) []byte { + length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:]) + + // copy on read + dst := make([]byte, len(data)-int(headersSizeInBytes+length)) + copy(dst, data[headersSizeInBytes+length:]) + + return dst +} + +func readTimestampFromEntry(data []byte) uint64 { + return binary.LittleEndian.Uint64(data) +} + +func readKeyFromEntry(data []byte) string { + length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:]) + + // copy on read + dst := make([]byte, length) + copy(dst, data[headersSizeInBytes:headersSizeInBytes+length]) + + return bytesToString(dst) +} + +func readHashFromEntry(data []byte) uint64 { + return binary.LittleEndian.Uint64(data[timestampSizeInBytes:]) +} + +func resetKeyFromEntry(data []byte) { + binary.LittleEndian.PutUint64(data[timestampSizeInBytes:], 0) +} diff --git a/vendor/github.com/allegro/bigcache/entry_not_found_error.go b/vendor/github.com/allegro/bigcache/entry_not_found_error.go new file mode 100644 index 0000000000..883bdc29eb --- /dev/null +++ b/vendor/github.com/allegro/bigcache/entry_not_found_error.go @@ -0,0 +1,17 @@ +package bigcache + +import "fmt" + +// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key +type EntryNotFoundError struct { + key string +} + +func notFound(key string) error { + return &EntryNotFoundError{key} +} + +// Error returned when entry does not exist. +func (e EntryNotFoundError) Error() string { + return fmt.Sprintf("Entry %q not found", e.key) +} diff --git a/vendor/github.com/allegro/bigcache/fnv.go b/vendor/github.com/allegro/bigcache/fnv.go new file mode 100644 index 0000000000..188c9aa6dc --- /dev/null +++ b/vendor/github.com/allegro/bigcache/fnv.go @@ -0,0 +1,28 @@ +package bigcache + +// newDefaultHasher returns a new 64-bit FNV-1a Hasher which makes no memory allocations. +// Its Sum64 method will lay the value out in big-endian byte order. +// See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function +func newDefaultHasher() Hasher { + return fnv64a{} +} + +type fnv64a struct{} + +const ( + // offset64 FNVa offset basis. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash + offset64 = 14695981039346656037 + // prime64 FNVa prime value. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash + prime64 = 1099511628211 +) + +// Sum64 gets the string and returns its uint64 hash value. +func (f fnv64a) Sum64(key string) uint64 { + var hash uint64 = offset64 + for i := 0; i < len(key); i++ { + hash ^= uint64(key[i]) + hash *= prime64 + } + + return hash +} diff --git a/vendor/github.com/allegro/bigcache/hash.go b/vendor/github.com/allegro/bigcache/hash.go new file mode 100644 index 0000000000..5f8ade774d --- /dev/null +++ b/vendor/github.com/allegro/bigcache/hash.go @@ -0,0 +1,8 @@ +package bigcache + +// Hasher is responsible for generating unsigned, 64 bit hash of provided string. Hasher should minimize collisions +// (generating same hash for different strings) and while performance is also important fast functions are preferable (i.e. +// you can use FarmHash family). +type Hasher interface { + Sum64(string) uint64 +} diff --git a/vendor/github.com/allegro/bigcache/iterator.go b/vendor/github.com/allegro/bigcache/iterator.go new file mode 100644 index 0000000000..70b98d9004 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/iterator.go @@ -0,0 +1,122 @@ +package bigcache + +import "sync" + +type iteratorError string + +func (e iteratorError) Error() string { + return string(e) +} + +// ErrInvalidIteratorState is reported when iterator is in invalid state +const ErrInvalidIteratorState = iteratorError("Iterator is in invalid state. Use SetNext() to move to next position") + +// ErrCannotRetrieveEntry is reported when entry cannot be retrieved from underlying +const ErrCannotRetrieveEntry = iteratorError("Could not retrieve entry from cache") + +var emptyEntryInfo = EntryInfo{} + +// EntryInfo holds informations about entry in the cache +type EntryInfo struct { + timestamp uint64 + hash uint64 + key string + value []byte +} + +// Key returns entry's underlying key +func (e EntryInfo) Key() string { + return e.key +} + +// Hash returns entry's hash value +func (e EntryInfo) Hash() uint64 { + return e.hash +} + +// Timestamp returns entry's timestamp (time of insertion) +func (e EntryInfo) Timestamp() uint64 { + return e.timestamp +} + +// Value returns entry's underlying value +func (e EntryInfo) Value() []byte { + return e.value +} + +// EntryInfoIterator allows to iterate over entries in the cache +type EntryInfoIterator struct { + mutex sync.Mutex + cache *BigCache + currentShard int + currentIndex int + elements []uint32 + elementsCount int + valid bool +} + +// SetNext moves to next element and returns true if it exists. +func (it *EntryInfoIterator) SetNext() bool { + it.mutex.Lock() + + it.valid = false + it.currentIndex++ + + if it.elementsCount > it.currentIndex { + it.valid = true + it.mutex.Unlock() + return true + } + + for i := it.currentShard + 1; i < it.cache.config.Shards; i++ { + it.elements, it.elementsCount = it.cache.shards[i].copyKeys() + + // Non empty shard - stick with it + if it.elementsCount > 0 { + it.currentIndex = 0 + it.currentShard = i + it.valid = true + it.mutex.Unlock() + return true + } + } + it.mutex.Unlock() + return false +} + +func newIterator(cache *BigCache) *EntryInfoIterator { + elements, count := cache.shards[0].copyKeys() + + return &EntryInfoIterator{ + cache: cache, + currentShard: 0, + currentIndex: -1, + elements: elements, + elementsCount: count, + } +} + +// Value returns current value from the iterator +func (it *EntryInfoIterator) Value() (EntryInfo, error) { + it.mutex.Lock() + + if !it.valid { + it.mutex.Unlock() + return emptyEntryInfo, ErrInvalidIteratorState + } + + entry, err := it.cache.shards[it.currentShard].getEntry(int(it.elements[it.currentIndex])) + + if err != nil { + it.mutex.Unlock() + return emptyEntryInfo, ErrCannotRetrieveEntry + } + it.mutex.Unlock() + + return EntryInfo{ + timestamp: readTimestampFromEntry(entry), + hash: readHashFromEntry(entry), + key: readKeyFromEntry(entry), + value: readEntry(entry), + }, nil +} diff --git a/vendor/github.com/allegro/bigcache/logger.go b/vendor/github.com/allegro/bigcache/logger.go new file mode 100644 index 0000000000..50e84abc80 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/logger.go @@ -0,0 +1,30 @@ +package bigcache + +import ( + "log" + "os" +) + +// Logger is invoked when `Config.Verbose=true` +type Logger interface { + Printf(format string, v ...interface{}) +} + +// this is a safeguard, breaking on compile time in case +// `log.Logger` does not adhere to our `Logger` interface. +// see https://golang.org/doc/faq#guarantee_satisfies_interface +var _ Logger = &log.Logger{} + +// DefaultLogger returns a `Logger` implementation +// backed by stdlib's log +func DefaultLogger() *log.Logger { + return log.New(os.Stdout, "", log.LstdFlags) +} + +func newLogger(custom Logger) Logger { + if custom != nil { + return custom + } + + return DefaultLogger() +} diff --git a/vendor/github.com/allegro/bigcache/queue/bytes_queue.go b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go new file mode 100644 index 0000000000..0285c72cd5 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go @@ -0,0 +1,210 @@ +package queue + +import ( + "encoding/binary" + "log" + "time" +) + +const ( + // Number of bytes used to keep information about entry size + headerEntrySize = 4 + // Bytes before left margin are not used. Zero index means element does not exist in queue, useful while reading slice from index + leftMarginIndex = 1 + // Minimum empty blob size in bytes. Empty blob fills space between tail and head in additional memory allocation. + // It keeps entries indexes unchanged + minimumEmptyBlobSize = 32 + headerEntrySize +) + +// BytesQueue is a non-thread safe queue type of fifo based on bytes array. +// For every push operation index of entry is returned. It can be used to read the entry later +type BytesQueue struct { + array []byte + capacity int + maxCapacity int + head int + tail int + count int + rightMargin int + headerBuffer []byte + verbose bool + initialCapacity int +} + +type queueError struct { + message string +} + +// NewBytesQueue initialize new bytes queue. +// Initial capacity is used in bytes array allocation +// When verbose flag is set then information about memory allocation are printed +func NewBytesQueue(initialCapacity int, maxCapacity int, verbose bool) *BytesQueue { + return &BytesQueue{ + array: make([]byte, initialCapacity), + capacity: initialCapacity, + maxCapacity: maxCapacity, + headerBuffer: make([]byte, headerEntrySize), + tail: leftMarginIndex, + head: leftMarginIndex, + rightMargin: leftMarginIndex, + verbose: verbose, + initialCapacity: initialCapacity, + } +} + +// Reset removes all entries from queue +func (q *BytesQueue) Reset() { + // Just reset indexes + q.tail = leftMarginIndex + q.head = leftMarginIndex + q.rightMargin = leftMarginIndex + q.count = 0 +} + +// Push copies entry at the end of queue and moves tail pointer. Allocates more space if needed. +// Returns index for pushed data or error if maximum size queue limit is reached. +func (q *BytesQueue) Push(data []byte) (int, error) { + dataLen := len(data) + + if q.availableSpaceAfterTail() < dataLen+headerEntrySize { + if q.availableSpaceBeforeHead() >= dataLen+headerEntrySize { + q.tail = leftMarginIndex + } else if q.capacity+headerEntrySize+dataLen >= q.maxCapacity && q.maxCapacity > 0 { + return -1, &queueError{"Full queue. Maximum size limit reached."} + } else { + q.allocateAdditionalMemory(dataLen + headerEntrySize) + } + } + + index := q.tail + + q.push(data, dataLen) + + return index, nil +} + +func (q *BytesQueue) allocateAdditionalMemory(minimum int) { + start := time.Now() + if q.capacity < minimum { + q.capacity += minimum + } + q.capacity = q.capacity * 2 + if q.capacity > q.maxCapacity && q.maxCapacity > 0 { + q.capacity = q.maxCapacity + } + + oldArray := q.array + q.array = make([]byte, q.capacity) + + if leftMarginIndex != q.rightMargin { + copy(q.array, oldArray[:q.rightMargin]) + + if q.tail < q.head { + emptyBlobLen := q.head - q.tail - headerEntrySize + q.push(make([]byte, emptyBlobLen), emptyBlobLen) + q.head = leftMarginIndex + q.tail = q.rightMargin + } + } + + if q.verbose { + log.Printf("Allocated new queue in %s; Capacity: %d \n", time.Since(start), q.capacity) + } +} + +func (q *BytesQueue) push(data []byte, len int) { + binary.LittleEndian.PutUint32(q.headerBuffer, uint32(len)) + q.copy(q.headerBuffer, headerEntrySize) + + q.copy(data, len) + + if q.tail > q.head { + q.rightMargin = q.tail + } + + q.count++ +} + +func (q *BytesQueue) copy(data []byte, len int) { + q.tail += copy(q.array[q.tail:], data[:len]) +} + +// Pop reads the oldest entry from queue and moves head pointer to the next one +func (q *BytesQueue) Pop() ([]byte, error) { + data, size, err := q.peek(q.head) + if err != nil { + return nil, err + } + + q.head += headerEntrySize + size + q.count-- + + if q.head == q.rightMargin { + q.head = leftMarginIndex + if q.tail == q.rightMargin { + q.tail = leftMarginIndex + } + q.rightMargin = q.tail + } + + return data, nil +} + +// Peek reads the oldest entry from list without moving head pointer +func (q *BytesQueue) Peek() ([]byte, error) { + data, _, err := q.peek(q.head) + return data, err +} + +// Get reads entry from index +func (q *BytesQueue) Get(index int) ([]byte, error) { + data, _, err := q.peek(index) + return data, err +} + +// Capacity returns number of allocated bytes for queue +func (q *BytesQueue) Capacity() int { + return q.capacity +} + +// Len returns number of entries kept in queue +func (q *BytesQueue) Len() int { + return q.count +} + +// Error returns error message +func (e *queueError) Error() string { + return e.message +} + +func (q *BytesQueue) peek(index int) ([]byte, int, error) { + + if q.count == 0 { + return nil, 0, &queueError{"Empty queue"} + } + + if index <= 0 { + return nil, 0, &queueError{"Index must be grater than zero. Invalid index."} + } + + if index+headerEntrySize >= len(q.array) { + return nil, 0, &queueError{"Index out of range"} + } + + blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize])) + return q.array[index+headerEntrySize : index+headerEntrySize+blockSize], blockSize, nil +} + +func (q *BytesQueue) availableSpaceAfterTail() int { + if q.tail >= q.head { + return q.capacity - q.tail + } + return q.head - q.tail - minimumEmptyBlobSize +} + +func (q *BytesQueue) availableSpaceBeforeHead() int { + if q.tail >= q.head { + return q.head - leftMarginIndex - minimumEmptyBlobSize + } + return q.head - q.tail - minimumEmptyBlobSize +} diff --git a/vendor/github.com/allegro/bigcache/shard.go b/vendor/github.com/allegro/bigcache/shard.go new file mode 100644 index 0000000000..56a9fb0895 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/shard.go @@ -0,0 +1,236 @@ +package bigcache + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/allegro/bigcache/queue" +) + +type onRemoveCallback func(wrappedEntry []byte, reason RemoveReason) + +type cacheShard struct { + hashmap map[uint64]uint32 + entries queue.BytesQueue + lock sync.RWMutex + entryBuffer []byte + onRemove onRemoveCallback + + isVerbose bool + logger Logger + clock clock + lifeWindow uint64 + + stats Stats +} + +func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) { + s.lock.RLock() + itemIndex := s.hashmap[hashedKey] + + if itemIndex == 0 { + s.lock.RUnlock() + s.miss() + return nil, notFound(key) + } + + wrappedEntry, err := s.entries.Get(int(itemIndex)) + if err != nil { + s.lock.RUnlock() + s.miss() + return nil, err + } + if entryKey := readKeyFromEntry(wrappedEntry); key != entryKey { + if s.isVerbose { + s.logger.Printf("Collision detected. Both %q and %q have the same hash %x", key, entryKey, hashedKey) + } + s.lock.RUnlock() + s.collision() + return nil, notFound(key) + } + s.lock.RUnlock() + s.hit() + return readEntry(wrappedEntry), nil +} + +func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error { + currentTimestamp := uint64(s.clock.epoch()) + + s.lock.Lock() + + if previousIndex := s.hashmap[hashedKey]; previousIndex != 0 { + if previousEntry, err := s.entries.Get(int(previousIndex)); err == nil { + resetKeyFromEntry(previousEntry) + } + } + + if oldestEntry, err := s.entries.Peek(); err == nil { + s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry) + } + + w := wrapEntry(currentTimestamp, hashedKey, key, entry, &s.entryBuffer) + + for { + if index, err := s.entries.Push(w); err == nil { + s.hashmap[hashedKey] = uint32(index) + s.lock.Unlock() + return nil + } + if s.removeOldestEntry(NoSpace) != nil { + s.lock.Unlock() + return fmt.Errorf("entry is bigger than max shard size") + } + } +} + +func (s *cacheShard) del(key string, hashedKey uint64) error { + s.lock.RLock() + itemIndex := s.hashmap[hashedKey] + + if itemIndex == 0 { + s.lock.RUnlock() + s.delmiss() + return notFound(key) + } + + wrappedEntry, err := s.entries.Get(int(itemIndex)) + if err != nil { + s.lock.RUnlock() + s.delmiss() + return err + } + s.lock.RUnlock() + + s.lock.Lock() + { + delete(s.hashmap, hashedKey) + s.onRemove(wrappedEntry, Deleted) + resetKeyFromEntry(wrappedEntry) + } + s.lock.Unlock() + + s.delhit() + return nil +} + +func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool { + oldestTimestamp := readTimestampFromEntry(oldestEntry) + if currentTimestamp-oldestTimestamp > s.lifeWindow { + evict(Expired) + return true + } + return false +} + +func (s *cacheShard) cleanUp(currentTimestamp uint64) { + s.lock.Lock() + for { + if oldestEntry, err := s.entries.Peek(); err != nil { + break + } else if evicted := s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry); !evicted { + break + } + } + s.lock.Unlock() +} + +func (s *cacheShard) getOldestEntry() ([]byte, error) { + return s.entries.Peek() +} + +func (s *cacheShard) getEntry(index int) ([]byte, error) { + return s.entries.Get(index) +} + +func (s *cacheShard) copyKeys() (keys []uint32, next int) { + keys = make([]uint32, len(s.hashmap)) + + s.lock.RLock() + + for _, index := range s.hashmap { + keys[next] = index + next++ + } + + s.lock.RUnlock() + return keys, next +} + +func (s *cacheShard) removeOldestEntry(reason RemoveReason) error { + oldest, err := s.entries.Pop() + if err == nil { + hash := readHashFromEntry(oldest) + delete(s.hashmap, hash) + s.onRemove(oldest, reason) + return nil + } + return err +} + +func (s *cacheShard) reset(config Config) { + s.lock.Lock() + s.hashmap = make(map[uint64]uint32, config.initialShardSize()) + s.entryBuffer = make([]byte, config.MaxEntrySize+headersSizeInBytes) + s.entries.Reset() + s.lock.Unlock() +} + +func (s *cacheShard) len() int { + s.lock.RLock() + res := len(s.hashmap) + s.lock.RUnlock() + return res +} + +func (s *cacheShard) capacity() int { + s.lock.RLock() + res := s.entries.Capacity() + s.lock.RUnlock() + return res +} + +func (s *cacheShard) getStats() Stats { + var stats = Stats{ + Hits: atomic.LoadInt64(&s.stats.Hits), + Misses: atomic.LoadInt64(&s.stats.Misses), + DelHits: atomic.LoadInt64(&s.stats.DelHits), + DelMisses: atomic.LoadInt64(&s.stats.DelMisses), + Collisions: atomic.LoadInt64(&s.stats.Collisions), + } + return stats +} + +func (s *cacheShard) hit() { + atomic.AddInt64(&s.stats.Hits, 1) +} + +func (s *cacheShard) miss() { + atomic.AddInt64(&s.stats.Misses, 1) +} + +func (s *cacheShard) delhit() { + atomic.AddInt64(&s.stats.DelHits, 1) +} + +func (s *cacheShard) delmiss() { + atomic.AddInt64(&s.stats.DelMisses, 1) +} + +func (s *cacheShard) collision() { + atomic.AddInt64(&s.stats.Collisions, 1) +} + +func initNewShard(config Config, callback onRemoveCallback, clock clock) *cacheShard { + return &cacheShard{ + hashmap: make(map[uint64]uint32, config.initialShardSize()), + entries: *queue.NewBytesQueue(config.initialShardSize()*config.MaxEntrySize, config.maximumShardSize(), config.Verbose), + entryBuffer: make([]byte, config.MaxEntrySize+headersSizeInBytes), + onRemove: callback, + + isVerbose: config.Verbose, + logger: newLogger(config.Logger), + clock: clock, + lifeWindow: uint64(config.LifeWindow.Seconds()), + } +} diff --git a/vendor/github.com/allegro/bigcache/stats.go b/vendor/github.com/allegro/bigcache/stats.go new file mode 100644 index 0000000000..07157132a2 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/stats.go @@ -0,0 +1,15 @@ +package bigcache + +// Stats stores cache statistics +type Stats struct { + // Hits is a number of successfully found keys + Hits int64 `json:"hits"` + // Misses is a number of not found keys + Misses int64 `json:"misses"` + // DelHits is a number of successfully deleted keys + DelHits int64 `json:"delete_hits"` + // DelMisses is a number of not deleted keys + DelMisses int64 `json:"delete_misses"` + // Collisions is a number of happened key-collisions + Collisions int64 `json:"collisions"` +} diff --git a/vendor/github.com/allegro/bigcache/utils.go b/vendor/github.com/allegro/bigcache/utils.go new file mode 100644 index 0000000000..ca1df79b93 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/utils.go @@ -0,0 +1,16 @@ +package bigcache + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func convertMBToBytes(value int) int { + return value * 1024 * 1024 +} + +func isPowerOfTwo(number int) bool { + return (number & (number - 1)) == 0 +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 06268535e9..1bfe09da72 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -38,6 +38,18 @@ "revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338", "revisionTime": "2018-01-16T20:38:02Z" }, + { + "checksumSHA1": "9Niiu1GNhWUrXnGZrl8AU4EzbVE=", + "path": "github.com/allegro/bigcache", + "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca", + "revisionTime": "2018-10-22T20:06:25Z" + }, + { + "checksumSHA1": "zqToN+R6KybEskp1D4G/lAOKXU4=", + "path": "github.com/allegro/bigcache/queue", + "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca", + "revisionTime": "2018-10-22T20:06:25Z" + }, { "checksumSHA1": "USkefO0g1U9mr+8hagv3fpSkrxg=", "path": "github.com/aristanetworks/goarista/monotime", From a6942b9f25bd6c67a5ab9e80ab95aae25a08da6d Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 15 Nov 2018 14:57:03 +0100 Subject: [PATCH 081/100] swarm/storage: Batched database migration (#18113) --- swarm/storage/ldbstore.go | 139 ++++++++++++++++++++++++--------- swarm/storage/ldbstore_test.go | 32 ++++++++ 2 files changed, 135 insertions(+), 36 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index fbae59facf..bd4f6b9162 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -284,7 +284,7 @@ func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { return val } -func parseGCIdxKey(key []byte) (byte, []byte) { +func parseIdxKey(key []byte) (byte, []byte) { return key[0], key[1:] } @@ -589,7 +589,7 @@ func (s *LDBStore) CleanGCIndex() error { it.Seek([]byte{keyGCIdx}) var gcDeletes int for it.Valid() { - rowType, _ := parseGCIdxKey(it.Key()) + rowType, _ := parseIdxKey(it.Key()) if rowType != keyGCIdx { break } @@ -601,47 +601,113 @@ func (s *LDBStore) CleanGCIndex() error { if err := s.db.Write(&batch); err != nil { return err } - - it.Seek([]byte{keyIndex}) - var idx dpaDBIndex - var poPtrs [256]uint64 - for it.Valid() { - rowType, chunkHash := parseGCIdxKey(it.Key()) - if rowType != keyIndex { - break - } - err := decodeIndex(it.Value(), &idx) - if err != nil { - return fmt.Errorf("corrupt index: %v", err) - } - po := s.po(chunkHash) - - // if we don't find the data key, remove the entry - dataKey := getDataKey(idx.Idx, po) - _, err = s.db.Get(dataKey) - if err != nil { - log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) - batch.Delete(it.Key()) - } else { - gcIdxKey := getGCIdxKey(&idx) - gcIdxData := getGCIdxValue(&idx, po, chunkHash) - batch.Put(gcIdxKey, gcIdxData) - log.Trace("clean ok", "key", chunkHash, "gcKey", gcIdxKey, "gcData", gcIdxData) - okEntryCount++ - if idx.Idx > poPtrs[po] { - poPtrs[po] = idx.Idx - } - } - totalEntryCount++ - it.Next() - } + batch.Reset() it.Release() + + // corrected po index pointer values + var poPtrs [256]uint64 + + // set to true if chunk count not on 4096 iteration boundary + var doneIterating bool + + // last key index in previous iteration + lastIdxKey := []byte{keyIndex} + + // counter for debug output + var cleanBatchCount int + + // go through all key index entries + for !doneIterating { + cleanBatchCount++ + var idxs []dpaDBIndex + var chunkHashes [][]byte + var pos []uint8 + it := s.db.NewIterator() + + it.Seek(lastIdxKey) + + // 4096 is just a nice number, don't look for any hidden meaning here... + var i int + for i = 0; i < 4096; i++ { + + // this really shouldn't happen unless database is empty + // but let's keep it to be safe + if !it.Valid() { + doneIterating = true + break + } + + // if it's not keyindex anymore we're done iterating + rowType, chunkHash := parseIdxKey(it.Key()) + if rowType != keyIndex { + doneIterating = true + break + } + + // decode the retrieved index + var idx dpaDBIndex + err := decodeIndex(it.Value(), &idx) + if err != nil { + return fmt.Errorf("corrupt index: %v", err) + } + po := s.po(chunkHash) + lastIdxKey = it.Key() + + // if we don't find the data key, remove the entry + // if we find it, add to the array of new gc indices to create + dataKey := getDataKey(idx.Idx, po) + _, err = s.db.Get(dataKey) + if err != nil { + log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) + batch.Delete(it.Key()) + } else { + idxs = append(idxs, idx) + chunkHashes = append(chunkHashes, chunkHash) + pos = append(pos, po) + okEntryCount++ + if idx.Idx > poPtrs[po] { + poPtrs[po] = idx.Idx + } + } + totalEntryCount++ + it.Next() + } + it.Release() + + // flush the key index corrections + err := s.db.Write(&batch) + if err != nil { + return err + } + batch.Reset() + + // add correct gc indices + for i, okIdx := range idxs { + gcIdxKey := getGCIdxKey(&okIdx) + gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i]) + batch.Put(gcIdxKey, gcIdxData) + log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData) + } + + // flush them + err = s.db.Write(&batch) + if err != nil { + return err + } + batch.Reset() + + log.Debug("clean gc index pass", "batch", cleanBatchCount, "checked", i, "kept", len(idxs)) + } + log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len()) + // lastly add updated entry count var entryCount [8]byte binary.BigEndian.PutUint64(entryCount[:], okEntryCount) batch.Put(keyEntryCnt, entryCount[:]) + + // and add the new po index pointers var poKey [2]byte poKey[0] = keyDistanceCnt for i, poPtr := range poPtrs { @@ -655,6 +721,7 @@ func (s *LDBStore) CleanGCIndex() error { } } + // if you made it this far your harddisk has survived. Congratulations return s.db.Write(&batch) } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 07557980c6..9c10b36290 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -761,6 +761,38 @@ func TestCleanIndex(t *testing.T) { t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal) } } + + // check that the iterator quits properly + chunks, err = mputRandomChunks(ldb, 4100, 4096) + if err != nil { + t.Fatal(err) + } + + po = ldb.po(chunks[4099].Address()[:]) + dataKey = make([]byte, 10) + dataKey[0] = keyData + dataKey[1] = byte(po) + binary.BigEndian.PutUint64(dataKey[2:], 4099+3) + if _, err := ldb.db.Get(dataKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Delete(dataKey); err != nil { + t.Fatal(err) + } + + if err := ldb.CleanGCIndex(); err != nil { + t.Fatal(err) + } + + // entrycount should now be one less of added chunks + c, err = ldb.db.Get(keyEntryCnt) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + entryCount = binary.BigEndian.Uint64(c) + if entryCount != 4099+2 { + t.Fatalf("expected entrycnt to be 2, was %d", c) + } } func waitGc(ctx context.Context, ldb *LDBStore) { From b91766fe6db6e484f2fd75f617307a48ff85600e Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Thu, 15 Nov 2018 22:31:24 +0800 Subject: [PATCH 082/100] eth: fix comment typo (#18114) * consensus/clique: fix comment typo * eth,eth/downloader: fix comment typo --- eth/downloader/queue.go | 2 +- eth/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 863cc8de14..7c33953811 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -325,7 +325,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { } // Make sure no duplicate requests are executed if _, ok := q.blockTaskPool[hash]; ok { - log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) + log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) continue } if _, ok := q.receiptTaskPool[hash]; ok { diff --git a/eth/handler.go b/eth/handler.go index bd227a84e0..741fc9d5a5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -658,7 +658,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { p.SetHead(trueHead, trueTD) // Schedule a sync if above ours. Note, this will not fire a sync for a gap of - // a singe block (as the true TD is below the propagated block), however this + // a single block (as the true TD is below the propagated block), however this // scenario should easily be covered by the fetcher. currentBlock := pm.blockchain.CurrentBlock() if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 { From 324027640bcaf137b8c9e96bc26f0833711497af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 15 Nov 2018 21:06:27 +0100 Subject: [PATCH 083/100] swarm/network/simulation: use simulations.Event instead p2p.PeerEvent (#18098) --- swarm/network/simulation/events.go | 114 +++++++++++++++------ swarm/network/simulation/example_test.go | 13 ++- swarm/network/stream/delivery_test.go | 8 +- swarm/network/stream/intervals_test.go | 5 +- swarm/network/stream/snapshot_sync_test.go | 11 +- swarm/network/stream/syncer_test.go | 5 +- 6 files changed, 101 insertions(+), 55 deletions(-) diff --git a/swarm/network/simulation/events.go b/swarm/network/simulation/events.go index 594d36225c..d73c3af4ee 100644 --- a/swarm/network/simulation/events.go +++ b/swarm/network/simulation/events.go @@ -20,16 +20,18 @@ import ( "context" "sync" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" ) // PeerEvent is the type of the channel returned by Simulation.PeerEvents. type PeerEvent struct { // NodeID is the ID of node that the event is caught on. NodeID enode.ID + // PeerID is the ID of the peer node that the event is caught on. + PeerID enode.ID // Event is the event that is caught. - Event *p2p.PeerEvent + Event *simulations.Event // Error is the error that may have happened during event watching. Error error } @@ -37,9 +39,13 @@ type PeerEvent struct { // PeerEventsFilter defines a filter on PeerEvents to exclude messages with // defined properties. Use PeerEventsFilter methods to set required options. type PeerEventsFilter struct { - t *p2p.PeerEventType - protocol *string - msgCode *uint64 + eventType simulations.EventType + + connUp *bool + + msgReceive *bool + protocol *string + msgCode *uint64 } // NewPeerEventsFilter returns a new PeerEventsFilter instance. @@ -47,20 +53,48 @@ func NewPeerEventsFilter() *PeerEventsFilter { return &PeerEventsFilter{} } -// Type sets the filter to only one peer event type. -func (f *PeerEventsFilter) Type(t p2p.PeerEventType) *PeerEventsFilter { - f.t = &t +// Connect sets the filter to events when two nodes connect. +func (f *PeerEventsFilter) Connect() *PeerEventsFilter { + f.eventType = simulations.EventTypeConn + b := true + f.connUp = &b + return f +} + +// Drop sets the filter to events when two nodes disconnect. +func (f *PeerEventsFilter) Drop() *PeerEventsFilter { + f.eventType = simulations.EventTypeConn + b := false + f.connUp = &b + return f +} + +// ReceivedMessages sets the filter to only messages that are received. +func (f *PeerEventsFilter) ReceivedMessages() *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg + b := true + f.msgReceive = &b + return f +} + +// SentMessages sets the filter to only messages that are sent. +func (f *PeerEventsFilter) SentMessages() *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg + b := false + f.msgReceive = &b return f } // Protocol sets the filter to only one message protocol. func (f *PeerEventsFilter) Protocol(p string) *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg f.protocol = &p return f } // MsgCode sets the filter to only one msg code. func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg f.msgCode = &c return f } @@ -80,19 +114,8 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ... go func(id enode.ID) { defer s.shutdownWG.Done() - client, err := s.Net.GetNode(id).Client() - if err != nil { - subsWG.Done() - eventC <- PeerEvent{NodeID: id, Error: err} - return - } - events := make(chan *p2p.PeerEvent) - sub, err := client.Subscribe(ctx, "admin", events, "peerEvents") - if err != nil { - subsWG.Done() - eventC <- PeerEvent{NodeID: id, Error: err} - return - } + events := make(chan *simulations.Event) + sub := s.Net.Events().Subscribe(events) defer sub.Unsubscribe() subsWG.Done() @@ -110,28 +133,55 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ... case <-s.Done(): return case e := <-events: + // ignore control events + if e.Control { + continue + } match := len(filters) == 0 // if there are no filters match all events for _, f := range filters { - if f.t != nil && *f.t != e.Type { - continue + if f.eventType == simulations.EventTypeConn && e.Conn != nil { + if *f.connUp != e.Conn.Up { + continue + } + // all connection filter parameters matched, break the loop + match = true + break } - if f.protocol != nil && *f.protocol != e.Protocol { - continue + if f.eventType == simulations.EventTypeMsg && e.Msg != nil { + if f.msgReceive != nil && *f.msgReceive != e.Msg.Received { + continue + } + if f.protocol != nil && *f.protocol != e.Msg.Protocol { + continue + } + if f.msgCode != nil && *f.msgCode != e.Msg.Code { + continue + } + // all message filter parameters matched, break the loop + match = true + break } - if f.msgCode != nil && e.MsgCode != nil && *f.msgCode != *e.MsgCode { - continue + } + var peerID enode.ID + switch e.Type { + case simulations.EventTypeConn: + peerID = e.Conn.One + if peerID == id { + peerID = e.Conn.Other + } + case simulations.EventTypeMsg: + peerID = e.Msg.One + if peerID == id { + peerID = e.Msg.Other } - // all filter parameters matched, break the loop - match = true - break } if match { select { - case eventC <- PeerEvent{NodeID: id, Event: e}: + case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Event: e}: case <-ctx.Done(): if err := ctx.Err(); err != nil { select { - case eventC <- PeerEvent{NodeID: id, Error: err}: + case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Error: err}: case <-s.Done(): } } diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index 84b0634b45..bacc64d53a 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" @@ -87,7 +86,7 @@ func ExampleSimulation_PeerEvents() { log.Error("peer event", "err", e.Error) continue } - log.Info("peer event", "node", e.NodeID, "peer", e.Event.Peer, "msgcode", e.Event.MsgCode) + log.Info("peer event", "node", e.NodeID, "peer", e.PeerID, "type", e.Event.Type) } }() } @@ -100,7 +99,7 @@ func ExampleSimulation_PeerEvents_disconnections() { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { @@ -109,7 +108,7 @@ func ExampleSimulation_PeerEvents_disconnections() { log.Error("peer drop", "err", d.Error) continue } - log.Warn("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Warn("peer drop", "node", d.NodeID, "peer", d.PeerID) } }() } @@ -124,8 +123,8 @@ func ExampleSimulation_PeerEvents_multipleFilters() { context.Background(), sim.NodeIDs(), // Watch when bzz messages 1 and 4 are received. - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(1), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(4), + simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(1), + simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(4), ) go func() { @@ -134,7 +133,7 @@ func ExampleSimulation_PeerEvents_multipleFilters() { log.Error("bzz message", "err", m.Error) continue } - log.Info("bzz message", "node", m.NodeID, "peer", m.Event.Peer) + log.Info("bzz message", "node", m.NodeID, "peer", m.PeerID) } }() } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index c9a530115c..6b6025115e 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -565,13 +565,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } @@ -697,13 +697,13 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) b.Fatal(d.Error) } } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 037984f220..b9525d4a49 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" @@ -154,7 +153,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) @@ -165,7 +164,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 4bd7f38f5b..96c37bddcb 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" @@ -210,12 +209,12 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal("unexpected disconnect") cancelSimRun() } @@ -402,12 +401,12 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal("unexpected disconnect") cancelSimRun() } @@ -428,7 +427,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) var subscriptionCount int - filter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(4) + filter := simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("stream").MsgCode(4) eventC := sim.PeerEvents(ctx, nodeIDs, filter) for j, node := range nodeIDs { diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index a543cae058..f4e0554511 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/log" @@ -151,13 +150,13 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } From ffe2fc3bc4d77ad3f503d2bc1cdd62eac8d03c5b Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 15 Nov 2018 17:41:19 -0500 Subject: [PATCH 084/100] Swarm accounting (#18050) * swarm: completed 1st phase of swap accounting * swarm: swap accounting for swarm with p2p accounting * swarm/swap: addressed PR comments * swarm/swap: ignore ErrNotFound on stateStore.Get() * swarm/swap: GetPeerBalance test; add TODO for chequebook API check * swarm/network/stream: fix NewRegistry calls with new arguments * swarm/swap: address @justelad's PR comments --- swarm/network/stream/common_test.go | 2 +- swarm/network/stream/delivery_test.go | 8 +- swarm/network/stream/intervals_test.go | 2 +- .../network/stream/snapshot_retrieval_test.go | 2 +- swarm/network/stream/snapshot_sync_test.go | 4 +- swarm/network/stream/stream.go | 76 +++++--- swarm/network/stream/syncer_test.go | 2 +- swarm/swap/swap.go | 93 +++++++++ swarm/swap/swap_test.go | 184 ++++++++++++++++++ swarm/swarm.go | 18 +- 10 files changed, 353 insertions(+), 38 deletions(-) create mode 100644 swarm/swap/swap.go create mode 100644 swarm/swap/swap_test.go diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 721b873b78..c5f1fa176a 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -114,7 +114,7 @@ func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest delivery := NewDelivery(to, netStore) netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions) + streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil) teardown := func() { streamer.Close() removeDataDir() diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 6b6025115e..a6173a3892 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -290,7 +290,7 @@ func TestRequestFromPeers(t *testing.T) { Peer: protocolsPeer, }, to) to.On(peer) - r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil) // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished sp := &Peer{ @@ -331,7 +331,7 @@ func TestRequestFromPeersWithLightNode(t *testing.T) { Peer: protocolsPeer, }, to) to.On(peer) - r := NewRegistry(addr.ID(), delivery, nil, nil, nil) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil) // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished sp := &Peer{ Peer: protocolsPeer, @@ -480,7 +480,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck SkipCheck: skipCheck, Syncing: SyncingDisabled, Retrieval: RetrievalEnabled, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) @@ -655,7 +655,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip Syncing: SyncingDisabled, Retrieval: RetrievalDisabled, SyncUpdateDelay: 0, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index b9525d4a49..defb6df50a 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -84,7 +84,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { Retrieval: RetrievalDisabled, Syncing: SyncingRegisterOnly, SkipCheck: skipCheck, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) { diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index ad15193414..5ea0b1511b 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -130,7 +130,7 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no Retrieval: RetrievalEnabled, Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 96c37bddcb..6b92c32ae9 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -166,7 +166,7 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic Retrieval: RetrievalDisabled, Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) @@ -360,7 +360,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingRegisterOnly, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 695ff0c502..32e107823b 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "math" + "reflect" "sync" "time" @@ -87,6 +88,9 @@ type Registry struct { intervalsStore state.Store autoRetrieval bool //automatically subscribe to retrieve request stream maxPeerServers int + spec *protocols.Spec //this protocol's spec + balance protocols.Balance //implements protocols.Balance, for accounting + prices protocols.Prices //implements protocols.Prices, provides prices to accounting } // RegistryOptions holds optional values for NewRegistry constructor. @@ -99,7 +103,7 @@ type RegistryOptions struct { } // NewRegistry is Streamer constructor -func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry { +func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions, balance protocols.Balance) *Registry { if options == nil { options = &RegistryOptions{} } @@ -119,7 +123,10 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy intervalsStore: intervalsStore, autoRetrieval: retrieval, maxPeerServers: options.MaxPeerServers, + balance: balance, } + streamer.setupSpec() + streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer @@ -228,6 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy return streamer } +//we need to construct a spec instance per node instance +func (r *Registry) setupSpec() { + //first create the "bare" spec + r.createSpec() + //if balance is nil, this node has been started without swap support (swapEnabled flag is false) + if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() { + //swap is enabled, so setup the hook + r.spec.Hook = protocols.NewAccounting(r.balance, r.prices) + } +} + // RegisterClient registers an incoming streamer constructor func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) { r.clientMu.Lock() @@ -492,7 +510,7 @@ func (r *Registry) updateSyncing() { } func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := protocols.NewPeer(p, rw, Spec) + peer := protocols.NewPeer(p, rw, r.spec) bp := network.NewBzzPeer(peer) np := network.NewPeer(bp, r.delivery.kad) r.delivery.kad.On(np) @@ -716,35 +734,43 @@ func (c *clientParams) clientCreated() { close(c.clientCreatedC) } -// Spec is the spec of the streamer protocol -var Spec = &protocols.Spec{ - Name: "stream", - Version: 8, - MaxMsgSize: 10 * 1024 * 1024, - Messages: []interface{}{ - UnsubscribeMsg{}, - OfferedHashesMsg{}, - WantedHashesMsg{}, - TakeoverProofMsg{}, - SubscribeMsg{}, - RetrieveRequestMsg{}, - ChunkDeliveryMsgRetrieval{}, - SubscribeErrorMsg{}, - RequestSubscriptionMsg{}, - QuitMsg{}, - ChunkDeliveryMsgSyncing{}, - }, +//GetSpec returns the streamer spec to callers +//This used to be a global variable but for simulations with +//multiple nodes its fields (notably the Hook) would be overwritten +func (r *Registry) GetSpec() *protocols.Spec { + return r.spec +} + +func (r *Registry) createSpec() { + // Spec is the spec of the streamer protocol + var spec = &protocols.Spec{ + Name: "stream", + Version: 8, + MaxMsgSize: 10 * 1024 * 1024, + Messages: []interface{}{ + UnsubscribeMsg{}, + OfferedHashesMsg{}, + WantedHashesMsg{}, + TakeoverProofMsg{}, + SubscribeMsg{}, + RetrieveRequestMsg{}, + ChunkDeliveryMsgRetrieval{}, + SubscribeErrorMsg{}, + RequestSubscriptionMsg{}, + QuitMsg{}, + ChunkDeliveryMsgSyncing{}, + }, + } + r.spec = spec } func (r *Registry) Protocols() []p2p.Protocol { return []p2p.Protocol{ { - Name: Spec.Name, - Version: Spec.Version, - Length: Spec.Length(), + Name: r.spec.Name, + Version: r.spec.Version, + Length: r.spec.Length(), Run: r.runProtocol, - // NodeInfo: , - // PeerInfo: , }, } } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index f4e0554511..fe20bab266 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -118,7 +118,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck Retrieval: RetrievalDisabled, Syncing: SyncingAutoSubscribe, SkipCheck: skipCheck, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/swap/swap.go b/swarm/swap/swap.go new file mode 100644 index 0000000000..137eb141d9 --- /dev/null +++ b/swarm/swap/swap.go @@ -0,0 +1,93 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swap + +import ( + "errors" + "fmt" + "strconv" + "sync" + + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/state" +) + +// SwAP Swarm Accounting Protocol +// a peer to peer micropayment system +// A node maintains an individual balance with every peer +// Only messages which have a price will be accounted for +type Swap struct { + stateStore state.Store //stateStore is needed in order to keep balances across sessions + lock sync.RWMutex //lock the balances + balances map[enode.ID]int64 //map of balances for each peer +} + +// New - swap constructor +func New(stateStore state.Store) (swap *Swap) { + swap = &Swap{ + stateStore: stateStore, + balances: make(map[enode.ID]int64), + } + return +} + +//Swap implements the protocols.Balance interface +//Add is the (sole) accounting function +func (s *Swap) Add(amount int64, peer *protocols.Peer) (err error) { + s.lock.Lock() + defer s.lock.Unlock() + + //load existing balances from the state store + err = s.loadState(peer) + if err != nil && err != state.ErrNotFound { + return + } + //adjust the balance + //if amount is negative, it will decrease, otherwise increase + s.balances[peer.ID()] += amount + //save the new balance to the state store + peerBalance := s.balances[peer.ID()] + err = s.stateStore.Put(peer.ID().String(), &peerBalance) + + log.Debug(fmt.Sprintf("balance for peer %s: %s", peer.ID().String(), strconv.FormatInt(peerBalance, 10))) + return err +} + +//GetPeerBalance returns the balance for a given peer +func (swap *Swap) GetPeerBalance(peer enode.ID) (int64, error) { + swap.lock.RLock() + defer swap.lock.RUnlock() + if p, ok := swap.balances[peer]; ok { + return p, nil + } + return 0, errors.New("Peer not found") +} + +//load balances from the state store (persisted) +func (s *Swap) loadState(peer *protocols.Peer) (err error) { + var peerBalance int64 + peerID := peer.ID() + //only load if the current instance doesn't already have this peer's + //balance in memory + if _, ok := s.balances[peerID]; !ok { + err = s.stateStore.Get(peerID.String(), &peerBalance) + s.balances[peerID] = peerBalance + } + return +} diff --git a/swarm/swap/swap_test.go b/swarm/swap/swap_test.go new file mode 100644 index 0000000000..f2e3ba168a --- /dev/null +++ b/swarm/swap/swap_test.go @@ -0,0 +1,184 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swap + +import ( + "flag" + "fmt" + "io/ioutil" + mrand "math/rand" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/state" + colorable "github.com/mattn/go-colorable" +) + +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + mrand.Seed(time.Now().UnixNano()) + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + +//Test getting a peer's balance +func TestGetPeerBalance(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + //test for correct value + testPeer := newDummyPeer() + swap.balances[testPeer.ID()] = 888 + b, err := swap.GetPeerBalance(testPeer.ID()) + if err != nil { + t.Fatal(err) + } + if b != 888 { + t.Fatalf("Expected peer's balance to be %d, but is %d", 888, b) + } + + //test for inexistent node + id := adapters.RandomNodeConfig().ID + _, err = swap.GetPeerBalance(id) + if err == nil { + t.Fatal("Expected call to fail, but it didn't!") + } + if err.Error() != "Peer not found" { + t.Fatalf("Expected test to fail with %s, but is %s", "Peer not found", err.Error()) + } +} + +//Test that repeated bookings do correct accounting +func TestRepeatedBookings(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + testPeer := newDummyPeer() + amount := mrand.Intn(100) + cnt := 1 + mrand.Intn(10) + for i := 0; i < cnt; i++ { + swap.Add(int64(amount), testPeer.Peer) + } + expectedBalance := int64(cnt * amount) + realBalance := swap.balances[testPeer.ID()] + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After %d credits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance)) + } + + testPeer2 := newDummyPeer() + amount = mrand.Intn(100) + cnt = 1 + mrand.Intn(10) + for i := 0; i < cnt; i++ { + swap.Add(0-int64(amount), testPeer2.Peer) + } + expectedBalance = int64(0 - (cnt * amount)) + realBalance = swap.balances[testPeer2.ID()] + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After %d debits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance)) + } + + //mixed debits and credits + amount1 := mrand.Intn(100) + amount2 := mrand.Intn(55) + amount3 := mrand.Intn(999) + swap.Add(int64(amount1), testPeer2.Peer) + swap.Add(int64(0-amount2), testPeer2.Peer) + swap.Add(int64(0-amount3), testPeer2.Peer) + + expectedBalance = expectedBalance + int64(amount1-amount2-amount3) + realBalance = swap.balances[testPeer2.ID()] + + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After mixed debits and credits, expected balance to be: %d, but is: %d", expectedBalance, realBalance)) + } +} + +//try restoring a balance from state store +//this is simulated by creating a node, +//assigning it an arbitrary balance, +//then closing the state store. +//Then we re-open the state store and check that +//the balance is still the same +func TestRestoreBalanceFromStateStore(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + testPeer := newDummyPeer() + swap.balances[testPeer.ID()] = -8888 + + tmpBalance := swap.balances[testPeer.ID()] + swap.stateStore.Put(testPeer.ID().String(), &tmpBalance) + + swap.stateStore.Close() + swap.stateStore = nil + + stateStore, err := state.NewDBStore(testDir) + if err != nil { + t.Fatal(err) + } + + var newBalance int64 + stateStore.Get(testPeer.ID().String(), &newBalance) + + //compare the balances + if tmpBalance != newBalance { + t.Fatal(fmt.Sprintf("Unexpected balance value after sending cheap message test. Expected balance: %d, balance is: %d", + tmpBalance, newBalance)) + } +} + +//create a test swap account +//creates a stateStore for persistence and a Swap account +func createTestSwap(t *testing.T) (*Swap, string) { + dir, err := ioutil.TempDir("", "swap_test_store") + if err != nil { + t.Fatal(err) + } + stateStore, err2 := state.NewDBStore(dir) + if err2 != nil { + t.Fatal(err2) + } + swap := New(stateStore) + return swap, dir +} + +type dummyPeer struct { + *protocols.Peer +} + +//creates a dummy protocols.Peer with dummy MsgReadWriter +func newDummyPeer() *dummyPeer { + id := adapters.RandomNodeConfig().ID + protoPeer := protocols.NewPeer(p2p.NewPeer(id, "testPeer", nil), nil, nil) + dummy := &dummyPeer{ + Peer: protoPeer, + } + return dummy +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 1fb5443fd4..dc3756d3af 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -51,6 +51,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/swap" "github.com/ethereum/go-ethereum/swarm/tracing" ) @@ -78,6 +79,7 @@ type Swarm struct { netStore *storage.NetStore sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit ps *pss.Pss + swap *swap.Swap tracerClose io.Closer } @@ -171,6 +173,14 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e delivery := stream.NewDelivery(to, self.netStore) self.netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, config.DeliverySkipCheck).New + if config.SwapEnabled { + balancesStore, err := state.NewDBStore(filepath.Join(config.Path, "balances.db")) + if err != nil { + return nil, err + } + self.swap = swap.New(balancesStore) + } + var nodeID enode.ID if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil { return nil, err @@ -193,7 +203,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e SyncUpdateDelay: config.SyncUpdateDelay, MaxPeerServers: config.MaxStreamPeerServers, } - self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions) + self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions, self.swap) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) @@ -216,7 +226,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e log.Debug("Setup local storage") - self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) + self.bzz = network.NewBzz(bzzconfig, to, stateStore, self.streamer.GetSpec(), self.streamer.Run) // Pss = postal service over swarm (devp2p over bzz) self.ps, err = pss.NewPss(to, config.Pss) @@ -353,7 +363,9 @@ func (self *Swarm) Start(srv *p2p.Server) error { newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String())) log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) // set chequebook - if self.config.SwapEnabled { + //TODO: Currently if swap is enabled and no chequebook (or inexistent) contract is provided, the node would crash. + //Once we integrate back the contracts, this check MUST be revisited + if self.config.SwapEnabled && self.config.SwapAPI != "" { ctx := context.Background() // The initial setup has no deadline. err := self.SetChequebook(ctx) if err != nil { From 68be45e5f89a5bde33376a8e103c133392f18516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Kurowski?= Date: Fri, 16 Nov 2018 10:50:48 +0100 Subject: [PATCH 085/100] trie: return hasher to pool (#18116) * trie: return hasher to pool * trie: minor code formatting fix --- trie/iterator.go | 2 ++ trie/proof.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/trie/iterator.go b/trie/iterator.go index 00b890eb89..77f1681665 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -181,6 +181,8 @@ func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { hasher := newHasher(0, 0, nil) + defer returnHasherToPool(hasher) + proofs := make([][]byte, 0, len(it.stack)) for i, item := range it.stack[:len(it.stack)-1] { diff --git a/trie/proof.go b/trie/proof.go index 6cb8f4d5f7..62c47beda2 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -66,6 +66,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { } } hasher := newHasher(0, 0, nil) + defer returnHasherToPool(hasher) + for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. From 51b2f1620cc22aa410dee24a6021bb767e0ff998 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 12 Nov 2018 14:18:56 +0100 Subject: [PATCH 086/100] downloader: different sync strategy --- eth/downloader/downloader.go | 122 ++++++++++++++++++++---------- eth/downloader/downloader_test.go | 76 +++++++++++++++++++ 2 files changed, 157 insertions(+), 41 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 56c54c8ed5..1f52fc6db4 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -430,7 +430,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I } height := latest.Number.Uint64() - origin, err := d.findAncestor(p, height) + origin, err := d.findAncestor(p, latest) if err != nil { return err } @@ -587,41 +587,85 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) { } } +// calculateRequestSpan calculates what headers to request from a peer when trying to determine the +// common ancestor. +// It returns parameters to be used for peer.RequestHeadersByNumber: +// from - starting block number +// count - number of headers to request +// skip - number of headers to skip +// and also returns 'max', the last block which is expected to be returned by the remote peers, +// given the (from,count,skip) +func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { + var ( + from int + count int + MaxCount = MaxHeaderFetch / 16 + ) + // requestHead is the highest block that we will ask for. If requestHead is not offset, + // the highest block that we will get is 16 blocks back from head, which means we + // will fetch 14 or 15 blocks unnecessarily in the case the height difference + // between us and the peer is 1-2 blocks, which is most common + requestHead := int(remoteHeight) - 1 + if requestHead < 0 { + requestHead = 0 + } + // requestBottom is the lowest block we want included in the query + // Ideally, we want to include just below own head + requestBottom := int(localHeight - 1) + if requestBottom < 0 { + requestBottom = 0 + } + totalSpan := requestHead - requestBottom + span := 1 + totalSpan/MaxCount + if span < 2 { + span = 2 + } + if span > 16 { + span = 16 + } + + count = 1 + totalSpan/span + if count > MaxCount { + count = MaxCount + } + if count < 2 { + count = 2 + } + from = requestHead - (count-1)*span + if from < 0 { + from = 0 + } + max := from + (count-1)*span + return int64(from), count, span - 1, uint64(max) +} + // findAncestor tries to locate the common ancestor link of the local chain and // a remote peers blockchain. In the general case when our node was in sync and // on the correct chain, checking the top N links should already get us a match. // In the rare scenario when we ended up on a long reorganisation (i.e. none of // the head links match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) { +func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) { // Figure out the valid ancestor range to prevent rewrite attacks - floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64() + var ( + floor = int64(-1) + localHeight uint64 + remoteHeight = remoteHeader.Number.Uint64() + ) + switch d.mode { + case FullSync: + localHeight = d.blockchain.CurrentBlock().NumberU64() + case FastSync: + localHeight = d.blockchain.CurrentFastBlock().NumberU64() + default: + localHeight = d.lightchain.CurrentHeader().Number.Uint64() + } + p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) + if localHeight >= MaxForkAncestry { + floor = int64(localHeight - MaxForkAncestry) + } - if d.mode == FullSync { - ceil = d.blockchain.CurrentBlock().NumberU64() - } else if d.mode == FastSync { - ceil = d.blockchain.CurrentFastBlock().NumberU64() - } - if ceil >= MaxForkAncestry { - floor = int64(ceil - MaxForkAncestry) - } - p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) - - // Request the topmost blocks to short circuit binary ancestor lookup - head := ceil - if head > height { - head = height - } - from := int64(head) - int64(MaxHeaderFetch) - if from < 0 { - from = 0 - } - // Span out with 15 block gaps into the future to catch bad head reports - limit := 2 * MaxHeaderFetch / 16 - count := 1 + int((int64(ceil)-from)/16) - if count > limit { - count = limit - } - go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false) + from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) + go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) // Wait for the remote response to the head fetch number, hash := uint64(0), common.Hash{} @@ -647,9 +691,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return 0, errEmptyHeaderSet } // Make sure the peer's reply conforms to the request - for i := 0; i < len(headers); i++ { - if number := headers[i].Number.Int64(); number != from+int64(i)*16 { - p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number) + for i, header := range headers { + expectNumber := from + int64(i)*int64((skip+1)) + if number := header.Number.Int64(); number != expectNumber { + p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number) return 0, errInvalidChain } } @@ -657,20 +702,15 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err finished = true for i := len(headers) - 1; i >= 0; i-- { // Skip any headers that underflow/overflow our requested set - if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil { + if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max { continue } // Otherwise check if we already know the header or not h := headers[i].Hash() n := headers[i].Number.Uint64() - if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) { + if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || + (d.mode != FullSync && d.lightchain.HasHeader(h, n)) { number, hash = n, h - - // If every header is known, even future ones, the peer straight out lied about its head - if number > height && i == limit-1 { - p.log.Warn("Lied about chain head", "reported", height, "found", number) - return 0, errStallingPeer - } break } } @@ -694,7 +734,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return number, nil } // Ancestor not found, we need to binary search over our chain - start, end := uint64(0), head + start, end := uint64(0), remoteHeight if floor > 0 { start = uint64(floor) } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 1fe02d8849..b30976d721 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "strings" "sync" "sync/atomic" "testing" @@ -1479,3 +1480,78 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int } return nil } + +func TestRemoteHeaderRequestSpan(t *testing.T) { + testCases := []struct { + remoteHeight uint64 + localHeight uint64 + expected []int + }{ + // Remote is way higher. We should ask for the remote head and go backwards + {1500, 1000, + []int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499}, + }, + {15000, 13006, + []int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999}, + }, + //Remote is pretty close to us. We don't have to fetch as many + {1200, 1150, + []int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199}, + }, + // Remote is equal to us (so on a fork with higher td) + // We should get the closest couple of ancestors + {1500, 1500, + []int{1497, 1499}, + }, + // We're higher than the remote! Odd + {1000, 1500, + []int{997, 999}, + }, + // Check some weird edgecases that it behaves somewhat rationally + {0, 1500, + []int{0, 2}, + }, + {6000000, 0, + []int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999}, + }, + {0, 0, + []int{0, 2}, + }, + } + reqs := func(from, count, span int) []int { + var r []int + num := from + for len(r) < count { + r = append(r, num) + num += span + 1 + } + return r + } + for i, tt := range testCases { + from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight) + data := reqs(int(from), count, span) + + if max != uint64(data[len(data)-1]) { + t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max) + } + failed := false + if len(data) != len(tt.expected) { + failed = true + t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data)) + } else { + for j, n := range data { + if n != tt.expected[j] { + failed = true + break + } + } + } + if failed { + res := strings.Replace(fmt.Sprint(data), " ", ",", -1) + exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1) + fmt.Printf("got: %v\n", res) + fmt.Printf("exp: %v\n", exp) + t.Errorf("test %d: wrong values", i) + } + } +} From accc0fab4f407eaeab428127bd5395a28f371f9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 16 Nov 2018 13:15:05 +0200 Subject: [PATCH 087/100] core, eth/downloader: fix ancestor lookup for fast sync --- core/blockchain.go | 13 ++++++++++-- core/rawdb/accessors_chain.go | 9 +++++++++ eth/downloader/downloader.go | 33 +++++++++++++++++++++++++++---- eth/downloader/downloader_test.go | 32 ++++++++++++++++++++---------- 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 22f130ce67..d173b2de29 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -561,6 +561,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { return rawdb.HasBody(bc.db, hash, number) } +// HasFastBlock checks if a fast block is fully present in the database or not. +func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool { + if !bc.HasBlock(hash, number) { + return false + } + if bc.receiptsCache.Contains(hash) { + return true + } + return rawdb.HasReceipts(bc.db, hash, number) +} + // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { _, err := bc.stateCache.OpenTrie(hash) @@ -618,12 +629,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { if receipts, ok := bc.receiptsCache.Get(hash); ok { return receipts.(types.Receipts) } - number := rawdb.ReadHeaderNumber(bc.db, hash) if number == nil { return nil } - receipts := rawdb.ReadReceipts(bc.db, hash, *number) bc.receiptsCache.Add(hash, receipts) return receipts diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 6660e17de1..491a125c65 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { } } +// HasReceipts verifies the existence of all the transaction receipts belonging +// to a block. +func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool { + if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { + return false + } + return true +} + // ReadReceipts retrieves all the transaction receipts belonging to a block. func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1f52fc6db4..f81a5cbac2 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -181,6 +181,9 @@ type BlockChain interface { // HasBlock verifies a block's presence in the local chain. HasBlock(common.Hash, uint64) bool + // HasFastBlock verifies a fast block's presence in the local chain. + HasFastBlock(common.Hash, uint64) bool + // GetBlockByHash retrieves a block from the local chain. GetBlockByHash(common.Hash) *types.Block @@ -663,8 +666,9 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) if localHeight >= MaxForkAncestry { floor = int64(localHeight - MaxForkAncestry) } - from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) + + p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip) go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) // Wait for the remote response to the head fetch @@ -708,8 +712,17 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) // Otherwise check if we already know the header or not h := headers[i].Hash() n := headers[i].Number.Uint64() - if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || - (d.mode != FullSync && d.lightchain.HasHeader(h, n)) { + + var known bool + switch d.mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if known { number, hash = n, h break } @@ -738,6 +751,8 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) if floor > 0 { start = uint64(floor) } + p.log.Trace("Binary searching for common ancestor", "start", start, "end", end) + for start+1 < end { // Split our chain interval in two, and request the hash to cross check check := (start + end) / 2 @@ -770,7 +785,17 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) // Modify the search interval based on the response h := headers[0].Hash() n := headers[0].Number.Uint64() - if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) { + + var known bool + switch d.mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if !known { end = check break } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index b30976d721..1a42965d39 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -115,6 +115,15 @@ func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool { return dl.GetBlockByHash(hash) != nil } +// HasFastBlock checks if a block is present in the testers canonical chain. +func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + + _, ok := dl.ownReceipts[hash] + return ok +} + // GetHeader retrieves a header from the testers canonical chain. func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header { dl.lock.RLock() @@ -235,6 +244,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { dl.ownHeaders[block.Hash()] = block.Header() } dl.ownBlocks[block.Hash()] = block + dl.ownReceipts[block.Hash()] = make(types.Receipts, 0) dl.stateDb.Put(block.Root().Bytes(), []byte{0x00}) dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty()) } @@ -375,28 +385,28 @@ func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { // assertOwnChain checks if the local chain contains the correct number of items // of the various chain components. func assertOwnChain(t *testing.T, tester *downloadTester, length int) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + assertOwnForkedChain(t, tester, 1, []int{length}) } // assertOwnForkedChain checks if the local forked chain contains the correct // number of items of the various chain components. func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) { - // Initialize the counters for the first fork - headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + // Initialize the counters for the first fork + headers, blocks, receipts := lengths[0], lengths[0], lengths[0] - if receipts < 0 { - receipts = 1 - } // Update the counters for each subsequent fork for _, length := range lengths[1:] { headers += length - common blocks += length - common - receipts += length - common - fsMinFullBlocks + receipts += length - common } - switch tester.downloader.mode { - case FullSync: - receipts = 1 - case LightSync: + if tester.downloader.mode == LightSync { blocks, receipts = 1, 1 } if hs := len(tester.ownHeaders); hs != headers { @@ -1150,7 +1160,9 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + // Mark this method as a helper to report errors at callsite, not in here t.Helper() + p := d.Progress() p.KnownStates, p.PulledStates = 0, 0 want.KnownStates, want.PulledStates = 0, 0 From d136e985e80e4c3dc27678b0f54d6e34dd671850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 16 Nov 2018 16:35:39 +0200 Subject: [PATCH 088/100] trie: go fmt package --- trie/proof.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/proof.go b/trie/proof.go index 62c47beda2..f90ecd7d88 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -67,7 +67,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { } hasher := newHasher(0, 0, nil) defer returnHasherToPool(hasher) - + for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. From 51e2e78d26b9e8c7b6297cf51662cf22debbb7c6 Mon Sep 17 00:00:00 2001 From: Elad Date: Sun, 18 Nov 2018 15:24:03 +0530 Subject: [PATCH 089/100] swarm/api/http: change request served msg log level (#18127) --- swarm/api/http/middleware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index ccc040c54d..f5f70138b3 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) - log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) }) } From 3333fe660f00b519a9d16398ca3801ab9313be16 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 19 Nov 2018 12:26:45 +0100 Subject: [PATCH 090/100] swarm/storage: speed up garbage collection and rpc tests (#18128) --- swarm/storage/ldbstore_test.go | 9 +++++---- swarm/storage/mock/rpc/rpc_test.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 9c10b36290..e8b9ae39bc 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -344,17 +344,18 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { func TestLDBStoreCollectGarbage(t *testing.T) { // below max ronud - cap := defaultMaxGCRound / 2 + initialCap := defaultMaxGCRound / 100 + cap := initialCap / 2 t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) // at max round - cap = defaultMaxGCRound + cap = initialCap t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) // more than max around, not on threshold - cap = defaultMaxGCRound * 1.1 + cap = initialCap + 500 t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) @@ -578,7 +579,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { // TestLDBStoreCollectGarbageAccessUnlikeIndex tests garbage collection where accesscount differs from indexcount func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { - capacity := defaultMaxGCRound * 2 + capacity := defaultMaxGCRound / 100 * 2 n := capacity - 1 ldb, cleanup := newLDBStore(t) diff --git a/swarm/storage/mock/rpc/rpc_test.go b/swarm/storage/mock/rpc/rpc_test.go index 52b634a445..f62340edec 100644 --- a/swarm/storage/mock/rpc/rpc_test.go +++ b/swarm/storage/mock/rpc/rpc_test.go @@ -37,5 +37,5 @@ func TestRPCStore(t *testing.T) { store := NewGlobalStore(rpc.DialInProc(server)) defer store.Close() - test.MockStore(t, store, 100) + test.MockStore(t, store, 30) } From 6b6c4d1c2754f8dd70172ab58d7ee33cf9058c7d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 19 Nov 2018 14:57:22 +0100 Subject: [PATCH 091/100] cmd/swarm: speed up tests - use global cluster (#18129) --- cmd/swarm/access_test.go | 79 ++++++++++++++++-------------- cmd/swarm/export_test.go | 4 +- cmd/swarm/feeds_test.go | 2 - cmd/swarm/fs_test.go | 6 --- cmd/swarm/run_test.go | 11 +++++ cmd/swarm/upload_test.go | 101 +++++++++++++++++++-------------------- 6 files changed, 106 insertions(+), 97 deletions(-) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index e812cd8fd1..9357c577e3 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// +build !windows - package main import ( @@ -28,6 +26,7 @@ import ( gorand "math/rand" "net/http" "os" + "runtime" "strings" "testing" "time" @@ -37,8 +36,7 @@ import ( "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" - swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + swarmapi "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -49,22 +47,41 @@ const ( var DefaultCurve = crypto.S256() -// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. +func TestACT(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + initCluster(t) + + cases := []struct { + name string + f func(t *testing.T) + }{ + {"Password", testPassword}, + {"PK", testPK}, + {"ACTWithoutBogus", testACTWithoutBogus}, + {"ACTWithBogus", testACTWithBogus}, + } + + for _, tc := range cases { + t.Run(tc.name, tc.f) + } +} + +// testPassword tests for the correct creation of an ACT manifest protected by a password. // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded // is then fetched through 2nd node. since the tested code is not key-aware - we can just // fetch from the 2nd node using HTTP BasicAuth -func TestAccessPassword(t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - +func testPassword(t *testing.T) { dataFilename := testutil.TempFileWithContent(t, data) defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, "--bzzapi", - srv.URL, //it doesn't matter through which node we upload content + cluster.Nodes[0].URL, "up", "--encrypt", dataFilename) @@ -138,16 +155,17 @@ func TestAccessPassword(t *testing.T) { if a.Publisher != "" { t.Fatal("should be empty") } - client := swarm.NewClient(srv.URL) + + client := swarmapi.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) if err != nil { t.Fatal(err) } - httpClient := &http.Client{} + url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash - url := srv.URL + "/" + "bzz:/" + hash + httpClient := &http.Client{} response, err := httpClient.Get(url) if err != nil { t.Fatal(err) @@ -189,7 +207,7 @@ func TestAccessPassword(t *testing.T) { //download file with 'swarm down' with wrong password up = runSwarm(t, "--bzzapi", - srv.URL, + cluster.Nodes[0].URL, "down", "bzz:/"+hash, tmp, @@ -203,16 +221,12 @@ func TestAccessPassword(t *testing.T) { up.ExpectExit() } -// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). +// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears. // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware, // the test will fail if the proxy's given private key is not granted on the ACT. -func TestAccessPK(t *testing.T) { - // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 2) - defer cluster.Shutdown() - +func testPK(t *testing.T) { dataFilename := testutil.TempFileWithContent(t, data) defer os.RemoveAll(dataFilename) @@ -318,7 +332,7 @@ func TestAccessPK(t *testing.T) { if a.Publisher != pkComp { t.Fatal("publisher key did not match") } - client := swarm.NewClient(cluster.Nodes[0].URL) + client := swarmapi.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) if err != nil { @@ -344,29 +358,24 @@ func TestAccessPK(t *testing.T) { } } -// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) -func TestAccessACT(t *testing.T) { - testAccessACT(t, 0) +// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) +func testACTWithoutBogus(t *testing.T) { + testACT(t, 0) } -// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest) -func TestAccessACTScale(t *testing.T) { - testAccessACT(t, 1000) +// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest) +func testACTWithBogus(t *testing.T) { + testACT(t, 100) } -// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection +// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access. // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails. // the publisher uploads through one of the nodes then disappears. -func testAccessACT(t *testing.T, bogusEntries int) { - // Setup Swarm and upload a test file to it - const clusterSize = 3 - cluster := newTestCluster(t, clusterSize) - defer cluster.Shutdown() - +func testACT(t *testing.T, bogusEntries int) { var uploadThroughNode = cluster.Nodes[0] - client := swarm.NewClient(uploadThroughNode.URL) + client := swarmapi.NewClient(uploadThroughNode.URL) r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`) diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go index f1bc2f2658..e8671eea79 100644 --- a/cmd/swarm/export_test.go +++ b/cmd/swarm/export_test.go @@ -43,8 +43,8 @@ func TestCLISwarmExportImport(t *testing.T) { } cluster := newTestCluster(t, 1) - // generate random 10mb file - content := testutil.RandomBytes(1, 10000000) + // generate random 1mb file + content := testutil.RandomBytes(1, 1000000) fileName := testutil.TempFileWithContent(t, string(content)) defer os.Remove(fileName) diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go index fc3f72ab15..a0cedf0d36 100644 --- a/cmd/swarm/feeds_test.go +++ b/cmd/swarm/feeds_test.go @@ -36,7 +36,6 @@ import ( ) func TestCLIFeedUpdate(t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") }, nil) @@ -44,7 +43,6 @@ func TestCLIFeedUpdate(t *testing.T) { defer srv.Close() // create a private key file for signing - privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979" privKey, _ := crypto.HexToECDSA(privkeyHex) address := crypto.PubkeyToAddress(privKey.PublicKey) diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index 3b722515ee..ac4223b661 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -29,14 +29,8 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" ) -func init() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) -} - type testFile struct { filePath string content string diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 416fa7a503..680d238d06 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -57,6 +57,17 @@ func init() { }) } +const clusterSize = 3 + +var clusteronce sync.Once +var cluster *testCluster + +func initCluster(t *testing.T) { + clusteronce.Do(func() { + cluster = newTestCluster(t, clusterSize) + }) +} + func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 5f9844950e..616486e37c 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -31,8 +31,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" - swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + swarmapi "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/mattn/go-colorable" ) @@ -42,42 +41,50 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) } -// TestCLISwarmUp tests that running 'swarm up' makes the resulting file +func TestSwarmUp(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + initCluster(t) + + cases := []struct { + name string + f func(t *testing.T) + }{ + {"NoEncryption", testNoEncryption}, + {"Encrypted", testEncrypted}, + {"RecursiveNoEncryption", testRecursiveNoEncryption}, + {"RecursiveEncrypted", testRecursiveEncrypted}, + {"DefaultPathAll", testDefaultPathAll}, + } + + for _, tc := range cases { + t.Run(tc.name, tc.f) + } +} + +// testNoEncryption tests that running 'swarm up' makes the resulting file // available from all nodes via the HTTP API -func TestCLISwarmUp(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - - testCLISwarmUp(false, t) -} -func TestCLISwarmUpRecursive(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUpRecursive(false, t) +func testNoEncryption(t *testing.T) { + testDefault(false, t) } -// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file +// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file // available from all nodes via the HTTP API -func TestCLISwarmUpEncrypted(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUp(true, t) -} -func TestCLISwarmUpEncryptedRecursive(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUpRecursive(true, t) +func testEncrypted(t *testing.T) { + testDefault(true, t) } -func testCLISwarmUp(toEncrypt bool, t *testing.T) { - log.Info("starting 3 node cluster") - cluster := newTestCluster(t, 3) - defer cluster.Shutdown() +func testRecursiveNoEncryption(t *testing.T) { + testRecursive(false, t) +} +func testRecursiveEncrypted(t *testing.T) { + testRecursive(true, t) +} + +func testDefault(toEncrypt bool, t *testing.T) { tmpFileName := testutil.TempFileWithContent(t, data) defer os.Remove(tmpFileName) @@ -182,11 +189,7 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) { } } -func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { - fmt.Println("starting 3 node cluster") - cluster := newTestCluster(t, 3) - defer cluster.Shutdown() - +func testRecursive(toEncrypt bool, t *testing.T) { tmpUploadDir, err := ioutil.TempDir("", "swarm-test") if err != nil { t.Fatal(err) @@ -253,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { switch mode := fi.Mode(); { case mode.IsRegular(): - if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil { + if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil { t.Fatalf("encountered an error opening the file returned from the CLI: %v", err) } else { ff := make([]byte, len(data)) @@ -274,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } } -// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute +// testDefaultPathAll tests swarm recursive upload with relative and absolute // default paths and with encryption. -func TestCLISwarmUpDefaultPath(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - testCLISwarmUpDefaultPath(false, false, t) - testCLISwarmUpDefaultPath(false, true, t) - testCLISwarmUpDefaultPath(true, false, t) - testCLISwarmUpDefaultPath(true, true, t) +func testDefaultPathAll(t *testing.T) { + testDefaultPath(false, false, t) + testDefaultPath(false, true, t) + testDefaultPath(true, false, t) + testDefaultPath(true, true, t) } -func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - +func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") if err != nil { t.Fatal(err) @@ -312,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T args := []string{ "--bzzapi", - srv.URL, + cluster.Nodes[0].URL, "--recursive", "--defaultpath", defaultPath, @@ -329,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T up.ExpectExit() hash := matches[0] - client := swarm.NewClient(srv.URL) + client := swarmapi.NewClient(cluster.Nodes[0].URL) m, isEncrypted, err := client.DownloadManifest(hash) if err != nil { From d31f1f4fdb905acb3d7c766683e584bb17f8df0d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 19 Nov 2018 14:58:10 +0100 Subject: [PATCH 092/100] cmd/swarm/swarm-smoke: update smoke tests to fit the new scheme for the k8s cluster (#18104) --- cmd/swarm/swarm-smoke/main.go | 4 ++-- cmd/swarm/swarm-smoke/upload_and_sync.go | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 97054dd479..4ff17fd5b7 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -45,8 +45,8 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ Name: "cluster-endpoint", - Value: "testing", - Usage: "cluster to point to (local, open or testing)", + Value: "prod", + Usage: "cluster to point to (prod or a given namespace)", Destination: &cluster, }, cli.IntFlag{ diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 358141c7cc..7872421d3d 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" crand "crypto/rand" + "crypto/tls" "errors" "fmt" "io" @@ -32,6 +33,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" @@ -39,18 +41,13 @@ import ( func generateEndpoints(scheme string, cluster string, from int, to int) { if cluster == "prod" { - cluster = "" - } else if cluster == "local" { for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port)) + endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } - return } else { - cluster = cluster + "." - } - - for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster)) + for port := from; port <= to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster)) + } } if includeLocalhost { @@ -59,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) { } func cliUploadAndSync(c *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, from, to) @@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { time.Sleep(3 * time.Second) log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - res, err := http.Get(endpoint + "/bzz:/" + hash + "/") + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + res, err := client.Get(endpoint + "/bzz:/" + hash + "/") if err != nil { log.Warn(err.Error(), "ruid", ruid) return err From 7bf7bd2f5088d8abb21daef42b8dfce27b194bc7 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 20 Nov 2018 08:23:43 +0100 Subject: [PATCH 093/100] internal/cmdtest: Expose process exit status and errors (#18046) --- internal/cmdtest/test_cmd.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 20e82ec2a9..f8b1b43c01 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -27,6 +27,7 @@ import ( "regexp" "strings" "sync" + "syscall" "testing" "text/template" "time" @@ -50,6 +51,8 @@ type TestCmd struct { stdout *bufio.Reader stdin io.WriteCloser stderr *testlogger + // Err will contain the process exit error or interrupt signal error + Err error } // Run exec's the current binary using name as argv[0] which will trigger the @@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() { } func (tt *TestCmd) WaitExit() { - tt.cmd.Wait() + tt.Err = tt.cmd.Wait() } func (tt *TestCmd) Interrupt() { - tt.cmd.Process.Signal(os.Interrupt) + tt.Err = tt.cmd.Process.Signal(os.Interrupt) +} + +// ExitStatus exposes the process' OS exit code +// It will only return a valid value after the process has finished. +func (tt *TestCmd) ExitStatus() int { + if tt.Err != nil { + exitErr := tt.Err.(*exec.ExitError) + if exitErr != nil { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + return status.ExitStatus() + } + } + } + return 0 } // StderrText returns any stderr output written so far. From d876f214e5500962d6acc1f99a6f2f7c5f63db8b Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Tue, 20 Nov 2018 08:30:38 +0100 Subject: [PATCH 094/100] swarm/storage: move 'running migrations for' log line (#18120) So that we only see the log message when we actually have to migrate. --- swarm/storage/localstore.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index fa98848ddc..111821ff65 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -194,7 +194,8 @@ func (ls *LocalStore) Close() { ls.DbStore.Close() } -// Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match +// Migrate checks the datastore schema vs the runtime schema and runs +// migrations if they don't match func (ls *LocalStore) Migrate() error { actualDbSchema, err := ls.DbStore.GetSchema() if err != nil { @@ -202,12 +203,12 @@ func (ls *LocalStore) Migrate() error { return err } - log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) - if actualDbSchema == CurrentDbSchema { return nil } + log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) + if actualDbSchema == DbSchemaNone { ls.migrateFromNoneToPurity() actualDbSchema = DbSchemaPurity From 3d997b6decfaa42e37521ae20bf58886c8b2de8f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 20 Nov 2018 10:08:02 +0100 Subject: [PATCH 095/100] whisper: log errors on failed tests (#18134) Debug traces to investigate a travis issue on MacOS --- whisper/whisperv5/peer_test.go | 2 +- whisper/whisperv6/peer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index 35616aaaf6..2449532070 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -139,7 +139,7 @@ func initialize(t *testing.T) { err = node.server.Start() if err != nil { - t.Fatalf("failed to start server %d.", i) + t.Fatalf("failed to start server %d. err: %v", i, err) } for j := 0; j < i; j++ { diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 65e62d96cc..c5b044e1a6 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -232,7 +232,7 @@ func initialize(t *testing.T) { func startServer(t *testing.T, s *p2p.Server) { err := s.Start() if err != nil { - t.Fatalf("failed to start the fisrt server.") + t.Fatalf("failed to start the first server. err: %v", err) } atomic.AddInt64(&result.started, 1) From 5d80a1b6652b1c5eb50b73e9582d9000829d7c9a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 20 Nov 2018 20:14:37 +0100 Subject: [PATCH 096/100] whisper/mailserver: reduce the max number of opened files (#18142) This should reduce the occurences of travis failures on MacOS Also fix some linter warnings --- whisper/mailserver/mailserver.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index af9418d9f8..d7af4baae3 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// Package mailserver provides a naive, example mailserver implementation package mailserver import ( @@ -26,9 +27,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" ) +// WMailServer represents the state data of the mailserver. type WMailServer struct { db *leveldb.DB w *whisper.Whisper @@ -42,6 +45,8 @@ type DBKey struct { raw []byte } +// NewDbKey is a helper function that creates a levelDB +// key from a hash and an integer. func NewDbKey(t uint32, h common.Hash) *DBKey { const sz = common.HashLength + 4 var k DBKey @@ -53,6 +58,7 @@ func NewDbKey(t uint32, h common.Hash) *DBKey { return &k } +// Init initializes the mail server. func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) error { var err error if len(path) == 0 { @@ -63,7 +69,7 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p return fmt.Errorf("password is not specified") } - s.db, err = leveldb.OpenFile(path, nil) + s.db, err = leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: 32}) if err != nil { return fmt.Errorf("open DB file: %s", err) } @@ -82,12 +88,14 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p return nil } +// Close cleans up before shutdown. func (s *WMailServer) Close() { if s.db != nil { s.db.Close() } } +// Archive stores the func (s *WMailServer) Archive(env *whisper.Envelope) { key := NewDbKey(env.Expiry-env.TTL, env.Hash()) rawEnvelope, err := rlp.EncodeToBytes(env) @@ -101,6 +109,8 @@ func (s *WMailServer) Archive(env *whisper.Envelope) { } } +// DeliverMail responds with saved messages upon request by the +// messages' owner. func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) { if peer == nil { log.Error("Whisper peer is nil") From c7e522fd17040485a5c2966c45a5c45c0558945e Mon Sep 17 00:00:00 2001 From: a-sklyarov Date: Wed, 21 Nov 2018 10:38:49 +0000 Subject: [PATCH 097/100] Update minimum required Go version in README.md (#18151) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f308fb1011..4b62bfde03 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki. -Building geth requires both a Go (version 1.7 or later) and a C compiler. +Building geth requires both a Go (version 1.9 or later) and a C compiler. You can install them using your favourite package manager. Once the dependencies are installed, run From 3fd87f219342ca97a6595263a2aa28bec65fee04 Mon Sep 17 00:00:00 2001 From: mr_franklin Date: Wed, 21 Nov 2018 18:52:02 +0800 Subject: [PATCH 098/100] core: fix comment typo (#18144) --- core/tx_pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index f6da5da2a7..fc35d1f243 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error { // addTxsLocked attempts to queue a batch of transactions if they are valid, // whilst assuming the transaction pool lock is already held. func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error { - // Add the batch of transaction, tracking the accepted ones + // Add the batch of transactions, tracking the accepted ones dirty := make(map[common.Address]struct{}) errs := make([]error, len(txs)) From 4c181e4fb98bb88503cccd6147026b6c2b7b56f6 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 21 Nov 2018 14:36:56 +0100 Subject: [PATCH 099/100] swarm/state: refactor InmemoryStore (#18143) --- swarm/network/simulations/overlay.go | 4 +- swarm/network/stream/delivery.go | 2 +- swarm/state/dbstore.go | 21 +++++++ swarm/state/inmemorystore.go | 94 ---------------------------- swarm/state/store.go | 26 -------- 5 files changed, 24 insertions(+), 123 deletions(-) delete mode 100644 swarm/state/inmemorystore.go delete mode 100644 swarm/state/store.go diff --git a/swarm/network/simulations/overlay.go b/swarm/network/simulations/overlay.go index caf7ff1f2d..284ae6398a 100644 --- a/swarm/network/simulations/overlay.go +++ b/swarm/network/simulations/overlay.go @@ -64,12 +64,12 @@ func init() { type Simulation struct { mtx sync.Mutex - stores map[enode.ID]*state.InmemoryStore + stores map[enode.ID]state.Store } func NewSimulation() *Simulation { return &Simulation{ - stores: make(map[enode.ID]*state.InmemoryStore), + stores: make(map[enode.ID]state.Store), } } diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 0109fbdef2..64d754336f 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -169,7 +169,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * go func() { chunk, err := d.chunkStore.Get(ctx, req.Addr) if err != nil { - log.Warn("ChunkStore.Get can not retrieve chunk", "err", err) + log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) return } if req.SkipCheck { diff --git a/swarm/state/dbstore.go b/swarm/state/dbstore.go index b0aa92e272..fc5dd8f7ca 100644 --- a/swarm/state/dbstore.go +++ b/swarm/state/dbstore.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" ) // ErrNotFound is returned when no results are returned from the database @@ -30,6 +31,15 @@ var ErrNotFound = errors.New("ErrorNotFound") // ErrInvalidArgument is returned when the argument type does not match the expected type var ErrInvalidArgument = errors.New("ErrorInvalidArgument") +// Store defines methods required to get, set, delete values for different keys +// and close the underlying resources. +type Store interface { + Get(key string, i interface{}) (err error) + Put(key string, i interface{}) (err error) + Delete(key string) (err error) + Close() error +} + // DBStore uses LevelDB to store values. type DBStore struct { db *leveldb.DB @@ -46,6 +56,17 @@ func NewDBStore(path string) (s *DBStore, err error) { }, nil } +// NewInmemoryStore returns a new instance of DBStore. To be used only in tests and simulations. +func NewInmemoryStore() *DBStore { + db, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + panic(err) + } + return &DBStore{ + db: db, + } +} + // Get retrieves a persisted value for a specific key. If there is no results // ErrNotFound is returned. The provided parameter should be either a byte slice or // a struct that implements the encoding.BinaryUnmarshaler interface diff --git a/swarm/state/inmemorystore.go b/swarm/state/inmemorystore.go deleted file mode 100644 index 3ba48592bc..0000000000 --- a/swarm/state/inmemorystore.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package state - -import ( - "encoding" - "encoding/json" - "sync" -) - -// InmemoryStore is the reference implementation of Store interface that is supposed -// to be used in tests. -type InmemoryStore struct { - db map[string][]byte - mu sync.RWMutex -} - -// NewInmemoryStore returns a new instance of InmemoryStore. -func NewInmemoryStore() *InmemoryStore { - return &InmemoryStore{ - db: make(map[string][]byte), - } -} - -// Get retrieves a value stored for a specific key. If there is no value found, -// ErrNotFound is returned. -func (s *InmemoryStore) Get(key string, i interface{}) (err error) { - s.mu.RLock() - defer s.mu.RUnlock() - - bytes, ok := s.db[key] - if !ok { - return ErrNotFound - } - - unmarshaler, ok := i.(encoding.BinaryUnmarshaler) - if !ok { - return json.Unmarshal(bytes, i) - } - - return unmarshaler.UnmarshalBinary(bytes) -} - -// Put stores a value for a specific key. -func (s *InmemoryStore) Put(key string, i interface{}) (err error) { - s.mu.Lock() - defer s.mu.Unlock() - var bytes []byte - - marshaler, ok := i.(encoding.BinaryMarshaler) - if !ok { - if bytes, err = json.Marshal(i); err != nil { - return err - } - } else { - if bytes, err = marshaler.MarshalBinary(); err != nil { - return err - } - } - - s.db[key] = bytes - return nil -} - -// Delete removes value stored under a specific key. -func (s *InmemoryStore) Delete(key string) (err error) { - s.mu.Lock() - defer s.mu.Unlock() - - if _, ok := s.db[key]; !ok { - return ErrNotFound - } - delete(s.db, key) - return nil -} - -// Close does not do anything. -func (s *InmemoryStore) Close() error { - return nil -} diff --git a/swarm/state/store.go b/swarm/state/store.go deleted file mode 100644 index fb7fe258f1..0000000000 --- a/swarm/state/store.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package state - -// Store defines methods required to get, set, delete values for different keys -// and close the underlying resources. -type Store interface { - Get(key string, i interface{}) (err error) - Put(key string, i interface{}) (err error) - Delete(key string) (err error) - Close() error -} From 070caec4bd72b1a958ece5f2b8fcc2bb92d44796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Wed, 21 Nov 2018 20:49:13 +0100 Subject: [PATCH 100/100] swarm/network/stream: use swarm/mock/mem as mock global store (#18157) --- swarm/network/stream/common_test.go | 16 ------------- swarm/network/stream/snapshot_sync_test.go | 27 +++++----------------- swarm/network/stream/syncer_test.go | 21 ++++------------- 3 files changed, 11 insertions(+), 53 deletions(-) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index c5f1fa176a..e0a7f7e120 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -38,7 +38,6 @@ import ( "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -69,21 +68,6 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) } -func createGlobalStore() (string, *mockdb.GlobalStore, error) { - var globalStore *mockdb.GlobalStore - globalStoreDir, err := ioutil.TempDir("", "global.store") - if err != nil { - log.Error("Error initiating global store temp directory!", "err", err) - return "", nil, err - } - globalStore, err = mockdb.NewGlobalStore(globalStoreDir) - if err != nil { - log.Error("Error initiating global store!", "err", err) - return "", nil, err - } - return globalStoreDir, globalStore, nil -} - func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { // setup addr := network.RandomAddr() // tested peers peer address diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 6b92c32ae9..4e56f71b54 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -35,7 +35,8 @@ import ( "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -268,20 +269,9 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. - var gDir string - var globalStore *mockdb.GlobalStore + var globalStore mock.GlobalStorer if *useMockStore { - gDir, globalStore, err = createGlobalStore() - if err != nil { - return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil") - } - defer func() { - os.RemoveAll(gDir) - err := globalStore.Close() - if err != nil { - log.Error("Error closing global store! %v", "err", err) - } - }() + globalStore = mockmem.NewGlobalStore() } REPEAT: for { @@ -476,14 +466,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) return err } - var gDir string - var globalStore *mockdb.GlobalStore + var globalStore mock.GlobalStorer if *useMockStore { - gDir, globalStore, err = createGlobalStore() - if err != nil { - return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil") - } - defer os.RemoveAll(gDir) + globalStore = mockmem.NewGlobalStore() } // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index fe20bab266..5764efc92d 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -35,7 +35,8 @@ import ( "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -48,7 +49,7 @@ func TestSyncerSimulation(t *testing.T) { testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1) } -func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) { +func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) { address := common.BytesToAddress(id.Bytes()) mockStore := globalStore.NewNodeStore(address) params := storage.NewDefaultLocalStoreParams() @@ -70,8 +71,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { var store storage.ChunkStore - var globalStore *mockdb.GlobalStore - var gDir, datadir string + var datadir string node := ctx.Config.Node() addr := network.NewAddr(node) @@ -79,11 +79,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck addr.OAddr[0] = byte(0) if *useMockStore { - gDir, globalStore, err = createGlobalStore() - if err != nil { - return nil, nil, fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil") - } - store, datadir, err = createMockStore(globalStore, node.ID(), addr) + store, datadir, err = createMockStore(mockmem.NewGlobalStore(), node.ID(), addr) } else { store, datadir, err = createTestLocalStorageForID(node.ID(), addr) } @@ -94,13 +90,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck cleanup = func() { store.Close() os.RemoveAll(datadir) - if *useMockStore { - err := globalStore.Close() - if err != nil { - log.Error("Error closing global store! %v", "err", err) - } - os.RemoveAll(gDir) - } } localStore := store.(*storage.LocalStore) netStore, err := storage.NewNetStore(localStore, nil)