Merge branch 'master' of github.com:ethereum/go-ethereum

.travis.yml

@@ -29,6 +29,14 @@ matrix:
     - os: osx
       go: 1.11.x
       script:
+        - echo "Increase the maximum number of open file descriptors on macOS"
+        - NOFILE=20480
+        - sudo sysctl -w kern.maxfiles=$NOFILE
+        - sudo sysctl -w kern.maxfilesperproc=$NOFILE
+        - sudo launchctl limit maxfiles $NOFILE $NOFILE
+        - sudo launchctl limit maxfiles
+        - ulimit -S -n $NOFILE
+        - ulimit -n
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
@@ -148,7 +156,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go

README.md

@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

cmd/geth/main.go

@@ -89,6 +89,7 @@ var (
 		utils.LightKDFFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
+		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
 		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,

cmd/geth/usage.go

@@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
 			utils.TrieCacheGenFlag,
 		},

cmd/swarm/access.go

@@ -114,6 +114,9 @@ func accessNewPass(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {
@@ -147,6 +150,9 @@ func accessNewPK(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {

cmd/swarm/access_test.go

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !windows
-
 package main
 
 import (
@@ -28,6 +26,7 @@ import (
 	gorand "math/rand"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -37,7 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -48,22 +47,41 @@ const (
 
 var DefaultCurve = crypto.S256()
 
-// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
+func TestACT(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"Password", testPassword},
+		{"PK", testPK},
+		{"ACTWithoutBogus", testACTWithoutBogus},
+		{"ACTWithBogus", testACTWithBogus},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testPassword tests for the correct creation of an ACT manifest protected by a password.
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
 // is then fetched through 2nd node. since the tested code is not key-aware - we can just
 // fetch from the 2nd node using HTTP BasicAuth
-func TestAccessPassword(t *testing.T) {
+func testPassword(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
-	defer srv.Close()
-
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
 		"--bzzapi",
-		srv.URL, //it doesn't matter through which node we upload content
+		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
 		dataFilename)
@@ -137,16 +155,17 @@ func TestAccessPassword(t *testing.T) {
 	if a.Publisher != "" {
 		t.Fatal("should be empty")
 	}
-	client := swarm.NewClient(srv.URL)
+
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	httpClient := &http.Client{}
+	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
 
-	url := srv.URL + "/" + "bzz:/" + hash
+	httpClient := &http.Client{}
 	response, err := httpClient.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -188,7 +207,7 @@ func TestAccessPassword(t *testing.T) {
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
 		"--bzzapi",
-		srv.URL,
+		cluster.Nodes[0].URL,
 		"down",
 		"bzz:/"+hash,
 		tmp,
@@ -202,16 +221,12 @@ func TestAccessPassword(t *testing.T) {
 	up.ExpectExit()
 }
 
-// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
+// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
 // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
 // the test will fail if the proxy's given private key is not granted on the ACT.
-func TestAccessPK(t *testing.T) {
+func testPK(t *testing.T) {
-	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 2)
-	defer cluster.Shutdown()
-
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
@@ -317,7 +332,7 @@ func TestAccessPK(t *testing.T) {
 	if a.Publisher != pkComp {
 		t.Fatal("publisher key did not match")
 	}
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
@@ -343,29 +358,24 @@ func TestAccessPK(t *testing.T) {
 	}
 }
 
-// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
+// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
-func TestAccessACT(t *testing.T) {
+func testACTWithoutBogus(t *testing.T) {
-	testAccessACT(t, 0)
+	testACT(t, 0)
 }
 
-// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
+// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
-func TestAccessACTScale(t *testing.T) {
+func testACTWithBogus(t *testing.T) {
-	testAccessACT(t, 1000)
+	testACT(t, 100)
 }
 
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
 // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
 // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
 // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
 // the publisher uploads through one of the nodes then disappears.
-func testAccessACT(t *testing.T, bogusEntries int) {
+func testACT(t *testing.T, bogusEntries int) {
-	// Setup Swarm and upload a test file to it
-	const clusterSize = 3
-	cluster := newTestCluster(t, clusterSize)
-	defer cluster.Shutdown()
-
 	var uploadThroughNode = cluster.Nodes[0]
-	client := swarm.NewClient(uploadThroughNode.URL)
+	client := swarmapi.NewClient(uploadThroughNode.URL)
 
 	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
 	nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)

cmd/swarm/config_test.go

@@ -26,14 +26,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	"github.com/ethereum/go-ethereum/swarm/api"
-
-	"github.com/docker/docker/pkg/reexec"
 )
 
-func TestDumpConfig(t *testing.T) {
+func TestConfigDump(t *testing.T) {
 	swarm := runSwarm(t, "dumpconfig")
 	defaultConf := api.NewConfig()
 	out, err := tomlSettings.Marshal(&defaultConf)
@@ -91,8 +91,8 @@ func TestConfigCmdLineOverrides(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
 		fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name),
 		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--datadir", dir,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -189,9 +189,9 @@ func TestConfigFileOverrides(t *testing.T) {
 	flags := []string{
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
-		"--ens-api", "",
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
-		"--datadir", dir,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -407,9 +407,9 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
-		"--ens-api", "",
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--datadir", dir,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -466,7 +466,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 	node.Shutdown()
 }
 
-func TestValidateConfig(t *testing.T) {
+func TestConfigValidate(t *testing.T) {
 	for _, c := range []struct {
 		cfg *api.Config
 		err string

cmd/swarm/export_test.go

@@ -19,9 +19,7 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"runtime"
@@ -29,6 +27,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // TestCLISwarmExportImport perform the following test:
@@ -44,12 +43,13 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 	cluster := newTestCluster(t, 1)
 
-	// generate random 10mb file
+	// generate random 1mb file
-	f, cleanup := generateRandomFile(t, 10000000)
+	content := testutil.RandomBytes(1, 1000000)
-	defer cleanup()
+	fileName := testutil.TempFileWithContent(t, string(content))
+	defer os.Remove(fileName)
 
 	// upload the file with 'swarm up' and expect a hash
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
 	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
 	up.ExpectExit()
 	hash := matches[0]
@@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 
 	// compare downloaded file with the generated random file
-	mustEqualFiles(t, f, res.Body)
+	mustEqualFiles(t, bytes.NewReader(content), res.Body)
 }
 
 func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
 		t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
 	}
 }
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	// write 10mb random data to file
-	buf := make([]byte, 10000000)
-	_, err = rand.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
-}

cmd/swarm/feeds_test.go

@@ -20,49 +20,35 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestCLIFeedUpdate(t *testing.T) {
-
+	srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
-	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
 		return swarmhttp.NewServer(api, "")
 	}, nil)
 	log.Info("starting a test swarm server")
 	defer srv.Close()
 
 	// create a private key file for signing
-	pkfile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer pkfile.Close()
-	defer os.Remove(pkfile.Name())
-
 	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
 	privKey, _ := crypto.HexToECDSA(privkeyHex)
 	address := crypto.PubkeyToAddress(privKey.PublicKey)
 
-	// save the private key to a file
+	pkFileName := testutil.TempFileWithContent(t, privkeyHex)
-	_, err = io.WriteString(pkfile, privkeyHex)
+	defer os.Remove(pkFileName)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// compose a topic. We'll be doing quotes about Miguel de Cervantes
 	var topic feed.Topic
@@ -76,7 +62,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	flags := []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "update",
 		"--topic", topic.Hex(),
 		"--name", name,
@@ -89,13 +75,10 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	// now try to get the update using the client
 	client := swarm.NewClient(srv.URL)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// build the same topic as before, this time
 	// we use NewTopic to create a topic automatically.
-	topic, err = feed.NewTopic(name, subject)
+	topic, err := feed.NewTopic(name, subject)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +136,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 	// test publishing a manifest
 	flags = []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "create",
 		"--topic", topic.Hex(),
 	}

cmd/swarm/flags.go

@@ -88,7 +88,7 @@ var (
 	}
 	SwarmApiFlag = cli.StringFlag{
 		Name:  "bzzapi",
-		Usage: "Swarm HTTP endpoint",
+		Usage: "Specifies the Swarm HTTP endpoint to connect to",
 		Value: "http://127.0.0.1:8500",
 	}
 	SwarmRecursiveFlag = cli.BoolFlag{

cmd/swarm/fs.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/fuse"
 	"gopkg.in/urfave/cli.v1"
@@ -41,27 +41,24 @@ var fsCommand = cli.Command{
 			Action:             mount,
 			CustomHelpTemplate: helpTemplate,
 			Name:               "mount",
-			Flags:              []cli.Flag{utils.IPCPathFlag},
 			Usage:              "mount a swarm hash to a mount point",
-			ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
+			ArgsUsage:          "swarm fs mount <manifest hash> <mount point>",
 			Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
 		},
 		{
 			Action:             unmount,
 			CustomHelpTemplate: helpTemplate,
 			Name:               "unmount",
-			Flags:              []cli.Flag{utils.IPCPathFlag},
 			Usage:              "unmount a swarmfs mount",
-			ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
+			ArgsUsage:          "swarm fs unmount <mount point>",
 			Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
 		},
 		{
 			Action:             listMounts,
 			CustomHelpTemplate: helpTemplate,
 			Name:               "list",
-			Flags:              []cli.Flag{utils.IPCPathFlag},
 			Usage:              "list swarmfs mounts",
-			ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
+			ArgsUsage:          "swarm fs list",
 			Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
 		},
 	},
@@ -70,7 +67,7 @@ var fsCommand = cli.Command{
 func mount(cliContext *cli.Context) {
 	args := cliContext.Args()
 	if len(args) < 2 {
-		utils.Fatalf("Usage: swarm fs mount --ipcpath <path to bzzd.ipc> <manifestHash> <file name>")
+		utils.Fatalf("Usage: swarm fs mount <manifestHash> <file name>")
 	}
 
 	client, err := dialRPC(cliContext)
@@ -97,7 +94,7 @@ func unmount(cliContext *cli.Context) {
 	args := cliContext.Args()
 
 	if len(args) < 1 {
-		utils.Fatalf("Usage: swarm fs unmount --ipcpath <path to bzzd.ipc> <mount path>")
+		utils.Fatalf("Usage: swarm fs unmount <mount path>")
 	}
 	client, err := dialRPC(cliContext)
 	if err != nil {
@@ -145,20 +142,21 @@ func listMounts(cliContext *cli.Context) {
 }
 
 func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
-	var endpoint string
+	endpoint := getIPCEndpoint(ctx)
+	log.Info("IPC endpoint", "path", endpoint)
+	return rpc.Dial(endpoint)
+}
 
-	if ctx.IsSet(utils.IPCPathFlag.Name) {
+func getIPCEndpoint(ctx *cli.Context) string {
-		endpoint = ctx.String(utils.IPCPathFlag.Name)
+	cfg := defaultNodeConfig
-	} else {
+	utils.SetNodeConfig(ctx, &cfg)
-		utils.Fatalf("swarm ipc endpoint not specified")
-	}
 
-	if endpoint == "" {
+	endpoint := cfg.IPCEndpoint()
-		endpoint = node.DefaultIPCEndpoint(clientIdentifier)
+
-	} else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
+	if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
 		// Backwards compatibility with geth < 1.5 which required
 		// these prefixes.
 		endpoint = endpoint[4:]
 	}
-	return rpc.Dial(endpoint)
+	return endpoint
 }

cmd/swarm/fs_test.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -28,20 +29,35 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 )
 
-func init() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
-}
-
 type testFile struct {
 	filePath string
 	content  string
 }
 
+// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list
+// can find and correctly connect to a running Swarm node on the default
+// IPCPath.
+func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	handlingNode := cluster.Nodes[0]
+	list := runSwarm(t, []string{
+		"--datadir", handlingNode.Dir,
+		"fs",
+		"list",
+	}...)
+
+	list.WaitExit()
+	if list.Err != nil {
+		t.Fatal(list.Err)
+	}
+}
+
 // TestCLISwarmFs is a high-level test of swarmfs
 //
 // This test fails on travis for macOS as this executable exits with code 1
@@ -65,9 +81,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	mount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"mount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		mhash,
 		mountPoint,
 	}...)
@@ -80,6 +96,9 @@ func TestCLISwarmFs(t *testing.T) {
 		t.Fatal(err)
 	}
 	dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	dummyContent := "somerandomtestcontentthatshouldbeasserted"
 	dirs := []string{
@@ -104,9 +123,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	unmount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"unmount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		mountPoint,
 	}...)
 	_, matches := unmount.ExpectRegexp(hashRegexp)
@@ -139,9 +158,9 @@ func TestCLISwarmFs(t *testing.T) {
 
 	//remount, check files
 	newMount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"mount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		hash, // the latest hash
 		secondMountPoint,
 	}...)
@@ -175,9 +194,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	unmountSec := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"unmount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		secondMountPoint,
 	}...)
 

cmd/swarm/manifest_test.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 // TestManifestChange tests manifest add, update and remove
@@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) {
 // Argument encrypt controls whether to use encryption or not.
 func testManifestChange(t *testing.T, encrypt bool) {
 	t.Parallel()
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
@@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
 
 func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 	t.Parallel()
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")

cmd/swarm/run_test.go

@@ -42,7 +42,6 @@ import (
 	"github.com/ethereum/go-ethereum/swarm"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@@ -58,7 +57,18 @@ func init() {
 	})
 }
 
-func serverFunc(api *api.API) testutil.TestServer {
+const clusterSize = 3
+
+var clusteronce sync.Once
+var cluster *testCluster
+
+func initCluster(t *testing.T) {
+	clusteronce.Do(func() {
+		cluster = newTestCluster(t, clusterSize)
+	})
+}
+
+func serverFunc(api *api.API) swarmhttp.TestServer {
 	return swarmhttp.NewServer(api, "")
 }
 func TestMain(m *testing.M) {

cmd/swarm/swarm-smoke/main.go

@@ -45,8 +45,8 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "cluster-endpoint",
-			Value:       "testing",
+			Value:       "prod",
-			Usage:       "cluster to point to (local, open or testing)",
+			Usage:       "cluster to point to (prod or a given namespace)",
 			Destination: &cluster,
 		},
 		cli.IntFlag{

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	crand "crypto/rand"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -32,6 +33,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
+	colorable "github.com/mattn/go-colorable"
 	"github.com/pborman/uuid"
 
 	cli "gopkg.in/urfave/cli.v1"
@@ -39,18 +41,13 @@ import (
 
 func generateEndpoints(scheme string, cluster string, from int, to int) {
 	if cluster == "prod" {
-		cluster = ""
-	} else if cluster == "local" {
 		for port := from; port <= to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
+			endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
 		}
-		return
 	} else {
-		cluster = cluster + "."
-	}
-
 		for port := from; port <= to; port++ {
-		endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
+			endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster))
+		}
 	}
 
 	if includeLocalhost {
@@ -59,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
 }
 
 func cliUploadAndSync(c *cli.Context) error {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+
 	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
 
 	generateEndpoints(scheme, cluster, from, to)
@@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
 	time.Sleep(3 * time.Second)
 
 	log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
-	res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}}
+	res, err := client.Get(endpoint + "/bzz:/" + hash + "/")
 	if err != nil {
 		log.Warn(err.Error(), "ruid", ruid)
 		return err

cmd/swarm/upload_test.go

@@ -31,7 +31,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 	"github.com/mattn/go-colorable"
 )
@@ -41,69 +41,66 @@ func init() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 }
 
-// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
+func TestSwarmUp(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"NoEncryption", testNoEncryption},
+		{"Encrypted", testEncrypted},
+		{"RecursiveNoEncryption", testRecursiveNoEncryption},
+		{"RecursiveEncrypted", testRecursiveEncrypted},
+		{"DefaultPathAll", testDefaultPathAll},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testNoEncryption tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUp(t *testing.T) {
+func testNoEncryption(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	testDefault(false, t)
-		t.Skip()
-	}
-
-	testCLISwarmUp(false, t)
-}
-func TestCLISwarmUpRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(false, t)
 }
 
-// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
+// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUpEncrypted(t *testing.T) {
+func testEncrypted(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	testDefault(true, t)
-		t.Skip()
-	}
-	testCLISwarmUp(true, t)
-}
-func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(true, t)
 }
 
-func testCLISwarmUp(toEncrypt bool, t *testing.T) {
+func testRecursiveNoEncryption(t *testing.T) {
-	log.Info("starting 3 node cluster")
+	testRecursive(false, t)
-	cluster := newTestCluster(t, 3)
+}
-	defer cluster.Shutdown()
 
-	// create a tmp file
+func testRecursiveEncrypted(t *testing.T) {
-	tmp, err := ioutil.TempFile("", "swarm-test")
+	testRecursive(true, t)
-	if err != nil {
+}
-		t.Fatal(err)
+
-	}
+func testDefault(toEncrypt bool, t *testing.T) {
-	defer tmp.Close()
+	tmpFileName := testutil.TempFileWithContent(t, data)
-	defer os.Remove(tmp.Name())
+	defer os.Remove(tmpFileName)
 
 	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	hashRegexp := `[a-f\d]{64}`
 	flags := []string{
 		"--bzzapi", cluster.Nodes[0].URL,
 		"up",
-		tmp.Name()}
+		tmpFileName}
 	if toEncrypt {
 		hashRegexp = `[a-f\d]{128}`
 		flags = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"up",
 			"--encrypt",
-			tmp.Name()}
+			tmpFileName}
 	}
 	// upload the file with 'swarm up' and expect a hash
 	log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@@ -192,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
 	}
 }
 
-func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
+func testRecursive(toEncrypt bool, t *testing.T) {
-	fmt.Println("starting 3 node cluster")
-	cluster := newTestCluster(t, 3)
-	defer cluster.Shutdown()
-
 	tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpUploadDir)
 	// create tmp files
-	data := "notsorandomdata"
 	for _, path := range []string{"tmp1", "tmp2"} {
 		if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
 			t.Fatal(err)
@@ -243,8 +235,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 		}
 		defer os.RemoveAll(tmpDownload)
 		bzzLocator := "bzz:/" + hash
-		flagss := []string{}
+		flagss := []string{
-		flagss = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"down",
 			"--recursive",
@@ -265,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 
 			switch mode := fi.Mode(); {
 			case mode.IsRegular():
-				if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
+				if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
 					t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
 				} else {
 					ff := make([]byte, len(data))
@@ -286,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 	}
 }
 
-// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// testDefaultPathAll tests swarm recursive upload with relative and absolute
 // default paths and with encryption.
-func TestCLISwarmUpDefaultPath(t *testing.T) {
+func testDefaultPathAll(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	testDefaultPath(false, false, t)
-		t.Skip()
+	testDefaultPath(false, true, t)
-	}
+	testDefaultPath(true, false, t)
-	testCLISwarmUpDefaultPath(false, false, t)
+	testDefaultPath(true, true, t)
-	testCLISwarmUpDefaultPath(false, true, t)
-	testCLISwarmUpDefaultPath(true, false, t)
-	testCLISwarmUpDefaultPath(true, true, t)
 }
 
-func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
+func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
-	defer srv.Close()
-
 	tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
 	if err != nil {
 		t.Fatal(err)
@@ -324,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 
 	args := []string{
 		"--bzzapi",
-		srv.URL,
+		cluster.Nodes[0].URL,
 		"--recursive",
 		"--defaultpath",
 		defaultPath,
@@ -341,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 	up.ExpectExit()
 	hash := matches[0]
 
-	client := swarm.NewClient(srv.URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	m, isEncrypted, err := client.DownloadManifest(hash)
 	if err != nil {

cmd/utils/cmd.go

@@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
 		// Accumulate the preimages and flush when enough ws gathered
 		preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
 		if len(preimages) > 1024 {
-			rawdb.WritePreimages(db, 0, preimages)
+			rawdb.WritePreimages(db, preimages)
 			preimages = make(map[common.Hash][]byte)
 		}
 	}
 	// Flush the last batch preimage data
 	if len(preimages) > 0 {
-		rawdb.WritePreimages(db, 0, preimages)
+		rawdb.WritePreimages(db, preimages)
 	}
 	return nil
 }

cmd/utils/flags.go

@@ -295,7 +295,12 @@ var (
 	CacheDatabaseFlag = cli.IntFlag{
 		Name:  "cache.database",
 		Usage: "Percentage of cache memory allowance to use for database io",
-		Value: 75,
+		Value: 50,
+	}
+	CacheTrieFlag = cli.IntFlag{
+		Name:  "cache.trie",
+		Usage: "Percentage of cache memory allowance to use for trie caching",
+		Value: 25,
 	}
 	CacheGCFlag = cli.IntFlag{
 		Name:  "cache.gc",
@@ -973,16 +978,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setWS(ctx, cfg)
 	setNodeUserIdent(ctx, cfg)
 
-	switch {
+	setDataDir(ctx, cfg)
-	case ctx.GlobalIsSet(DataDirFlag.Name):
-		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
-	case ctx.GlobalBool(DeveloperFlag.Name):
-		cfg.DataDir = "" // unless explicitly requested, use memory databases
-	case ctx.GlobalBool(TestnetFlag.Name):
-		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
-	case ctx.GlobalBool(RinkebyFlag.Name):
-		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
-	}
 
 	if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
 		cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
@@ -995,6 +991,19 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	}
 }
 
+func setDataDir(ctx *cli.Context, cfg *node.Config) {
+	switch {
+	case ctx.GlobalIsSet(DataDirFlag.Name):
+		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
+	case ctx.GlobalBool(DeveloperFlag.Name):
+		cfg.DataDir = "" // unless explicitly requested, use memory databases
+	case ctx.GlobalBool(TestnetFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
+	case ctx.GlobalBool(RinkebyFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
+	}
+}
+
 func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
 	if ctx.GlobalIsSet(GpoBlocksFlag.Name) {
 		cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name)
@@ -1157,8 +1166,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	}
 	cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
 
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
 		cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
@@ -1394,11 +1406,15 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	}
 	cache := &core.CacheConfig{
 		Disabled:       ctx.GlobalString(GCModeFlag.Name) == "archive",
-		TrieNodeLimit: eth.DefaultConfig.TrieCache,
+		TrieCleanLimit: eth.DefaultConfig.TrieCleanCache,
+		TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
 		TrieTimeLimit:  eth.DefaultConfig.TrieTimeout,
 	}
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
 	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)

common/compiler/solidity.go

@@ -31,14 +31,15 @@ import (
 
 var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
-// Contract contains information about a compiled contract, alongside its code.
+// Contract contains information about a compiled contract, alongside its code and runtime code.
 type Contract struct {
 	Code        string       `json:"code"`
+	RuntimeCode string       `json:"runtime-code"`
 	Info        ContractInfo `json:"info"`
 }
 
 // ContractInfo contains information about a compiled contract, including access
-// to the ABI definition, user and developer docs, and metadata.
+// to the ABI definition, source mapping, user and developer docs, and metadata.
 //
 // Depending on the source, language version, compiler version, and compiler
 // options will provide information about how the contract was compiled.
@@ -48,6 +49,8 @@ type ContractInfo struct {
 	LanguageVersion string      `json:"languageVersion"`
 	CompilerVersion string      `json:"compilerVersion"`
 	CompilerOptions string      `json:"compilerOptions"`
+	SrcMap          string      `json:"srcMap"`
+	SrcMapRuntime   string      `json:"srcMapRuntime"`
 	AbiDefinition   interface{} `json:"abiDefinition"`
 	UserDoc         interface{} `json:"userDoc"`
 	DeveloperDoc    interface{} `json:"developerDoc"`
@@ -63,14 +66,16 @@ type Solidity struct {
 // --combined-output format
 type solcOutput struct {
 	Contracts map[string]struct {
-		Bin, Abi, Devdoc, Userdoc, Metadata string
+		BinRuntime                                  string `json:"bin-runtime"`
+		SrcMapRuntime                               string `json:"srcmap-runtime"`
+		Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string
 	}
 	Version string
 }
 
 func (s *Solidity) makeArgs() []string {
 	p := []string{
-		"--combined-json", "bin,abi,userdoc,devdoc",
+		"--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
 		"--optimize", // code optimizer switched on
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
@@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro
 // provided source, language and compiler version, and compiler options are all
 // passed through into the Contract structs.
 //
-// The solc output is expected to contain ABI, user docs, and dev docs.
+// The solc output is expected to contain ABI, source mapping, user docs, and dev docs.
 //
 // Returns an error if the JSON is malformed or missing data, or if the JSON
 // embedded within the JSON is malformed.
@@ -185,12 +190,15 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
 		}
 		contracts[name] = &Contract{
 			Code:        "0x" + info.Bin,
+			RuntimeCode: "0x" + info.BinRuntime,
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",
 				LanguageVersion: languageVersion,
 				CompilerVersion: compilerVersion,
 				CompilerOptions: compilerOptions,
+				SrcMap:          info.SrcMap,
+				SrcMapRuntime:   info.SrcMapRuntime,
 				AbiDefinition:   abi,
 				UserDoc:         userdoc,
 				DeveloperDoc:    devdoc,

consensus/clique/clique.go

@@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash {
 	return sigHash(header)
 }
 
-// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
+// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
 func (c *Clique) Close() error {
 	return nil
 }

consensus/ethash/api.go

@@ -37,27 +37,28 @@ type API struct {
 //   result[0] - 32 bytes hex encoded current block header pow-hash
 //   result[1] - 32 bytes hex encoded seed hash used for DAG
 //   result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
-func (api *API) GetWork() ([3]string, error) {
+//   result[3] - hex encoded block number
+func (api *API) GetWork() ([4]string, error) {
 	if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest {
-		return [3]string{}, errors.New("not supported")
+		return [4]string{}, errors.New("not supported")
 	}
 
 	var (
-		workCh = make(chan [3]string, 1)
+		workCh = make(chan [4]string, 1)
 		errc   = make(chan error, 1)
 	)
 
 	select {
 	case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}:
 	case <-api.ethash.exitCh:
-		return [3]string{}, errEthashStopped
+		return [4]string{}, errEthashStopped
 	}
 
 	select {
 	case work := <-workCh:
 		return work, nil
 	case err := <-errc:
-		return [3]string{}, err
+		return [4]string{}, err
 	}
 }
 

consensus/ethash/ethash.go

@@ -432,7 +432,7 @@ type hashrate struct {
 // sealWork wraps a seal work package for remote sealer.
 type sealWork struct {
 	errc chan error
-	res  chan [3]string
+	res  chan [4]string
 }
 
 // Ethash is a consensus engine based on proof-of-work implementing the ethash

consensus/ethash/ethash_test.go

@@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) {
 	ethash.Seal(nil, block, results, nil)
 
 	var (
-		work [3]string
+		work [4]string
 		err  error
 	)
 	if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {

consensus/ethash/sealer.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 
 		results      chan<- *types.Block
 		currentBlock *types.Block
-		currentWork  [3]string
+		currentWork  [4]string
 
 		notifyTransport = &http.Transport{}
 		notifyClient    = &http.Client{
@@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 	//   result[0], 32 bytes hex encoded current block header pow-hash
 	//   result[1], 32 bytes hex encoded seed hash used for DAG
 	//   result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
+	//   result[3], hex encoded block number
 	makeWork := func(block *types.Block) {
 		hash := ethash.SealHash(block.Header())
 
 		currentWork[0] = hash.Hex()
 		currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
 		currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex()
+		currentWork[3] = hexutil.EncodeBig(block.Number())
 
 		// Trace the seal work fetched by remote sealer.
 		currentBlock = block

core/block_validator.go

@@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
 		return ErrKnownBlock
 	}
-	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
-		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
-			return consensus.ErrUnknownAncestor
-		}
-		return consensus.ErrPrunedAncestor
-	}
 	// Header validity is known at this point, check the uncles and transactions
 	header := block.Header()
 	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
@@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
 		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
 	}
+	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
+			return consensus.ErrUnknownAncestor
+		}
+		return consensus.ErrPrunedAncestor
+	}
 	return nil
 }
 

core/blockchain.go

@@ -69,7 +69,8 @@ const (
 // that's resident in a blockchain.
 type CacheConfig struct {
 	Disabled       bool          // Whether to disable trie write caching (archive node)
-	TrieNodeLimit int           // Memory limit (MB) at which to flush the current in-memory trie to disk
+	TrieCleanLimit int           // Memory allowance (MB) to use for caching trie nodes in memory
+	TrieDirtyLimit int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
 	TrieTimeLimit  time.Duration // Time limit after which to flush the current in-memory trie to disk
 }
 
@@ -140,7 +141,8 @@ type BlockChain struct {
 func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = &CacheConfig{
-			TrieNodeLimit: 256 * 1024 * 1024,
+			TrieCleanLimit: 256,
+			TrieDirtyLimit: 256,
 			TrieTimeLimit:  5 * time.Minute,
 		}
 	}
@@ -156,7 +158,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		cacheConfig:    cacheConfig,
 		db:             db,
 		triegc:         prque.New(nil),
-		stateCache:     state.NewDatabase(db),
+		stateCache:     state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
 		quit:           make(chan struct{}),
 		shouldPreserve: shouldPreserve,
 		bodyCache:      bodyCache,
@@ -393,6 +395,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
 	return state.New(root, bc.stateCache)
 }
 
+// StateCache returns the caching database underpinning the blockchain instance.
+func (bc *BlockChain) StateCache() state.Database {
+	return bc.stateCache
+}
+
 // Reset purges the entire blockchain, restoring it to its genesis state.
 func (bc *BlockChain) Reset() error {
 	return bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -438,7 +445,11 @@ func (bc *BlockChain) repair(head **types.Block) error {
 			return nil
 		}
 		// Otherwise rewind one block and recheck state availability there
-		(*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+		block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+		if block == nil {
+			return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
+		}
+		(*head) = block
 	}
 }
 
@@ -554,6 +565,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
 	return rawdb.HasBody(bc.db, hash, number)
 }
 
+// HasFastBlock checks if a fast block is fully present in the database or not.
+func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
+	if !bc.HasBlock(hash, number) {
+		return false
+	}
+	if bc.receiptsCache.Contains(hash) {
+		return true
+	}
+	return rawdb.HasReceipts(bc.db, hash, number)
+}
+
 // HasState checks if state trie is fully present in the database or not.
 func (bc *BlockChain) HasState(hash common.Hash) bool {
 	_, err := bc.stateCache.OpenTrie(hash)
@@ -611,12 +633,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
 		return receipts.(types.Receipts)
 	}
-
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
-
 	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
 	bc.receiptsCache.Add(hash, receipts)
 	return receipts
@@ -938,7 +958,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 			// If we exceeded our memory allowance, flush matured singleton nodes to disk
 			var (
 				nodes, imgs = triedb.Size()
-				limit       = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
+				limit       = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
 			)
 			if nodes > limit || imgs > 4*1024*1024 {
 				triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1002,7 +1022,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 		}
 		// Write the positional metadata for transaction/receipt lookups and preimages
 		rawdb.WriteTxLookupEntries(batch, block)
-		rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages())
+		rawdb.WritePreimages(batch, state.Preimages())
 
 		status = CanonStatTy
 	} else {
@@ -1020,6 +1040,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	return status, nil
 }
 
+// addFutureBlock checks if the block is within the max allowed window to get
+// accepted for future processing, and returns an error if the block is too far
+// ahead and was not added.
+func (bc *BlockChain) addFutureBlock(block *types.Block) error {
+	max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
+	if block.Time().Cmp(max) > 0 {
+		return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
+	}
+	bc.futureBlocks.Add(block.Hash(), block)
+	return nil
+}
+
 // InsertChain attempts to insert the given batch of blocks in to the canonical
 // chain or, otherwise, create a fork. If an error is returned it will return
 // the index number of the failing block as well an error describing what went
@@ -1027,18 +1059,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 //
 // After insertion is done, all accumulated events will be fired.
 func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
-	n, events, logs, err := bc.insertChain(chain)
-	bc.PostChainEvents(events, logs)
-	return n, err
-}
-
-// insertChain will execute the actual chain insertion and event aggregation. The
-// only reason this method exists as a separate one is to make locking cleaner
-// with deferred statements.
-func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) {
 	// Sanity check that we have something meaningful to import
 	if len(chain) == 0 {
-		return 0, nil, nil, nil
+		return 0, nil
 	}
 	// Do a sanity check that the provided chain is actually ordered and linked
 	for i := 1; i < len(chain); i++ {
@@ -1047,16 +1070,36 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
 				"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
 
-			return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
+			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
 				chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
 		}
 	}
 	// Pre-checks passed, start the full block imports
 	bc.wg.Add(1)
-	defer bc.wg.Done()
-
 	bc.chainmu.Lock()
-	defer bc.chainmu.Unlock()
+	n, events, logs, err := bc.insertChain(chain, true)
+	bc.chainmu.Unlock()
+	bc.wg.Done()
+
+	bc.PostChainEvents(events, logs)
+	return n, err
+}
+
+// insertChain is the internal implementation of insertChain, which assumes that
+// 1) chains are contiguous, and 2) The chain mutex is held.
+//
+// This method is split out so that import batches that require re-injecting
+// historical blocks can do so without releasing the lock, which could lead to
+// racey behaviour. If a sidechain import is in progress, and the historic state
+// is imported, but then new canon-head is added before the actual sidechain
+// completes, then the historic state could be pruned again
+func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) {
+	// If the chain is terminating, don't even bother starting u
+	if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+		return 0, nil, nil, nil
+	}
+	// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
+	senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
 
 	// A queued approach to delivering events. This is generally
 	// faster than direct delivery and requires much less mutex
@@ -1073,16 +1116,56 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 
 	for i, block := range chain {
 		headers[i] = block.Header()
-		seals[i] = true
+		seals[i] = verifySeals
 	}
 	abort, results := bc.engine.VerifyHeaders(bc, headers, seals)
 	defer close(abort)
 
-	// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
+	// Peek the error for the first block to decide the directing import logic
-	senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
+	it := newInsertIterator(chain, results, bc.Validator())
 
-	// Iterate over the blocks and insert when the verifier permits
+	block, err := it.next()
-	for i, block := range chain {
+	switch {
+	// First block is pruned, insert as sidechain and reorg only if TD grows enough
+	case err == consensus.ErrPrunedAncestor:
+		return bc.insertSidechain(it)
+
+	// First block is future, shove it (and all children) to the future queue (unknown ancestor)
+	case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
+		for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) {
+			if err := bc.addFutureBlock(block); err != nil {
+				return it.index, events, coalescedLogs, err
+			}
+			block, err = it.next()
+		}
+		stats.queued += it.processed()
+		stats.ignored += it.remaining()
+
+		// If there are any still remaining, mark as ignored
+		return it.index, events, coalescedLogs, err
+
+	// First block (and state) is known
+	//   1. We did a roll-back, and should now do a re-import
+	//   2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
+	// 	    from the canonical chain, which has not been verified.
+	case err == ErrKnownBlock:
+		// Skip all known blocks that behind us
+		current := bc.CurrentBlock().NumberU64()
+
+		for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
+			stats.ignored++
+			block, err = it.next()
+		}
+		// Falls through to the block import
+
+	// Some other error occurred, abort
+	case err != nil:
+		stats.ignored += len(it.chain)
+		bc.reportBlock(block, nil, err)
+		return it.index, events, coalescedLogs, err
+	}
+	// No validation errors for the first block (or chain prefix skipped)
+	for ; block != nil && err == nil; block, err = it.next() {
 		// If the chain is terminating, stop processing blocks
 		if atomic.LoadInt32(&bc.procInterrupt) == 1 {
 			log.Debug("Premature abort during blocks processing")
@@ -1091,115 +1174,45 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		// If the header is a banned one, straight out abort
 		if BadHashes[block.Hash()] {
 			bc.reportBlock(block, nil, ErrBlacklistedHash)
-			return i, events, coalescedLogs, ErrBlacklistedHash
+			return it.index, events, coalescedLogs, ErrBlacklistedHash
 		}
-		// Wait for the block's verification to complete
+		// Retrieve the parent block and it's state to execute on top
-		bstart := time.Now()
+		start := time.Now()
 
-		err := <-results
+		parent := it.previous()
-		if err == nil {
+		if parent == nil {
-			err = bc.Validator().ValidateBody(block)
-		}
-		switch {
-		case err == ErrKnownBlock:
-			// Block and state both already known. However if the current block is below
-			// this number we did a rollback and we should reimport it nonetheless.
-			if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
-				stats.ignored++
-				continue
-			}
-
-		case err == consensus.ErrFutureBlock:
-			// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
-			// the chain is discarded and processed at a later time if given.
-			max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
-			if block.Time().Cmp(max) > 0 {
-				return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
-			}
-			bc.futureBlocks.Add(block.Hash(), block)
-			stats.queued++
-			continue
-
-		case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
-			bc.futureBlocks.Add(block.Hash(), block)
-			stats.queued++
-			continue
-
-		case err == consensus.ErrPrunedAncestor:
-			// Block competing with the canonical chain, store in the db, but don't process
-			// until the competitor TD goes above the canonical TD
-			currentBlock := bc.CurrentBlock()
-			localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
-			externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
-			if localTd.Cmp(externTd) > 0 {
-				if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
-					return i, events, coalescedLogs, err
-				}
-				continue
-			}
-			// Competitor chain beat canonical, gather all blocks from the common ancestor
-			var winner []*types.Block
-
-			parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-			for !bc.HasState(parent.Root()) {
-				winner = append(winner, parent)
-				parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
-			}
-			for j := 0; j < len(winner)/2; j++ {
-				winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
-			}
-			// Import all the pruned blocks to make the state available
-			bc.chainmu.Unlock()
-			_, evs, logs, err := bc.insertChain(winner)
-			bc.chainmu.Lock()
-			events, coalescedLogs = evs, logs
-
-			if err != nil {
-				return i, events, coalescedLogs, err
-			}
-
-		case err != nil:
-			bc.reportBlock(block, nil, err)
-			return i, events, coalescedLogs, err
-		}
-		// Create a new statedb using the parent block and report an
-		// error if it fails.
-		var parent *types.Block
-		if i == 0 {
 			parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-		} else {
-			parent = chain[i-1]
 		}
 		state, err := state.New(parent.Root(), bc.stateCache)
 		if err != nil {
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		// Process block using the parent state as reference point.
 		receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig)
 		if err != nil {
 			bc.reportBlock(block, receipts, err)
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		// Validate the state using the default validator
-		err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas)
+		if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil {
-		if err != nil {
 			bc.reportBlock(block, receipts, err)
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
-		proctime := time.Since(bstart)
+		proctime := time.Since(start)
 
 		// Write the block to the chain and get the status.
 		status, err := bc.WriteBlockWithState(block, receipts, state)
 		if err != nil {
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		switch status {
 		case CanonStatTy:
-			log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()),
+			log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(),
-				"txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart)))
+				"uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(),
+				"elapsed", common.PrettyDuration(time.Since(start)),
+				"root", block.Root())
 
 			coalescedLogs = append(coalescedLogs, logs...)
-			blockInsertTimer.UpdateSince(bstart)
 			events = append(events, ChainEvent{block, block.Hash(), logs})
 			lastCanon = block
 
@@ -1207,78 +1220,153 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			bc.gcproc += proctime
 
 		case SideStatTy:
-			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
+			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(),
-				common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
+				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
-
+				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
-			blockInsertTimer.UpdateSince(bstart)
+				"root", block.Root())
 			events = append(events, ChainSideEvent{block})
 		}
+		blockInsertTimer.UpdateSince(start)
 		stats.processed++
 		stats.usedGas += usedGas
 
 		cache, _ := bc.stateCache.TrieDB().Size()
-		stats.report(chain, i, cache)
+		stats.report(chain, it.index, cache)
 	}
+	// Any blocks remaining here? The only ones we care about are the future ones
+	if block != nil && err == consensus.ErrFutureBlock {
+		if err := bc.addFutureBlock(block); err != nil {
+			return it.index, events, coalescedLogs, err
+		}
+		block, err = it.next()
+
+		for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() {
+			if err := bc.addFutureBlock(block); err != nil {
+				return it.index, events, coalescedLogs, err
+			}
+			stats.queued++
+		}
+	}
+	stats.ignored += it.remaining()
+
 	// Append a single chain head event if we've progressed the chain
 	if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
 		events = append(events, ChainHeadEvent{lastCanon})
 	}
-	return 0, events, coalescedLogs, nil
+	return it.index, events, coalescedLogs, err
 }
 
-// insertStats tracks and reports on block insertion.
+// insertSidechain is called when an import batch hits upon a pruned ancestor
-type insertStats struct {
+// error, which happens when a sidechain with a sufficiently old fork-block is
-	queued, processed, ignored int
+// found.
-	usedGas                    uint64
+//
-	lastIndex                  int
+// The method writes all (header-and-body-valid) blocks to disk, then tries to
-	startTime                  mclock.AbsTime
+// switch over to the new chain if the TD exceeded the current chain.
-}
+func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
-
-// statsReportLimit is the time limit during import and export after which we
-// always print out progress. This avoids the user wondering what's going on.
-const statsReportLimit = 8 * time.Second
-
-// report prints statistics if some number of blocks have been processed
-// or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
-	// Fetch the timings for the batch
 	var (
-		now     = mclock.Now()
+		externTd *big.Int
-		elapsed = time.Duration(now) - time.Duration(st.startTime)
+		current  = bc.CurrentBlock().NumberU64()
 	)
-	// If we're at the last block of the batch or report period reached, log
+	// The first sidechain block error is already verified to be ErrPrunedAncestor.
-	if index == len(chain)-1 || elapsed >= statsReportLimit {
+	// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
+	// ones. Any other errors means that the block is invalid, and should not be written
+	// to disk.
+	block, err := it.current(), consensus.ErrPrunedAncestor
+	for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
+		// Check the canonical state root for that number
+		if number := block.NumberU64(); current >= number {
+			if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
+				// This is most likely a shadow-state attack. When a fork is imported into the
+				// database, and it eventually reaches a block height which is not pruned, we
+				// just found that the state already exist! This means that the sidechain block
+				// refers to a state which already exists in our canon chain.
+				//
+				// If left unchecked, we would now proceed importing the blocks, without actually
+				// having verified the state of the previous blocks.
+				log.Warn("Sidechain ghost-state attack detected", "number", block.NumberU64(), "sideroot", block.Root(), "canonroot", canonical.Root())
+
+				// If someone legitimately side-mines blocks, they would still be imported as usual. However,
+				// we cannot risk writing unverified blocks to disk when they obviously target the pruning
+				// mechanism.
+				return it.index, nil, nil, errors.New("sidechain ghost-state attack")
+			}
+		}
+		if externTd == nil {
+			externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1)
+		}
+		externTd = new(big.Int).Add(externTd, block.Difficulty())
+
+		if !bc.HasBlock(block.Hash(), block.NumberU64()) {
+			start := time.Now()
+			if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
+				return it.index, nil, nil, err
+			}
+			log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
+				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
+				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
+				"root", block.Root())
+		}
+	}
+	// At this point, we've written all sidechain blocks to database. Loop ended
+	// either on some other error or all were processed. If there was some other
+	// error, we can ignore the rest of those blocks.
+	//
+	// If the externTd was larger than our local TD, we now need to reimport the previous
+	// blocks to regenerate the required state
+	localTd := bc.GetTd(bc.CurrentBlock().Hash(), current)
+	if localTd.Cmp(externTd) > 0 {
+		log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd)
+		return it.index, nil, nil, err
+	}
+	// Gather all the sidechain hashes (full blocks may be memory heavy)
 	var (
-			end = chain[index]
+		hashes  []common.Hash
-			txs = countTransactions(chain[st.lastIndex : index+1])
+		numbers []uint64
 	)
-		context := []interface{}{
+	parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64())
-			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
+	for parent != nil && !bc.HasState(parent.Root) {
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+		hashes = append(hashes, parent.Hash())
-			"number", end.Number(), "hash", end.Hash(),
+		numbers = append(numbers, parent.Number.Uint64())
-		}
-		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
-			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
-		}
-		context = append(context, []interface{}{"cache", cache}...)
 
-		if st.queued > 0 {
+		parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
-			context = append(context, []interface{}{"queued", st.queued}...)
 	}
-		if st.ignored > 0 {
+	if parent == nil {
-			context = append(context, []interface{}{"ignored", st.ignored}...)
+		return it.index, nil, nil, errors.New("missing parent")
 	}
-		log.Info("Imported new chain segment", context...)
+	// Import all the pruned blocks to make the state available
+	var (
+		blocks []*types.Block
+		memory common.StorageSize
+	)
+	for i := len(hashes) - 1; i >= 0; i-- {
+		// Append the next block to our batch
+		block := bc.GetBlock(hashes[i], numbers[i])
 
-		*st = insertStats{startTime: now, lastIndex: index + 1}
+		blocks = append(blocks, block)
-	}
+		memory += block.Size()
-}
 
-func countTransactions(chain []*types.Block) (c int) {
+		// If memory use grew too large, import and continue. Sadly we need to discard
-	for _, b := range chain {
+		// all raised events and logs from notifications since we're too heavy on the
-		c += len(b.Transactions())
+		// memory here.
+		if len(blocks) >= 2048 || memory > 64*1024*1024 {
+			log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64())
+			if _, _, _, err := bc.insertChain(blocks, false); err != nil {
+				return 0, nil, nil, err
 			}
-	return c
+			blocks, memory = blocks[:0], 0
+
+			// If the chain is terminating, stop processing blocks
+			if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+				log.Debug("Premature abort during blocks processing")
+				return 0, nil, nil, nil
+			}
+		}
+	}
+	if len(blocks) > 0 {
+		log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64())
+		return bc.insertChain(blocks, false)
+	}
+	return 0, nil, nil, nil
 }
 
 // reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them
@@ -1453,8 +1541,10 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e
 	bc.addBadBlock(block)
 
 	var receiptString string
-	for _, receipt := range receipts {
+	for i, receipt := range receipts {
-		receiptString += fmt.Sprintf("\t%v\n", receipt)
+		receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n",
+			i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(),
+			receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState)
 	}
 	log.Error(fmt.Sprintf(`
 ########## BAD BLOCK #########

core/blockchain_insert.go

@@ -0,0 +1,143 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// insertStats tracks and reports on block insertion.
+type insertStats struct {
+	queued, processed, ignored int
+	usedGas                    uint64
+	lastIndex                  int
+	startTime                  mclock.AbsTime
+}
+
+// statsReportLimit is the time limit during import and export after which we
+// always print out progress. This avoids the user wondering what's going on.
+const statsReportLimit = 8 * time.Second
+
+// report prints statistics if some number of blocks have been processed
+// or more than a few seconds have passed since the last message.
+func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
+	// Fetch the timings for the batch
+	var (
+		now     = mclock.Now()
+		elapsed = time.Duration(now) - time.Duration(st.startTime)
+	)
+	// If we're at the last block of the batch or report period reached, log
+	if index == len(chain)-1 || elapsed >= statsReportLimit {
+		// Count the number of transactions in this segment
+		var txs int
+		for _, block := range chain[st.lastIndex : index+1] {
+			txs += len(block.Transactions())
+		}
+		end := chain[index]
+
+		// Assemble the log context and send it to the logger
+		context := []interface{}{
+			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"number", end.Number(), "hash", end.Hash(),
+		}
+		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
+			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
+		}
+		context = append(context, []interface{}{"cache", cache}...)
+
+		if st.queued > 0 {
+			context = append(context, []interface{}{"queued", st.queued}...)
+		}
+		if st.ignored > 0 {
+			context = append(context, []interface{}{"ignored", st.ignored}...)
+		}
+		log.Info("Imported new chain segment", context...)
+
+		// Bump the stats reported to the next section
+		*st = insertStats{startTime: now, lastIndex: index + 1}
+	}
+}
+
+// insertIterator is a helper to assist during chain import.
+type insertIterator struct {
+	chain     types.Blocks
+	results   <-chan error
+	index     int
+	validator Validator
+}
+
+// newInsertIterator creates a new iterator based on the given blocks, which are
+// assumed to be a contiguous chain.
+func newInsertIterator(chain types.Blocks, results <-chan error, validator Validator) *insertIterator {
+	return &insertIterator{
+		chain:     chain,
+		results:   results,
+		index:     -1,
+		validator: validator,
+	}
+}
+
+// next returns the next block in the iterator, along with any potential validation
+// error for that block. When the end is reached, it will return (nil, nil).
+func (it *insertIterator) next() (*types.Block, error) {
+	if it.index+1 >= len(it.chain) {
+		it.index = len(it.chain)
+		return nil, nil
+	}
+	it.index++
+	if err := <-it.results; err != nil {
+		return it.chain[it.index], err
+	}
+	return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
+}
+
+// current returns the current block that's being processed.
+func (it *insertIterator) current() *types.Block {
+	if it.index < 0 || it.index+1 >= len(it.chain) {
+		return nil
+	}
+	return it.chain[it.index]
+}
+
+// previous returns the previous block was being processed, or nil
+func (it *insertIterator) previous() *types.Block {
+	if it.index < 1 {
+		return nil
+	}
+	return it.chain[it.index-1]
+}
+
+// first returns the first block in the it.
+func (it *insertIterator) first() *types.Block {
+	return it.chain[0]
+}
+
+// remaining returns the number of remaining blocks.
+func (it *insertIterator) remaining() int {
+	return len(it.chain) - it.index
+}
+
+// processed returns the number of processed blocks.
+func (it *insertIterator) processed() int {
+	return it.index + 1
+}

core/blockchain_test.go

@@ -579,11 +579,11 @@ func testInsertNonceError(t *testing.T, full bool) {
 			blockchain.hc.engine = blockchain.engine
 			failRes, err = blockchain.InsertHeaderChain(headers, 1)
 		}
-		// Check that the returned error indicates the failure.
+		// Check that the returned error indicates the failure
 		if failRes != failAt {
-			t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
+			t.Errorf("test %d: failure (%v) index mismatch: have %d, want %d", i, err, failRes, failAt)
 		}
-		// Check that all no blocks after the failing block have been inserted.
+		// Check that all blocks after the failing block have been inserted
 		for j := 0; j < i-failAt; j++ {
 			if full {
 				if block := blockchain.GetBlockByNumber(failNum + uint64(j)); block != nil {
@@ -1345,7 +1345,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		t.Fatalf("failed to insert shared chain: %v", err)
 	}
 	if _, err := chain.InsertChain(original); err != nil {
-		t.Fatalf("failed to insert shared chain: %v", err)
+		t.Fatalf("failed to insert original chain: %v", err)
 	}
 	// Ensure that the state associated with the forking point is pruned away
 	if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {

core/chain_makers.go

@@ -36,7 +36,6 @@ type BlockGen struct {
 	i       int
 	parent  *types.Block
 	chain   []*types.Block
-	chainReader consensus.ChainReader
 	header  *types.Header
 	statedb *state.StateDB
 
@@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) {
 // For index -1, PrevBlock returns the parent block given to GenerateChain.
 func (b *BlockGen) PrevBlock(index int) *types.Block {
 	if index >= b.i {
-		panic("block index out of range")
+		panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
 	}
 	if index == -1 {
 		return b.parent
@@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}
-	b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header())
+	chainreader := &fakeChainReader{config: b.config}
+	b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		config = params.TestChainConfig
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	chainreader := &fakeChainReader{config: config}
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
-		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
+		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
-		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
+		b.header = makeHeader(chainreader, parent, statedb, b.engine)
-		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
-		defer blockchain.Stop()
-
-		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
-		b.header = makeHeader(b.chainReader, parent, statedb, b.engine)
 
 		// Mutate the state and block according to any hard-fork specs
 		if daoBlock := config.DAOForkBlock; daoBlock != nil {
@@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		}
 		if b.engine != nil {
 			// Finalize and seal the block
-			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+			block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
 
 			// Write state changes to db
 			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
@@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd
 	})
 	return blocks
 }
+
+type fakeChainReader struct {
+	config  *params.ChainConfig
+	genesis *types.Block
+}
+
+// Config returns the chain configuration.
+func (cr *fakeChainReader) Config() *params.ChainConfig {
+	return cr.config
+}
+
+func (cr *fakeChainReader) CurrentHeader() *types.Header                            { return nil }
+func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header           { return nil }
+func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header          { return nil }
+func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block   { return nil }

core/rawdb/accessors_chain.go

@@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
 	}
 }
 
+// HasReceipts verifies the existence of all the transaction receipts belonging
+// to a block.
+func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
+	if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
+		return false
+	}
+	return true
+}
+
 // ReadReceipts retrieves all the transaction receipts belonging to a block.
 func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
 	// Retrieve the flattened receipt slice

core/rawdb/accessors_metadata.go

@@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte {
 	return data
 }
 
-// WritePreimages writes the provided set of preimages to the database. `number` is the
+// WritePreimages writes the provided set of preimages to the database.
-// current block number, and is used for debug messages only.
+func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) {
-func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) {
 	for hash, preimage := range preimages {
 		if err := db.Put(preimageKey(hash), preimage); err != nil {
 			log.Crit("Failed to store trie preimage", "err", err)

core/rawdb/schema.go

@@ -35,7 +35,7 @@ var (
 	// headBlockKey tracks the latest know full block's hash.
 	headBlockKey = []byte("LastBlock")
 
-	// headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync.
+	// headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
 	headFastBlockKey = []byte("LastFast")
 
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.

core/state/database.go

@@ -72,13 +72,19 @@ type Trie interface {
 }
 
 // NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains cached trie nodes in memory. The pool is an optional
+// concurrent use and retains a few recent expanded trie nodes in memory. To keep
-// intermediate trie-node memory pool between the low level storage layer and the
+// more historical state in memory, use the NewDatabaseWithCache constructor.
-// high level trie abstraction.
 func NewDatabase(db ethdb.Database) Database {
+	return NewDatabaseWithCache(db, 0)
+}
+
+// NewDatabase creates a backing store for state. The returned database is safe for
+// concurrent use and retains both a few recent expanded trie nodes in memory, as
+// well as a lot of collapsed RLP trie nodes in a large memory cache.
+func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	return &cachingDB{
-		db:            trie.NewDatabase(db),
+		db:            trie.NewDatabaseWithCache(db, cache),
 		codeSizeCache: csc,
 	}
 }

core/state/statedb.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"math/big"
 	"sort"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -87,8 +86,6 @@ type StateDB struct {
 	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
-
-	lock sync.Mutex
 }
 
 // Create a new state from a given trie.
@@ -496,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common
 // Copy creates a deep, independent copy of the state.
 // Snapshots of the copied state cannot be applied to the copy.
 func (self *StateDB) Copy() *StateDB {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-
 	// Copy all the basic fields, initialize the memory ones
 	state := &StateDB{
 		db:                self.db,

core/tx_pool.go

@@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
 // addTxsLocked attempts to queue a batch of transactions if they are valid,
 // whilst assuming the transaction pool lock is already held.
 func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
-	// Add the batch of transaction, tracking the accepted ones
+	// Add the batch of transactions, tracking the accepted ones
 	dirty := make(map[common.Address]struct{})
 	errs := make([]error, len(txs))
 

core/types/block.go

@@ -81,8 +81,8 @@ type Header struct {
 	GasUsed     uint64         `json:"gasUsed"          gencodec:"required"`
 	Time        *big.Int       `json:"timestamp"        gencodec:"required"`
 	Extra       []byte         `json:"extraData"        gencodec:"required"`
-	MixDigest   common.Hash    `json:"mixHash"          gencodec:"required"`
+	MixDigest   common.Hash    `json:"mixHash"`
-	Nonce       BlockNonce     `json:"nonce"            gencodec:"required"`
+	Nonce       BlockNonce     `json:"nonce"`
 }
 
 // field type overrides for gencodec

core/types/gen_header_json.go

@@ -13,6 +13,7 @@ import (
 
 var _ = (*headerMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (h Header) MarshalJSON() ([]byte, error) {
 	type Header struct {
 		ParentHash  common.Hash    `json:"parentHash"       gencodec:"required"`
@@ -28,8 +29,8 @@ func (h Header) MarshalJSON() ([]byte, error) {
 		GasUsed     hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
 		Time        *hexutil.Big   `json:"timestamp"        gencodec:"required"`
 		Extra       hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest   common.Hash    `json:"mixHash"          gencodec:"required"`
+		MixDigest   common.Hash    `json:"mixHash"`
-		Nonce       BlockNonce     `json:"nonce"            gencodec:"required"`
+		Nonce       BlockNonce     `json:"nonce"`
 		Hash        common.Hash    `json:"hash"`
 	}
 	var enc Header
@@ -52,6 +53,7 @@ func (h Header) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (h *Header) UnmarshalJSON(input []byte) error {
 	type Header struct {
 		ParentHash  *common.Hash    `json:"parentHash"       gencodec:"required"`
@@ -67,8 +69,8 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 		GasUsed     *hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
 		Time        *hexutil.Big    `json:"timestamp"        gencodec:"required"`
 		Extra       *hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest   *common.Hash    `json:"mixHash"          gencodec:"required"`
+		MixDigest   *common.Hash    `json:"mixHash"`
-		Nonce       *BlockNonce     `json:"nonce"            gencodec:"required"`
+		Nonce       *BlockNonce     `json:"nonce"`
 	}
 	var dec Header
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -126,13 +128,11 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 		return errors.New("missing required field 'extraData' for Header")
 	}
 	h.Extra = *dec.Extra
-	if dec.MixDigest == nil {
+	if dec.MixDigest != nil {
-		return errors.New("missing required field 'mixHash' for Header")
-	}
 		h.MixDigest = *dec.MixDigest
-	if dec.Nonce == nil {
-		return errors.New("missing required field 'nonce' for Header")
 	}
+	if dec.Nonce != nil {
 		h.Nonce = *dec.Nonce
+	}
 	return nil
 }

core/vm/instructions.go

@@ -544,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract,
 // this account should be regarded as a non-existent account and zero should be returned.
 func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	slot := stack.peek()
-	slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes())
+	address := common.BigToAddress(slot)
+	if interpreter.evm.StateDB.Empty(address) {
+		slot.SetUint64(0)
+	} else {
+		slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes())
+	}
 	return nil, nil
 }
 

core/vm/logger_test.go

@@ -46,7 +46,7 @@ type dummyStatedb struct {
 	state.StateDB
 }
 
-func (dummyStatedb) GetRefund() uint64 { return 1337 }
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
 
 func TestStoreCapture(t *testing.T) {
 	var (

eth/api.go

@@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
 	if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
 		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
 	}
+	triedb := api.eth.BlockChain().StateCache().TrieDB()
 
-	oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+	oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0)
 	if err != nil {
 		return nil, err
 	}
-	newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+	newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0)
 	if err != nil {
 		return nil, err
 	}
-
 	diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}))
 	iter := trie.NewIterator(diff)
 

eth/api_tracer.go

@@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 
 	// Ensure we have a valid starting state before doing any work
 	origin := start.NumberU64()
-	database := state.NewDatabase(api.eth.ChainDb())
+	database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis
 
 	if number := start.NumberU64(); number > 0 {
 		start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
@@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 	}
 	// Otherwise try to reexec blocks until we find a state or reach our limit
 	origin := block.NumberU64()
-	database := state.NewDatabase(api.eth.ChainDb())
+	database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16)
 
 	for i := uint64(0); i < reexec; i++ {
 		block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)

eth/backend.go

@@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 			EWASMInterpreter:        config.EWASMInterpreter,
 			EVMInterpreter:          config.EVMInterpreter,
 		}
-		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
+		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout}
 	)
 	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
 	if err != nil {

eth/config.go

@@ -45,8 +45,9 @@ var DefaultConfig = Config{
 	},
 	NetworkId:      1,
 	LightPeers:     100,
-	DatabaseCache: 768,
+	DatabaseCache:  512,
-	TrieCache:     256,
+	TrieCleanCache: 256,
+	TrieDirtyCache: 256,
 	TrieTimeout:    60 * time.Minute,
 	MinerGasFloor:  8000000,
 	MinerGasCeil:   8000000,
@@ -94,7 +95,8 @@ type Config struct {
 	SkipBcVersionCheck bool `toml:"-"`
 	DatabaseHandles    int  `toml:"-"`
 	DatabaseCache      int
-	TrieCache          int
+	TrieCleanCache     int
+	TrieDirtyCache     int
 	TrieTimeout        time.Duration
 
 	// Mining-related options
@@ -122,7 +124,7 @@ type Config struct {
 	// Miscellaneous options
 	DocRoot string `toml:"-"`
 
-	// Type of the EWASM interpreter ("" for detault)
+	// Type of the EWASM interpreter ("" for default)
 	EWASMInterpreter string
 	// Type of the EVM interpreter ("" for default)
 	EVMInterpreter string

eth/downloader/downloader.go

@@ -181,6 +181,9 @@ type BlockChain interface {
 	// HasBlock verifies a block's presence in the local chain.
 	HasBlock(common.Hash, uint64) bool
 
+	// HasFastBlock verifies a fast block's presence in the local chain.
+	HasFastBlock(common.Hash, uint64) bool
+
 	// GetBlockByHash retrieves a block from the local chain.
 	GetBlockByHash(common.Hash) *types.Block
 
@@ -430,7 +433,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
 	}
 	height := latest.Number.Uint64()
 
-	origin, err := d.findAncestor(p, height)
+	origin, err := d.findAncestor(p, latest)
 	if err != nil {
 		return err
 	}
@@ -587,41 +590,86 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) {
 	}
 }
 
+// calculateRequestSpan calculates what headers to request from a peer when trying to determine the
+// common ancestor.
+// It returns parameters to be used for peer.RequestHeadersByNumber:
+//  from - starting block number
+//  count - number of headers to request
+//  skip - number of headers to skip
+// and also returns 'max', the last block which is expected to be returned by the remote peers,
+// given the (from,count,skip)
+func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) {
+	var (
+		from     int
+		count    int
+		MaxCount = MaxHeaderFetch / 16
+	)
+	// requestHead is the highest block that we will ask for. If requestHead is not offset,
+	// the highest block that we will get is 16 blocks back from head, which means we
+	// will fetch 14 or 15 blocks unnecessarily in the case the height difference
+	// between us and the peer is 1-2 blocks, which is most common
+	requestHead := int(remoteHeight) - 1
+	if requestHead < 0 {
+		requestHead = 0
+	}
+	// requestBottom is the lowest block we want included in the query
+	// Ideally, we want to include just below own head
+	requestBottom := int(localHeight - 1)
+	if requestBottom < 0 {
+		requestBottom = 0
+	}
+	totalSpan := requestHead - requestBottom
+	span := 1 + totalSpan/MaxCount
+	if span < 2 {
+		span = 2
+	}
+	if span > 16 {
+		span = 16
+	}
+
+	count = 1 + totalSpan/span
+	if count > MaxCount {
+		count = MaxCount
+	}
+	if count < 2 {
+		count = 2
+	}
+	from = requestHead - (count-1)*span
+	if from < 0 {
+		from = 0
+	}
+	max := from + (count-1)*span
+	return int64(from), count, span - 1, uint64(max)
+}
+
 // findAncestor tries to locate the common ancestor link of the local chain and
 // a remote peers blockchain. In the general case when our node was in sync and
 // on the correct chain, checking the top N links should already get us a match.
 // In the rare scenario when we ended up on a long reorganisation (i.e. none of
 // the head links match), we do a binary search to find the common ancestor.
-func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) {
+func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) {
 	// Figure out the valid ancestor range to prevent rewrite attacks
-	floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
+	var (
+		floor        = int64(-1)
+		localHeight  uint64
+		remoteHeight = remoteHeader.Number.Uint64()
+	)
+	switch d.mode {
+	case FullSync:
+		localHeight = d.blockchain.CurrentBlock().NumberU64()
+	case FastSync:
+		localHeight = d.blockchain.CurrentFastBlock().NumberU64()
+	default:
+		localHeight = d.lightchain.CurrentHeader().Number.Uint64()
+	}
+	p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight)
+	if localHeight >= MaxForkAncestry {
+		floor = int64(localHeight - MaxForkAncestry)
+	}
+	from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight)
 
-	if d.mode == FullSync {
+	p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip)
-		ceil = d.blockchain.CurrentBlock().NumberU64()
+	go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false)
-	} else if d.mode == FastSync {
-		ceil = d.blockchain.CurrentFastBlock().NumberU64()
-	}
-	if ceil >= MaxForkAncestry {
-		floor = int64(ceil - MaxForkAncestry)
-	}
-	p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
-
-	// Request the topmost blocks to short circuit binary ancestor lookup
-	head := ceil
-	if head > height {
-		head = height
-	}
-	from := int64(head) - int64(MaxHeaderFetch)
-	if from < 0 {
-		from = 0
-	}
-	// Span out with 15 block gaps into the future to catch bad head reports
-	limit := 2 * MaxHeaderFetch / 16
-	count := 1 + int((int64(ceil)-from)/16)
-	if count > limit {
-		count = limit
-	}
-	go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false)
 
 	// Wait for the remote response to the head fetch
 	number, hash := uint64(0), common.Hash{}
@@ -647,9 +695,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				return 0, errEmptyHeaderSet
 			}
 			// Make sure the peer's reply conforms to the request
-			for i := 0; i < len(headers); i++ {
+			for i, header := range headers {
-				if number := headers[i].Number.Int64(); number != from+int64(i)*16 {
+				expectNumber := from + int64(i)*int64((skip+1))
-					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number)
+				if number := header.Number.Int64(); number != expectNumber {
+					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
 					return 0, errInvalidChain
 				}
 			}
@@ -657,20 +706,24 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 			finished = true
 			for i := len(headers) - 1; i >= 0; i-- {
 				// Skip any headers that underflow/overflow our requested set
-				if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil {
+				if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max {
 					continue
 				}
 				// Otherwise check if we already know the header or not
 				h := headers[i].Hash()
 				n := headers[i].Number.Uint64()
-				if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) {
-					number, hash = n, h
 
-					// If every header is known, even future ones, the peer straight out lied about its head
+				var known bool
-					if number > height && i == limit-1 {
+				switch d.mode {
-						p.log.Warn("Lied about chain head", "reported", height, "found", number)
+				case FullSync:
-						return 0, errStallingPeer
+					known = d.blockchain.HasBlock(h, n)
+				case FastSync:
+					known = d.blockchain.HasFastBlock(h, n)
+				default:
+					known = d.lightchain.HasHeader(h, n)
 				}
+				if known {
+					number, hash = n, h
 					break
 				}
 			}
@@ -694,10 +747,12 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 		return number, nil
 	}
 	// Ancestor not found, we need to binary search over our chain
-	start, end := uint64(0), head
+	start, end := uint64(0), remoteHeight
 	if floor > 0 {
 		start = uint64(floor)
 	}
+	p.log.Trace("Binary searching for common ancestor", "start", start, "end", end)
+
 	for start+1 < end {
 		// Split our chain interval in two, and request the hash to cross check
 		check := (start + end) / 2
@@ -730,7 +785,17 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				// Modify the search interval based on the response
 				h := headers[0].Hash()
 				n := headers[0].Number.Uint64()
-				if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) {
+
+				var known bool
+				switch d.mode {
+				case FullSync:
+					known = d.blockchain.HasBlock(h, n)
+				case FastSync:
+					known = d.blockchain.HasFastBlock(h, n)
+				default:
+					known = d.lightchain.HasHeader(h, n)
+				}
+				if !known {
 					end = check
 					break
 				}
@@ -740,6 +805,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 					return 0, errBadPeer
 				}
 				start = check
+				hash = h
 
 			case <-timeout:
 				p.log.Debug("Waiting for search header timed out", "elapsed", ttl)

eth/downloader/queue.go

@@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest,
 			}
 			// Add the peer to the expiry report along the number of failed requests
 			expiries[id] = len(request.Headers)
-		}
+
-	}
+			// Remove the expired requests from the pending pool directly
-	// Remove the expired requests from the pending pool
-	for id := range expiries {
 			delete(pendPool, id)
 		}
+	}
 	return expiries
 }
 

eth/downloader/statesync.go

@@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) {
 				s.d.dropPeer(req.peer.id)
 			}
 			// Process all the received blobs and check for stale delivery
-			if err = s.process(req); err != nil {
+			delivered, err := s.process(req)
+			if err != nil {
 				log.Warn("Node data write error", "err", err)
 				return err
 			}
-			req.peer.SetNodeDataIdle(len(req.response))
+			req.peer.SetNodeDataIdle(delivered)
 		}
 	}
 	return nil
@@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) {
 // process iterates over a batch of delivered state data, injecting each item
 // into a running state sync, re-queuing any items that were requested but not
 // delivered.
-func (s *stateSync) process(req *stateReq) error {
+// Returns whether the peer actually managed to deliver anything of value,
+// and any error that occurred
+func (s *stateSync) process(req *stateReq) (int, error) {
 	// Collect processing stats and update progress if valid data was received
-	duplicate, unexpected := 0, 0
+	duplicate, unexpected, successful := 0, 0, 0
 
 	defer func(start time.Time) {
 		if duplicate > 0 || unexpected > 0 {
@@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error {
 
 	// Iterate over all the delivered data and inject one-by-one into the trie
 	progress := false
-
 	for _, blob := range req.response {
 		prog, hash, err := s.processNodeData(blob)
 		switch err {
@@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error {
 			s.numUncommitted++
 			s.bytesUncommitted += len(blob)
 			progress = progress || prog
+			successful++
 		case trie.ErrNotRequested:
 			unexpected++
 		case trie.ErrAlreadyProcessed:
 			duplicate++
 		default:
-			return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
+			return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
 		}
 		if _, ok := req.tasks[hash]; ok {
 			delete(req.tasks, hash)
@@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error {
 		// If we've requested the node too many times already, it may be a malicious
 		// sync where nobody has the right data. Abort.
 		if len(task.attempts) >= npeers {
-			return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
+			return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
 		}
 		// Missing item, place into the retry queue.
 		s.tasks[hash] = task
 	}
-	return nil
+	return successful, nil
 }
 
 // processNodeData tries to inject a trie node data blob delivered from a remote

eth/downloader/testchain_test.go

@@ -0,0 +1,221 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package downloader
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// Test chain parameters.
+var (
+	testKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
+	testDB      = ethdb.NewMemDatabase()
+	testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000))
+)
+
+// The common prefix of all test chains:
+var testChainBase = newTestChain(blockCacheItems+200, testGenesis)
+
+// Different forks on top of the base chain:
+var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain
+
+func init() {
+	var forkLen = int(MaxForkAncestry + 50)
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }()
+	go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }()
+	go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }()
+	wg.Wait()
+}
+
+type testChain struct {
+	genesis  *types.Block
+	chain    []common.Hash
+	headerm  map[common.Hash]*types.Header
+	blockm   map[common.Hash]*types.Block
+	receiptm map[common.Hash][]*types.Receipt
+	tdm      map[common.Hash]*big.Int
+}
+
+// newTestChain creates a blockchain of the given length.
+func newTestChain(length int, genesis *types.Block) *testChain {
+	tc := new(testChain).copy(length)
+	tc.genesis = genesis
+	tc.chain = append(tc.chain, genesis.Hash())
+	tc.headerm[tc.genesis.Hash()] = tc.genesis.Header()
+	tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty()
+	tc.blockm[tc.genesis.Hash()] = tc.genesis
+	tc.generate(length-1, 0, genesis, false)
+	return tc
+}
+
+// makeFork creates a fork on top of the test chain.
+func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain {
+	fork := tc.copy(tc.len() + length)
+	fork.generate(length, seed, tc.headBlock(), heavy)
+	return fork
+}
+
+// shorten creates a copy of the chain with the given length. It panics if the
+// length is longer than the number of available blocks.
+func (tc *testChain) shorten(length int) *testChain {
+	if length > tc.len() {
+		panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len()))
+	}
+	return tc.copy(length)
+}
+
+func (tc *testChain) copy(newlen int) *testChain {
+	cpy := &testChain{
+		genesis:  tc.genesis,
+		headerm:  make(map[common.Hash]*types.Header, newlen),
+		blockm:   make(map[common.Hash]*types.Block, newlen),
+		receiptm: make(map[common.Hash][]*types.Receipt, newlen),
+		tdm:      make(map[common.Hash]*big.Int, newlen),
+	}
+	for i := 0; i < len(tc.chain) && i < newlen; i++ {
+		hash := tc.chain[i]
+		cpy.chain = append(cpy.chain, tc.chain[i])
+		cpy.tdm[hash] = tc.tdm[hash]
+		cpy.blockm[hash] = tc.blockm[hash]
+		cpy.headerm[hash] = tc.headerm[hash]
+		cpy.receiptm[hash] = tc.receiptm[hash]
+	}
+	return cpy
+}
+
+// generate creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent. In addition, every 22th block
+// contains a transaction and every 5th an uncle to allow testing correct block
+// reassembly.
+func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) {
+	// start := time.Now()
+	// defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }()
+
+	blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) {
+		block.SetCoinbase(common.Address{seed})
+		// If a heavy chain is requested, delay blocks to raise difficulty
+		if heavy {
+			block.OffsetTime(-1)
+		}
+		// Include transactions to the miner to make blocks more interesting.
+		if parent == tc.genesis && i%22 == 0 {
+			signer := types.MakeSigner(params.TestChainConfig, block.Number())
+			tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey)
+			if err != nil {
+				panic(err)
+			}
+			block.AddTx(tx)
+		}
+		// if the block number is a multiple of 5, add a bonus uncle to the block
+		if i > 0 && i%5 == 0 {
+			block.AddUncle(&types.Header{
+				ParentHash: block.PrevBlock(i - 1).Hash(),
+				Number:     big.NewInt(block.Number().Int64() - 1),
+			})
+		}
+	})
+
+	// Convert the block-chain into a hash-chain and header/block maps
+	td := new(big.Int).Set(tc.td(parent.Hash()))
+	for i, b := range blocks {
+		td := td.Add(td, b.Difficulty())
+		hash := b.Hash()
+		tc.chain = append(tc.chain, hash)
+		tc.blockm[hash] = b
+		tc.headerm[hash] = b.Header()
+		tc.receiptm[hash] = receipts[i]
+		tc.tdm[hash] = new(big.Int).Set(td)
+	}
+}
+
+// len returns the total number of blocks in the chain.
+func (tc *testChain) len() int {
+	return len(tc.chain)
+}
+
+// headBlock returns the head of the chain.
+func (tc *testChain) headBlock() *types.Block {
+	return tc.blockm[tc.chain[len(tc.chain)-1]]
+}
+
+// td returns the total difficulty of the given block.
+func (tc *testChain) td(hash common.Hash) *big.Int {
+	return tc.tdm[hash]
+}
+
+// headersByHash returns headers in ascending order from the given hash.
+func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header {
+	num, _ := tc.hashToNumber(origin)
+	return tc.headersByNumber(num, amount, skip)
+}
+
+// headersByNumber returns headers in ascending order from the given number.
+func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header {
+	result := make([]*types.Header, 0, amount)
+	for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 {
+		if header, ok := tc.headerm[tc.chain[int(num)]]; ok {
+			result = append(result, header)
+		}
+	}
+	return result
+}
+
+// receipts returns the receipts of the given block hashes.
+func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt {
+	results := make([][]*types.Receipt, 0, len(hashes))
+	for _, hash := range hashes {
+		if receipt, ok := tc.receiptm[hash]; ok {
+			results = append(results, receipt)
+		}
+	}
+	return results
+}
+
+// bodies returns the block bodies of the given block hashes.
+func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) {
+	transactions := make([][]*types.Transaction, 0, len(hashes))
+	uncles := make([][]*types.Header, 0, len(hashes))
+	for _, hash := range hashes {
+		if block, ok := tc.blockm[hash]; ok {
+			transactions = append(transactions, block.Transactions())
+			uncles = append(uncles, block.Uncles())
+		}
+	}
+	return transactions, uncles
+}
+
+func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) {
+	for num, hash := range tc.chain {
+		if hash == target {
+			return uint64(num), true
+		}
+	}
+	return 0, false
+}

eth/gen_config.go

@@ -28,7 +28,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		SkipBcVersionCheck      bool `toml:"-"`
 		DatabaseHandles         int  `toml:"-"`
 		DatabaseCache           int
-		TrieCache               int
+		TrieCleanCache          int
+		TrieDirtyCache          int
 		TrieTimeout             time.Duration
 		Etherbase               common.Address `toml:",omitempty"`
 		MinerNotify             []string       `toml:",omitempty"`
@@ -43,6 +44,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		GPO                     gasprice.Config
 		EnablePreimageRecording bool
 		DocRoot                 string `toml:"-"`
+		EWASMInterpreter        string
+		EVMInterpreter          string
 	}
 	var enc Config
 	enc.Genesis = c.Genesis
@@ -54,7 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.SkipBcVersionCheck = c.SkipBcVersionCheck
 	enc.DatabaseHandles = c.DatabaseHandles
 	enc.DatabaseCache = c.DatabaseCache
-	enc.TrieCache = c.TrieCache
+	enc.TrieCleanCache = c.TrieCleanCache
+	enc.TrieDirtyCache = c.TrieDirtyCache
 	enc.TrieTimeout = c.TrieTimeout
 	enc.Etherbase = c.Etherbase
 	enc.MinerNotify = c.MinerNotify
@@ -69,6 +73,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.GPO = c.GPO
 	enc.EnablePreimageRecording = c.EnablePreimageRecording
 	enc.DocRoot = c.DocRoot
+	enc.EWASMInterpreter = c.EWASMInterpreter
+	enc.EVMInterpreter = c.EVMInterpreter
 	return &enc, nil
 }
 
@@ -84,7 +90,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		SkipBcVersionCheck      *bool `toml:"-"`
 		DatabaseHandles         *int  `toml:"-"`
 		DatabaseCache           *int
-		TrieCache               *int
+		TrieCleanCache          *int
+		TrieDirtyCache          *int
 		TrieTimeout             *time.Duration
 		Etherbase               *common.Address `toml:",omitempty"`
 		MinerNotify             []string        `toml:",omitempty"`
@@ -99,6 +106,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		GPO                     *gasprice.Config
 		EnablePreimageRecording *bool
 		DocRoot                 *string `toml:"-"`
+		EWASMInterpreter        *string
+		EVMInterpreter          *string
 	}
 	var dec Config
 	if err := unmarshal(&dec); err != nil {
@@ -131,8 +140,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.DatabaseCache != nil {
 		c.DatabaseCache = *dec.DatabaseCache
 	}
-	if dec.TrieCache != nil {
+	if dec.TrieCleanCache != nil {
-		c.TrieCache = *dec.TrieCache
+		c.TrieCleanCache = *dec.TrieCleanCache
+	}
+	if dec.TrieDirtyCache != nil {
+		c.TrieDirtyCache = *dec.TrieDirtyCache
 	}
 	if dec.TrieTimeout != nil {
 		c.TrieTimeout = *dec.TrieTimeout
@@ -176,5 +188,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.DocRoot != nil {
 		c.DocRoot = *dec.DocRoot
 	}
+	if dec.EWASMInterpreter != nil {
+		c.EWASMInterpreter = *dec.EWASMInterpreter
+	}
+	if dec.EVMInterpreter != nil {
+		c.EVMInterpreter = *dec.EVMInterpreter
+	}
 	return nil
 }

eth/handler.go

@@ -653,12 +653,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			trueHead = request.Block.ParentHash()
 			trueTD   = new(big.Int).Sub(request.TD, request.Block.Difficulty())
 		)
-		// Update the peers total difficulty if better than the previous
+		// Update the peer's total difficulty if better than the previous
 		if _, td := p.Head(); trueTD.Cmp(td) > 0 {
 			p.SetHead(trueHead, trueTD)
 
 			// Schedule a sync if above ours. Note, this will not fire a sync for a gap of
-			// a singe block (as the true TD is below the propagated block), however this
+			// a single block (as the true TD is below the propagated block), however this
 			// scenario should easily be covered by the fetcher.
 			currentBlock := pm.blockchain.CurrentBlock()
 			if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 {

eth/handler_test.go

@@ -585,7 +585,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
 			}
 		}(peer)
 	}
-	timeoutCh := time.NewTimer(time.Millisecond * 100).C
+	timeout := time.After(300 * time.Millisecond)
 	var receivedCount int
 outer:
 	for {
@@ -597,7 +597,7 @@ outer:
 			if receivedCount == totalPeers {
 				break outer
 			}
-		case <-timeoutCh:
+		case <-timeout:
 			break outer
 		}
 	}

eth/tracers/tracer_test.go

@@ -48,7 +48,7 @@ type dummyStatedb struct {
 	state.StateDB
 }
 
-func (dummyStatedb) GetRefund() uint64 { return 1337 }
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
 
 func runTrace(tracer *Tracer) (json.RawMessage, error) {
 	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})

ethclient/ethclient.go

@@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb
 // FilterLogs executes a filter query.
 func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
 	var result []types.Log
-	err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q))
+	arg, err := toFilterArg(q)
+	if err != nil {
+		return nil, err
+	}
+	err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg)
 	return result, err
 }
 
 // SubscribeFilterLogs subscribes to the results of a streaming filter query.
 func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
-	return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q))
+	arg, err := toFilterArg(q)
+	if err != nil {
+		return nil, err
+	}
+	return ec.c.EthSubscribe(ctx, ch, "logs", arg)
 }
 
-func toFilterArg(q ethereum.FilterQuery) interface{} {
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
 	arg := map[string]interface{}{
-		"fromBlock": toBlockNumArg(q.FromBlock),
-		"toBlock":   toBlockNumArg(q.ToBlock),
 		"address": q.Addresses,
 		"topics":  q.Topics,
 	}
+	if q.BlockHash != nil {
+		arg["blockHash"] = *q.BlockHash
+		if q.FromBlock != nil || q.ToBlock != nil {
+			return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+		}
+	} else {
 		if q.FromBlock == nil {
 			arg["fromBlock"] = "0x0"
+		} else {
+			arg["fromBlock"] = toBlockNumArg(q.FromBlock)
 		}
-	return arg
+		arg["toBlock"] = toBlockNumArg(q.ToBlock)
+	}
+	return arg, nil
 }
 
 // Pending State

ethclient/ethclient_test.go

@@ -16,7 +16,15 @@
 
 package ethclient
 
-import "github.com/ethereum/go-ethereum"
+import (
+	"fmt"
+	"math/big"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+)
 
 // Verify that Client implements the ethereum interfaces.
 var (
@@ -32,3 +40,113 @@ var (
 	// _ = ethereum.PendingStateEventer(&Client{})
 	_ = ethereum.PendingContractCaller(&Client{})
 )
+
+func TestToFilterArg(t *testing.T) {
+	blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+	addresses := []common.Address{
+		common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"),
+	}
+	blockHash := common.HexToHash(
+		"0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb",
+	)
+
+	for _, testCase := range []struct {
+		name   string
+		input  ethereum.FilterQuery
+		output interface{}
+		err    error
+	}{
+		{
+			"without BlockHash",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				FromBlock: big.NewInt(1),
+				ToBlock:   big.NewInt(2),
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"fromBlock": "0x1",
+				"toBlock":   "0x2",
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with nil fromBlock and nil toBlock",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"fromBlock": "0x0",
+				"toBlock":   "latest",
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with blockhash",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"blockHash": blockHash,
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with blockhash and from block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				FromBlock: big.NewInt(1),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+		{
+			"with blockhash and to block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				ToBlock:   big.NewInt(1),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+		{
+			"with blockhash and both from / to block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				FromBlock: big.NewInt(1),
+				ToBlock:   big.NewInt(2),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			output, err := toFilterArg(testCase.input)
+			if (testCase.err == nil) != (err == nil) {
+				t.Fatalf("expected error %v but got %v", testCase.err, err)
+			}
+			if testCase.err != nil {
+				if testCase.err.Error() != err.Error() {
+					t.Fatalf("expected error %v but got %v", testCase.err, err)
+				}
+			} else if !reflect.DeepEqual(testCase.output, output) {
+				t.Fatalf("expected filter arg %v but got %v", testCase.output, output)
+			}
+		})
+	}
+}

event/event_test.go

@@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) {
 	}
 }
 
-func emptySubscriber(mux *TypeMux, types ...interface{}) {
+func emptySubscriber(mux *TypeMux) {
 	s := mux.Subscribe(testEvent(0))
 	go func() {
 		for range s.Chan() {
@@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) {
 func BenchmarkPostConcurrent(b *testing.B) {
 	var mux = new(TypeMux)
 	defer mux.Stop()
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
 
 	var wg sync.WaitGroup
 	poster := func() {

event/filter/filter.go

@@ -1,95 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package filter implements event filters.
-package filter
-
-import "reflect"
-
-type Filter interface {
-	Compare(Filter) bool
-	Trigger(data interface{})
-}
-
-type FilterEvent struct {
-	filter Filter
-	data   interface{}
-}
-
-type Filters struct {
-	id       int
-	watchers map[int]Filter
-	ch       chan FilterEvent
-
-	quit chan struct{}
-}
-
-func New() *Filters {
-	return &Filters{
-		ch:       make(chan FilterEvent),
-		watchers: make(map[int]Filter),
-		quit:     make(chan struct{}),
-	}
-}
-
-func (f *Filters) Start() {
-	go f.loop()
-}
-
-func (f *Filters) Stop() {
-	close(f.quit)
-}
-
-func (f *Filters) Notify(filter Filter, data interface{}) {
-	f.ch <- FilterEvent{filter, data}
-}
-
-func (f *Filters) Install(watcher Filter) int {
-	f.watchers[f.id] = watcher
-	f.id++
-
-	return f.id - 1
-}
-
-func (f *Filters) Uninstall(id int) {
-	delete(f.watchers, id)
-}
-
-func (f *Filters) loop() {
-out:
-	for {
-		select {
-		case <-f.quit:
-			break out
-		case event := <-f.ch:
-			for _, watcher := range f.watchers {
-				if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) {
-					if watcher.Compare(event.filter) {
-						watcher.Trigger(event.data)
-					}
-				}
-			}
-		}
-	}
-}
-
-func (f *Filters) Match(a, b Filter) bool {
-	return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b)
-}
-
-func (f *Filters) Get(i int) Filter {
-	return f.watchers[i]
-}

event/filter/filter_test.go

@@ -1,60 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package filter
-
-import (
-	"testing"
-	"time"
-)
-
-// Simple test to check if baseline matching/mismatching filtering works.
-func TestFilters(t *testing.T) {
-	fm := New()
-	fm.Start()
-
-	// Register two filters to catch posted data
-	first := make(chan struct{})
-	fm.Install(Generic{
-		Str1: "hello",
-		Fn: func(data interface{}) {
-			first <- struct{}{}
-		},
-	})
-	second := make(chan struct{})
-	fm.Install(Generic{
-		Str1: "hello1",
-		Str2: "hello",
-		Fn: func(data interface{}) {
-			second <- struct{}{}
-		},
-	})
-	// Post an event that should only match the first filter
-	fm.Notify(Generic{Str1: "hello"}, true)
-	fm.Stop()
-
-	// Ensure only the mathcing filters fire
-	select {
-	case <-first:
-	case <-time.After(100 * time.Millisecond):
-		t.Error("matching filter timed out")
-	}
-	select {
-	case <-second:
-		t.Error("mismatching filter fired")
-	case <-time.After(100 * time.Millisecond):
-	}
-}

event/filter/generic_filter.go

@@ -1,48 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package filter
-
-type Generic struct {
-	Str1, Str2, Str3 string
-	Data             map[string]struct{}
-
-	Fn func(data interface{})
-}
-
-// self = registered, f = incoming
-func (self Generic) Compare(f Filter) bool {
-	var strMatch, dataMatch = true, true
-
-	filter := f.(Generic)
-	if (len(self.Str1) > 0 && filter.Str1 != self.Str1) ||
-		(len(self.Str2) > 0 && filter.Str2 != self.Str2) ||
-		(len(self.Str3) > 0 && filter.Str3 != self.Str3) {
-		strMatch = false
-	}
-
-	for k := range self.Data {
-		if _, ok := filter.Data[k]; !ok {
-			return false
-		}
-	}
-
-	return strMatch && dataMatch
-}
-
-func (self Generic) Trigger(data interface{}) {
-	self.Fn(data)
-}

internal/cmdtest/test_cmd.go

@@ -27,6 +27,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"text/template"
 	"time"
@@ -50,6 +51,8 @@ type TestCmd struct {
 	stdout *bufio.Reader
 	stdin  io.WriteCloser
 	stderr *testlogger
+	// Err will contain the process exit error or interrupt signal error
+	Err error
 }
 
 // Run exec's the current binary using name as argv[0] which will trigger the
@@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() {
 }
 
 func (tt *TestCmd) WaitExit() {
-	tt.cmd.Wait()
+	tt.Err = tt.cmd.Wait()
 }
 
 func (tt *TestCmd) Interrupt() {
-	tt.cmd.Process.Signal(os.Interrupt)
+	tt.Err = tt.cmd.Process.Signal(os.Interrupt)
+}
+
+// ExitStatus exposes the process' OS exit code
+// It will only return a valid value after the process has finished.
+func (tt *TestCmd) ExitStatus() int {
+	if tt.Err != nil {
+		exitErr := tt.Err.(*exec.ExitError)
+		if exitErr != nil {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+				return status.ExitStatus()
+			}
+		}
+	}
+	return 0
 }
 
 // StderrText returns any stderr output written so far.

internal/ethapi/api.go

@@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
 	return fetchKeystore(s.am).Lock(addr) == nil
 }
 
-// signTransactions sets defaults and signs the given transaction
+// signTransaction sets defaults and signs the given transaction
 // NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
 // and release it after the transaction has been submitted to the tx pool
 func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) {

internal/web3ext/web3ext.go

@@ -481,6 +481,12 @@ web3._extend({
 			params: 2,
 			inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex]
 		}),
+		new web3._extend.Method({
+			name: 'getProof',
+			call: 'eth_getProof',
+			params: 3,
+			inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter]
+		}),
 	],
 	properties: [
 		new web3._extend.Property({

les/fetcher.go

@@ -143,21 +143,21 @@ func (f *lightFetcher) syncLoop() {
 			var (
 				rq      *distReq
 				reqID   uint64
+				syncing bool
 			)
 			if !f.syncing && !(newAnnounce && s) {
-				rq, reqID = f.nextRequest()
+				rq, reqID, syncing = f.nextRequest()
 			}
-			syncing := f.syncing
 			f.lock.Unlock()
 
 			if rq != nil {
 				requesting = true
-				_, ok := <-f.pm.reqDist.queue(rq)
+				if _, ok := <-f.pm.reqDist.queue(rq); ok {
-				if !ok {
+					if syncing {
-					f.requestChn <- false
+						f.lock.Lock()
-				}
+						f.syncing = true
-
+						f.lock.Unlock()
-				if !syncing {
+					} else {
 						go func() {
 							time.Sleep(softRequestTimeout)
 							f.reqMu.Lock()
@@ -171,6 +171,9 @@ func (f *lightFetcher) syncLoop() {
 							f.requestChn <- false
 						}()
 					}
+				} else {
+					f.requestChn <- false
+				}
 			}
 		case reqID := <-f.timeoutChn:
 			f.reqMu.Lock()
@@ -209,6 +212,7 @@ func (f *lightFetcher) syncLoop() {
 			f.checkSyncedHeaders(p)
 			f.syncing = false
 			f.lock.Unlock()
+			f.requestChn <- false
 		}
 	}
 }
@@ -405,7 +409,7 @@ func (f *lightFetcher) requestedID(reqID uint64) bool {
 
 // nextRequest selects the peer and announced head to be requested next, amount
 // to be downloaded starting from the head backwards is also returned
-func (f *lightFetcher) nextRequest() (*distReq, uint64) {
+func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) {
 	var (
 		bestHash   common.Hash
 		bestAmount uint64
@@ -427,14 +431,12 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
 		}
 	}
 	if bestTd == f.maxConfirmedTd {
-		return nil, 0
+		return nil, 0, false
 	}
 
-	f.syncing = bestSyncing
-
 	var rq *distReq
 	reqID := genReqID()
-	if f.syncing {
+	if bestSyncing {
 		rq = &distReq{
 			getCost: func(dp distPeer) uint64 {
 				return 0
@@ -500,7 +502,7 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
 			},
 		}
 	}
-	return rq, reqID
+	return rq, reqID, bestSyncing
 }
 
 // deliverHeaders delivers header download request responses for processing

les/serverpool.go

@@ -683,7 +683,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
 }
 
 func encodePubkey64(pub *ecdsa.PublicKey) []byte {
-	return crypto.FromECDSAPub(pub)[:1]
+	return crypto.FromECDSAPub(pub)[1:]
 }
 
 func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {

light/postprocess.go

@@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co
 		diskdb:      db,
 		odr:         odr,
 		trieTable:   trieTable,
-		triedb:      trie.NewDatabase(trieTable),
+		triedb:      trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
 		sectionSize: size,
 	}
 	return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht")
@@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
 		diskdb:     db,
 		odr:        odr,
 		trieTable:  trieTable,
-		triedb:     trie.NewDatabase(trieTable),
+		triedb:     trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
 		parentSize: parentSize,
 		size:       size,
 	}

light/trie.go

@@ -108,7 +108,7 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) {
 func (t *odrTrie) TryUpdate(key, value []byte) error {
 	key = crypto.Keccak256(key)
 	return t.do(key, func() error {
-		return t.trie.TryDelete(key)
+		return t.trie.TryUpdate(key, value)
 	})
 }
 

metrics/registry.go

@@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() {
 	r.underlying.UnregisterAll()
 }
 
-var DefaultRegistry Registry = NewRegistry()
+var (
+	DefaultRegistry   = NewRegistry()
+	EphemeralRegistry = NewRegistry()
+)
 
 // Call the given function for each registered metric.
 func Each(f func(string, interface{})) {

miner/stress_clique.go

@@ -22,7 +22,6 @@ package main
 import (
 	"bytes"
 	"crypto/ecdsa"
-	"fmt"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -40,7 +39,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -62,11 +61,11 @@ func main() {
 
 	var (
 		nodes  []*node.Node
-		enodes []string
+		enodes []*enode.Node
 	)
 	for _, sealer := range sealers {
 		// Start the node and wait until it's up
-		node, err := makeSealer(genesis, enodes)
+		node, err := makeSealer(genesis)
 		if err != nil {
 			panic(err)
 		}
@@ -76,18 +75,12 @@ func main() {
 			time.Sleep(250 * time.Millisecond)
 		}
 		// Connect the node to al the previous ones
-		for _, enode := range enodes {
+		for _, n := range enodes {
-			enode, err := discover.ParseNode(enode)
+			node.Server().AddPeer(n)
-			if err != nil {
-				panic(err)
 		}
-			node.Server().AddPeer(enode)
+		// Start tracking the node and it's enode
-		}
-		// Start tracking the node and it's enode url
 		nodes = append(nodes, node)
-
+		enodes = append(enodes, node.Server().Self())
-		enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
-		enodes = append(enodes, enode)
 
 		// Inject the signer key and start sealing with it
 		store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core
 	return genesis
 }
 
-func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeSealer(genesis *core.Genesis) (*node.Node, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := ioutil.TempDir("", "")
 

miner/stress_ethash.go

@@ -21,7 +21,6 @@ package main
 
 import (
 	"crypto/ecdsa"
-	"fmt"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -41,7 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -62,11 +61,11 @@ func main() {
 
 	var (
 		nodes  []*node.Node
-		enodes []string
+		enodes []*enode.Node
 	)
 	for i := 0; i < 4; i++ {
 		// Start the node and wait until it's up
-		node, err := makeMiner(genesis, enodes)
+		node, err := makeMiner(genesis)
 		if err != nil {
 			panic(err)
 		}
@@ -76,18 +75,12 @@ func main() {
 			time.Sleep(250 * time.Millisecond)
 		}
 		// Connect the node to al the previous ones
-		for _, enode := range enodes {
+		for _, n := range enodes {
-			enode, err := discover.ParseNode(enode)
+			node.Server().AddPeer(n)
-			if err != nil {
-				panic(err)
 		}
-			node.Server().AddPeer(enode)
+		// Start tracking the node and it's enode
-		}
-		// Start tracking the node and it's enode url
 		nodes = append(nodes, node)
-
+		enodes = append(enodes, node.Server().Self())
-		enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
-		enodes = append(enodes, enode)
 
 		// Inject the signer key and start sealing with it
 		store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis {
 	return genesis
 }
 
-func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeMiner(genesis *core.Genesis) (*node.Node, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := ioutil.TempDir("", "")
 

node/node.go

@@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error {
 		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 			return err
 		}
-		n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace)
+		n.log.Debug("InProc registered", "namespace", api.Namespace)
 	}
 	n.inprocHandler = handler
 	return nil

p2p/discover/table.go

@@ -434,7 +434,7 @@ func (tab *Table) loadSeedNodes() {
 	for i := range seeds {
 		seed := seeds[i]
 		age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID())) }}
-		log.Debug("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
+		log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
 		tab.add(seed)
 	}
 }

p2p/metrics.go

@@ -47,8 +47,8 @@ var (
 	egressConnectMeter  = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections
 	egressTrafficMeter  = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil)  // Meter metering the cumulative egress traffic
 
-	PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsInboundTraffic+"/")  // Registry containing the peer ingress
+	PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/")  // Registry containing the peer ingress
-	PeerEgressRegistry  = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress
+	PeerEgressRegistry  = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress
 
 	meteredPeerFeed  event.Feed // Event feed for peer metrics
 	meteredPeerCount int32      // Actually stored peer connection count
@@ -74,7 +74,7 @@ const (
 type MeteredPeerEvent struct {
 	Type    MeteredPeerEventType // Type of peer event
 	IP      net.IP               // IP address of the peer
-	ID      string               // NodeID of the peer
+	ID      enode.ID             // NodeID of the peer
 	Elapsed time.Duration        // Time elapsed between the connection and the handshake/disconnection
 	Ingress uint64               // Ingress count at the moment of the event
 	Egress  uint64               // Egress count at the moment of the event
@@ -93,7 +93,7 @@ type meteredConn struct {
 
 	connected time.Time // Connection time of the peer
 	ip        net.IP    // IP address of the peer
-	id        string    // NodeID of the peer
+	id        enode.ID  // NodeID of the peer
 
 	// trafficMetered denotes if the peer is registered in the traffic registries.
 	// Its value is true if the metered peer count doesn't reach the limit in the
@@ -160,8 +160,7 @@ func (c *meteredConn) Write(b []byte) (n int, err error) {
 // handshakeDone is called when a peer handshake is done. Registers the peer to
 // the ingress and the egress traffic registries using the peer's IP and node ID,
 // also emits connect event.
-func (c *meteredConn) handshakeDone(nodeID enode.ID) {
+func (c *meteredConn) handshakeDone(id enode.ID) {
-	id := nodeID.String()
 	if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit {
 		// Don't register the peer in the traffic registries.
 		atomic.AddInt32(&meteredPeerCount, -1)
@@ -170,7 +169,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) {
 		c.lock.Unlock()
 		log.Warn("Metered peer count reached the limit")
 	} else {
-		key := fmt.Sprintf("%s/%s", c.ip, id)
+		key := fmt.Sprintf("%s/%s", c.ip, id.String())
 		c.lock.Lock()
 		c.id, c.trafficMetered = id, true
 		c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry)
@@ -190,7 +189,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) {
 func (c *meteredConn) Close() error {
 	err := c.Conn.Close()
 	c.lock.RLock()
-	if c.id == "" {
+	if c.id == (enode.ID{}) {
 		// If the peer disconnects before the handshake.
 		c.lock.RUnlock()
 		meteredPeerFeed.Send(MeteredPeerEvent{

p2p/protocols/protocol_test.go

@@ -318,7 +318,7 @@ func TestProtocolHook(t *testing.T) {
 	<-testHook.waitC
 
 	time.Sleep(100 * time.Millisecond)
-	err = tester.TestDisconnected(&p2ptest.Disconnect{tester.Nodes[1].ID(), testHook.err})
+	err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: tester.Nodes[1].ID(), Error: testHook.err})
 	if err != nil {
 		t.Fatalf("Expected a specific disconnect error, but got different one: %v", err)
 	}

p2p/rlpx.go

@@ -151,7 +151,7 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake,
 	}
 	if msg.Code == discMsg {
 		// Disconnect before protocol handshake is valid according to the
-		// spec and we send it ourself if the posthanshake checks fail.
+		// spec and we send it ourself if the post-handshake checks fail.
 		// We can't return the reason directly, though, because it is echoed
 		// back otherwise. Wrap it in a string instead.
 		var reason [1]DiscReason

p2p/server.go

@@ -228,7 +228,7 @@ type transport interface {
 	MsgReadWriter
 	// transports must provide Close because we use MsgPipe in some of
 	// the tests. Closing the actual network connection doesn't do
-	// anything in those tests because NsgPipe doesn't use it.
+	// anything in those tests because MsgPipe doesn't use it.
 	close(err error)
 }
 

p2p/simulations/http_test.go

@@ -18,6 +18,7 @@ package simulations
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"math/rand"
 	"net/http/httptest"
@@ -28,13 +29,26 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
+	colorable "github.com/mattn/go-colorable"
 )
 
+var (
+	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
+)
+
+func init() {
+	flag.Parse()
+
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+}
+
 // testService implements the node.Service interface and provides protocols
 // and APIs which are useful for testing nodes in a simulation network
 type testService struct {
@@ -584,9 +598,26 @@ func TestHTTPNodeRPC(t *testing.T) {
 // TestHTTPSnapshot tests creating and loading network snapshots
 func TestHTTPSnapshot(t *testing.T) {
 	// start the server
-	_, s := testHTTPServer(t)
+	network, s := testHTTPServer(t)
 	defer s.Close()
 
+	var eventsDone = make(chan struct{})
+	count := 1
+	eventsDoneChan := make(chan *Event)
+	eventSub := network.Events().Subscribe(eventsDoneChan)
+	go func() {
+		defer eventSub.Unsubscribe()
+		for event := range eventsDoneChan {
+			if event.Type == EventTypeConn && !event.Control {
+				count--
+				if count == 0 {
+					eventsDone <- struct{}{}
+					return
+				}
+			}
+		}
+	}()
+
 	// create a two-node network
 	client := NewClient(s.URL)
 	nodeCount := 2
@@ -620,7 +651,7 @@ func TestHTTPSnapshot(t *testing.T) {
 		}
 		states[i] = state
 	}
-
+	<-eventsDone
 	// create a snapshot
 	snap, err := client.CreateSnapshot()
 	if err != nil {
@@ -634,9 +665,23 @@ func TestHTTPSnapshot(t *testing.T) {
 	}
 
 	// create another network
-	_, s = testHTTPServer(t)
+	network2, s := testHTTPServer(t)
 	defer s.Close()
 	client = NewClient(s.URL)
+	count = 1
+	eventSub = network2.Events().Subscribe(eventsDoneChan)
+	go func() {
+		defer eventSub.Unsubscribe()
+		for event := range eventsDoneChan {
+			if event.Type == EventTypeConn && !event.Control {
+				count--
+				if count == 0 {
+					eventsDone <- struct{}{}
+					return
+				}
+			}
+		}
+	}()
 
 	// subscribe to events so we can check them later
 	events := make(chan *Event, 100)
@@ -651,6 +696,7 @@ func TestHTTPSnapshot(t *testing.T) {
 	if err := client.LoadSnapshot(snap); err != nil {
 		t.Fatalf("error loading snapshot: %s", err)
 	}
+	<-eventsDone
 
 	// check the nodes and connection exists
 	net, err := client.GetNetwork()
@@ -676,6 +722,9 @@ func TestHTTPSnapshot(t *testing.T) {
 	if conn.Other.String() != nodes[1].ID {
 		t.Fatalf("expected connection to have other=%q, got other=%q", nodes[1].ID, conn.Other)
 	}
+	if !conn.Up {
+		t.Fatal("should be up")
+	}
 
 	// check the node states were restored
 	for i, node := range nodes {

p2p/simulations/network.go

@@ -644,11 +644,18 @@ type NodeSnapshot struct {
 
 // Snapshot creates a network snapshot
 func (net *Network) Snapshot() (*Snapshot, error) {
+	return net.snapshot(nil, nil)
+}
+
+func (net *Network) SnapshotWithServices(addServices []string, removeServices []string) (*Snapshot, error) {
+	return net.snapshot(addServices, removeServices)
+}
+
+func (net *Network) snapshot(addServices []string, removeServices []string) (*Snapshot, error) {
 	net.lock.Lock()
 	defer net.lock.Unlock()
 	snap := &Snapshot{
 		Nodes: make([]NodeSnapshot, len(net.Nodes)),
-		Conns: make([]Conn, len(net.Conns)),
 	}
 	for i, node := range net.Nodes {
 		snap.Nodes[i] = NodeSnapshot{Node: *node}
@@ -660,9 +667,40 @@ func (net *Network) Snapshot() (*Snapshot, error) {
 			return nil, err
 		}
 		snap.Nodes[i].Snapshots = snapshots
+		for _, addSvc := range addServices {
+			haveSvc := false
+			for _, svc := range snap.Nodes[i].Node.Config.Services {
+				if svc == addSvc {
+					haveSvc = true
+					break
+				}
+			}
+			if !haveSvc {
+				snap.Nodes[i].Node.Config.Services = append(snap.Nodes[i].Node.Config.Services, addSvc)
+			}
+		}
+		if len(removeServices) > 0 {
+			var cleanedServices []string
+			for _, svc := range snap.Nodes[i].Node.Config.Services {
+				haveSvc := false
+				for _, rmSvc := range removeServices {
+					if rmSvc == svc {
+						haveSvc = true
+						break
+					}
+				}
+				if !haveSvc {
+					cleanedServices = append(cleanedServices, svc)
+				}
+
+			}
+			snap.Nodes[i].Node.Config.Services = cleanedServices
+		}
+	}
+	for _, conn := range net.Conns {
+		if conn.Up {
+			snap.Conns = append(snap.Conns, *conn)
 		}
-	for i, conn := range net.Conns {
-		snap.Conns[i] = *conn
 	}
 	return snap, nil
 }

params/config.go

@@ -49,10 +49,10 @@ var (
 	// MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network.
 	MainnetTrustedCheckpoint = &TrustedCheckpoint{
 		Name:         "mainnet",
-		SectionIndex: 195,
+		SectionIndex: 203,
-		SectionHead:  common.HexToHash("0x1cdd2a84cf6c1261ffccc88f6bcefb513abd7934a96c1e909fbf74767560f16b"),
+		SectionHead:  common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"),
-		CHTRoot:      common.HexToHash("0xe453333c20391d16b91b6fe11c104704f62c8dba15f69db73b4cdf7e100105eb"),
+		CHTRoot:      common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"),
-		BloomRoot:    common.HexToHash("0x47f30069473072e00d2cdca146dce40f0aad243dfc8221bf810822c091674efe"),
+		BloomRoot:    common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"),
 	}
 
 	// TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network.
@@ -73,10 +73,10 @@ var (
 	// TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network.
 	TestnetTrustedCheckpoint = &TrustedCheckpoint{
 		Name:         "testnet",
-		SectionIndex: 126,
+		SectionIndex: 134,
-		SectionHead:  common.HexToHash("0x48f7dd4c9c60be04bf15fd4d0bcac46ddd8caf6b01d6fb8f8e1f7955cdd1337a"),
+		SectionHead:  common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"),
-		CHTRoot:      common.HexToHash("0x6e54cb80a1884881ea1a114243af9012c95e0296b47f103b5ab124313968508e"),
+		CHTRoot:      common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"),
-		BloomRoot:    common.HexToHash("0xb55accf6dce6455b47db8510d15eff38d0ed7378829f3036d26b48e7d15da3f6"),
+		BloomRoot:    common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"),
 	}
 
 	// RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network.
@@ -100,10 +100,10 @@ var (
 	// RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network.
 	RinkebyTrustedCheckpoint = &TrustedCheckpoint{
 		Name:         "rinkeby",
-		SectionIndex: 93,
+		SectionIndex: 100,
-		SectionHead:  common.HexToHash("0xdefb94aa217ab38f2919f7318d1d5476bd2aabf1ec9148047fe03e555615e0b4"),
+		SectionHead:  common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"),
-		CHTRoot:      common.HexToHash("0x52c98c2fe508a8332c27dc10538f3fead43306e2b22b597587763c2fe6586da6"),
+		CHTRoot:      common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"),
-		BloomRoot:    common.HexToHash("0x93d83be0c1b12f732b1a027ecdfb16f39b0d020b8c10bfb90e76f3b01adfc5b6"),
+		BloomRoot:    common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"),
 	}
 
 	// AllEthashProtocolChanges contains every protocol change (EIPs) introduced
@@ -111,16 +111,16 @@ var (
 	//
 	// This configuration is intentionally not using keyed fields to force anyone
 	// adding flags to the config to also have to set these fields.
-	AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
+	AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
 
 	// AllCliqueProtocolChanges contains every protocol change (EIPs) introduced
 	// and accepted by the Ethereum core developers into the Clique consensus.
 	//
 	// This configuration is intentionally not using keyed fields to force anyone
 	// adding flags to the config to also have to set these fields.
-	AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
+	AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
 
-	TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
+	TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
 	TestRules       = TestChainConfig.Rules(new(big.Int))
 )
 

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1          // Major version component of the current release
 	VersionMinor = 8          // Minor version component of the current release
-	VersionPatch = 18         // Patch version component of the current release
+	VersionPatch = 19         // Patch version component of the current release
 	VersionMeta  = "unstable" // Version metadata to append to the version string
 )
 

rpc/client_example_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-// In this example, our client whishes to track the latest 'block number'
+// In this example, our client wishes to track the latest 'block number'
 // known to the server. The server supports two methods:
 //
 // eth_getBlockByNumber("latest", {})

rpc/doc.go

@@ -32,7 +32,7 @@ An example method:
  func (s *CalcService) Add(a, b int) (int, error)
 
 When the returned error isn't nil the returned integer is ignored and the error is
-send back to the client. Otherwise the returned integer is send back to the client.
+sent back to the client. Otherwise the returned integer is sent back to the client.
 
 Optional arguments are supported by accepting pointer values as arguments. E.g.
 if we want to do the addition in an optional finite field we can accept a mod

signer/core/abihelper_test.go

@@ -38,13 +38,13 @@ func verify(t *testing.T, jsondata, calldata string, exp []interface{}) {
 	cd := common.Hex2Bytes(calldata)
 	sigdata, argdata := cd[:4], cd[4:]
 	method, err := abispec.MethodById(sigdata)
-
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	data, err := method.Inputs.UnpackValues(argdata)
-
+	if err != nil {
+		t.Fatal(err)
+	}
 	if len(data) != len(exp) {
 		t.Fatalf("Mismatched length, expected %d, got %d", len(exp), len(data))
 	}

signer/rules/rules_test.go

@@ -151,7 +151,7 @@ func TestListRequest(t *testing.T) {
 		t.Errorf("Couldn't create evaluator %v", err)
 		return
 	}
-	resp, err := r.ApproveListing(&core.ListRequest{
+	resp, _ := r.ApproveListing(&core.ListRequest{
 		Accounts: accs,
 		Meta:     core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"},
 	})
@@ -515,7 +515,7 @@ func TestLimitWindow(t *testing.T) {
 		r.OnApprovedTx(response)
 	}
 	// Fourth should fail
-	resp, err := r.ApproveTx(dummyTx(h))
+	resp, _ := r.ApproveTx(dummyTx(h))
 	if resp.Approved {
 		t.Errorf("Expected check to resolve to 'Reject'")
 	}
@@ -609,8 +609,8 @@ func TestContextIsCleared(t *testing.T) {
 		t.Fatalf("Failed to load bootstrap js: %v", err)
 	}
 	tx := dummyTxWithV(0)
-	r1, err := r.ApproveTx(tx)
+	r1, _ := r.ApproveTx(tx)
-	r2, err := r.ApproveTx(tx)
+	r2, _ := r.ApproveTx(tx)
 	if r1.Approved != r2.Approved {
 		t.Errorf("Expected execution context to be cleared between executions")
 	}

swarm/api/act.go

@@ -458,6 +458,9 @@ func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees
 			return nil, nil, nil, err
 		}
 		sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt)
+		if err != nil {
+			return nil, nil, nil, err
+		}
 
 		hasher := sha3.NewKeccak256()
 		hasher.Write(append(sessionKey, 0))

swarm/api/api.go

@@ -472,7 +472,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 		// no entry found
 		status = http.StatusNotFound
 		apiGetNotFound.Inc(1)
-		err = fmt.Errorf("manifest entry for '%s' not found", path)
+		err = fmt.Errorf("Not found: could not find resource '%s'", path)
 		log.Trace("manifest entry not found", "key", contentAddr, "path", path)
 	}
 	return

swarm/api/client/client_test.go

@@ -33,10 +33,9 @@ import (
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
-func serverFunc(api *api.API) testutil.TestServer {
+func serverFunc(api *api.API) swarmhttp.TestServer {
 	return swarmhttp.NewServer(api, "")
 }
 
@@ -49,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) {
 }
 
 func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	client := NewClient(srv.URL)
@@ -90,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
 }
 
 func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	client := NewClient(srv.URL)
@@ -188,7 +187,7 @@ func newTestDirectory(t *testing.T) string {
 // TestClientUploadDownloadDirectory tests uploading and downloading a
 // directory of files to a swarm manifest
 func TestClientUploadDownloadDirectory(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	dir := newTestDirectory(t)
@@ -254,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) {
 }
 
 func testClientFileList(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	dir := newTestDirectory(t)
@@ -312,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
 // TestClientMultipartUpload tests uploading files to swarm using a multipart
 // upload
 func TestClientMultipartUpload(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	// define an uploader which uploads testDirFiles with some data
@@ -378,7 +377,7 @@ func TestClientCreateFeedMultihash(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	client := NewClient(srv.URL)
 	defer srv.Close()
 
@@ -440,7 +439,7 @@ func TestClientCreateUpdateFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	client := NewClient(srv.URL)
 	defer srv.Close()
 
@@ -457,6 +456,9 @@ func TestClientCreateUpdateFeed(t *testing.T) {
 	}
 
 	feedManifestHash, err := client.CreateFeedWithManifest(createRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882"
 	if feedManifestHash != correctManifestAddrHex {

swarm/api/filesystem.go

@@ -122,6 +122,10 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error
 			var wait func(context.Context) error
 			ctx := context.TODO()
 			hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt)
+			if err != nil {
+				errors[i] = err
+				return
+			}
 			if hash != nil {
 				list[i].Hash = hash.Hex()
 			}

swarm/api/http/middleware.go

@@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		writer := newLoggingResponseWriter(w)
 		h.ServeHTTP(writer, r)
-		log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
+		log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
 	})
 }
 

swarm/api/http/response_test.go

@@ -24,12 +24,10 @@ import (
 	"testing"
 
 	"golang.org/x/net/html"
-
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestError(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -55,7 +53,7 @@ func TestError(t *testing.T) {
 }
 
 func Test404Page(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -81,7 +79,7 @@ func Test404Page(t *testing.T) {
 }
 
 func Test500Page(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -106,7 +104,7 @@ func Test500Page(t *testing.T) {
 	}
 }
 func Test500PageWith0xHashPrefix(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) {
 }
 
 func TestJsonResponse(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response

swarm/api/http/server.go

@@ -366,7 +366,7 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api
 		}
 
 		var size int64
-		var reader io.Reader = part
+		var reader io.Reader
 		if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
 			size, err = strconv.ParseInt(contentLength, 10, 64)
 			if err != nil {
@@ -484,7 +484,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if updateRequest.IsUpdate() {
+	switch {
+	case updateRequest.IsUpdate():
 		// Verify that the signature is intact and that the signer is authorized
 		// to update this feed
 		// Check this early, to avoid creating a feed and then not being able to set its first update.
@@ -497,9 +498,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
 			respondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
-	}
+		fallthrough
-
+	case query.Get("manifest") == "1":
-	if query.Get("manifest") == "1" {
 		// we create a manifest so we can retrieve feed updates with bzz:// later
 		// this manifest has a special "feed type" manifest, and saves the
 		// feed identification used to retrieve feed updates later
@@ -519,6 +519,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, string(outdata))
 
 		w.Header().Add("Content-type", "application/json")
+	default:
+		respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest)
 	}
 }
 

swarm/api/http/server_test.go

@@ -20,7 +20,6 @@ import (
 	"archive/tar"
 	"bytes"
 	"context"
-	"crypto/rand"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -58,7 +57,7 @@ func init() {
 	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
 }
 
-func serverFunc(api *api.API) testutil.TestServer {
+func serverFunc(api *api.API) TestServer {
 	return NewServer(api, "")
 }
 
@@ -79,7 +78,7 @@ func TestBzzFeedMultihash(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	// add the data our multihash aliased manifest will point to
@@ -167,26 +166,19 @@ func TestBzzFeedMultihash(t *testing.T) {
 
 // Test Swarm feeds using the raw update methods
 func TestBzzFeed(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	signer, _ := newTestSigner()
 
 	defer srv.Close()
 
 	// data of update 1
-	update1Data := make([]byte, 666)
+	update1Data := testutil.RandomBytes(1, 666)
 	update1Timestamp := srv.CurrentTime
-	_, err := rand.Read(update1Data)
-	if err != nil {
-		t.Fatal(err)
-	}
 	//data for update 2
 	update2Data := []byte("foo")
 
 	topic, _ := feed.NewTopic("foo.eth", nil)
 	updateRequest := feed.NewFirstRequest(topic)
-	if err != nil {
-		t.Fatal(err)
-	}
 	updateRequest.SetData(update1Data)
 
 	if err := updateRequest.Sign(signer); err != nil {
@@ -263,7 +255,7 @@ func TestBzzFeed(t *testing.T) {
 	if resp.StatusCode == http.StatusOK {
 		t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK")
 	}
-	b, err = ioutil.ReadAll(resp.Body)
+	_, err = ioutil.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -333,15 +325,45 @@ func TestBzzFeed(t *testing.T) {
 	}
 	urlQuery = testUrl.Query()
 	body = updateRequest.AppendValues(urlQuery) // this adds all query parameters
+	goodQueryParameters := urlQuery.Encode()    // save the query parameters for a second attempt
+
+	// create bad query parameters in which the signature is missing
+	urlQuery.Del("signature")
 	testUrl.RawQuery = urlQuery.Encode()
 
+	// 1st attempt with bad query parameters in which the signature is missing
 	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
+	expectedCode := http.StatusBadRequest
-		t.Fatalf("Update returned %s", resp.Status)
+	if resp.StatusCode != expectedCode {
+		t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode)
+	}
+
+	// 2nd attempt with bad query parameters in which the signature is of incorrect length
+	urlQuery.Set("signature", "0xabcd") // should be 130 hex chars
+	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	expectedCode = http.StatusBadRequest
+	if resp.StatusCode != expectedCode {
+		t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode)
+	}
+
+	// 3rd attempt, with good query parameters:
+	testUrl.RawQuery = goodQueryParameters
+	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	expectedCode = http.StatusOK
+	if resp.StatusCode != expectedCode {
+		t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode)
 	}
 
 	// get latest update through bzz-feed directly
@@ -420,7 +442,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
 
 	addr := [3]storage.Address{}
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	for i, mf := range testmanifest {
@@ -461,6 +483,9 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
 		}
 		defer resp.Body.Close()
 		respbody, err = ioutil.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("Error while reading response body: %v", err)
+		}
 
 		if string(respbody) != testmanifest[v] {
 			isexpectedfailrequest := false
@@ -655,7 +680,7 @@ func TestBzzTar(t *testing.T) {
 }
 
 func testBzzTar(encrypted bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 	fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"}
 	fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
@@ -790,7 +815,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) {
 }
 
 func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	// create a manifest with some data at the root path
@@ -845,7 +870,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
 }
 
 func TestMethodsNotAllowed(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 	databytes := "bar"
 	for _, c := range []struct {
@@ -904,7 +929,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string
 }
 
 func TestGet(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	for _, testCase := range []struct {
@@ -987,7 +1012,7 @@ func TestGet(t *testing.T) {
 }
 
 func TestModify(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	swarmClient := swarm.NewClient(srv.URL)
@@ -1088,7 +1113,7 @@ func TestMultiPartUpload(t *testing.T) {
 	// POST /bzz:/ Content-Type: multipart/form-data
 	verbose := false
 	// Setup Swarm
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	url := fmt.Sprintf("%s/bzz:/", srv.URL)
@@ -1119,7 +1144,7 @@ func TestMultiPartUpload(t *testing.T) {
 // TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver
 func TestBzzGetFileWithResolver(t *testing.T) {
 	resolver := newTestResolveValidator("")
-	srv := testutil.NewTestSwarmServer(t, serverFunc, resolver)
+	srv := NewTestSwarmServer(t, serverFunc, resolver)
 	defer srv.Close()
 	fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"}
 	fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}

swarm/api/http/test_server.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package testutil
+package http
 
 import (
 	"io/ioutil"

swarm/api/manifest.go

@@ -557,7 +557,6 @@ func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manif
 			if path != entry.Path {
 				return nil, 0
 			}
-			pos = epl
 		}
 	}
 	return nil, 0

swarm/bmt/bmt_test.go

@@ -18,10 +18,8 @@ package bmt
 
 import (
 	"bytes"
-	crand "crypto/rand"
 	"encoding/binary"
 	"fmt"
-	"io"
 	"math/rand"
 	"sync"
 	"sync/atomic"
@@ -29,6 +27,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // the actual data length generated (could be longer than max datalength of the BMT)
@@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) {
 	})
 
 	// run the tests
-	for _, x := range tests {
+	for i, x := range tests {
 		for segmentCount := x.from; segmentCount <= x.to; segmentCount++ {
 			for length := 1; length <= segmentCount*32; length++ {
 				t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) {
-					data := make([]byte, length)
+					data := testutil.RandomBytes(i, length)
-					if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF {
-						t.Fatal(err)
-					}
 					expected := x.expected(data)
 					actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data)
 					if !bytes.Equal(actual, expected) {
@@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) {
 
 // tests sequential write with entire max size written in one go
 func TestSyncHasherCorrectness(t *testing.T) {
-	data := newData(BufferSize)
+	data := testutil.RandomBytes(1, BufferSize)
 	hasher := sha3.NewKeccak256
 	size := hasher().Size()
 
@@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) {
 
 // tests order-neutral concurrent writes with entire max size written in one go
 func TestAsyncCorrectness(t *testing.T) {
-	data := newData(BufferSize)
+	data := testutil.RandomBytes(1, BufferSize)
 	hasher := sha3.NewKeccak256
 	size := hasher().Size()
 	whs := []whenHash{first, last, random}
@@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) {
 	bmt := New(pool)
 
 	for i := 0; i < 100; i++ {
-		data := newData(BufferSize)
+		data := testutil.RandomBytes(1, BufferSize)
 		n := rand.Intn(bmt.Size())
 		err := testHasherCorrectness(bmt, hasher, data, n, segmentCount)
 		if err != nil {
@@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) {
 	for i := 0; i < cycles; i++ {
 		go func() {
 			bmt := New(pool)
-			data := newData(BufferSize)
+			data := testutil.RandomBytes(1, BufferSize)
 			n := rand.Intn(bmt.Size())
 			errc <- testHasherCorrectness(bmt, hasher, data, n, 128)
 		}()
@@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) {
 			defer pool.Drain(0)
 			n := count * 32
 			bmt := New(pool)
-			data := newData(n)
+			data := testutil.RandomBytes(1, n)
 			rbmt := NewRefHasher(hasher, count)
 			refHash := rbmt.Hash(data)
 			expHash := syncHash(bmt, nil, data)
@@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) {
 
 // benchmarks simple sha3 hash on chunks
 func benchmarkSHA3(t *testing.B, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	h := hasher()
 
@@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) {
 func benchmarkBMTBaseline(t *testing.B, n int) {
 	hasher := sha3.NewKeccak256
 	hashSize := hasher().Size()
-	data := newData(hashSize)
+	data := testutil.RandomBytes(1, hashSize)
 
 	t.ReportAllocs()
 	t.ResetTimer()
@@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) {
 
 // benchmarks BMT Hasher
 func benchmarkBMT(t *testing.B, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	bmt := New(pool)
@@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) {
 
 // benchmarks BMT hasher with asynchronous concurrent segment/section writes
 func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	bmt := New(pool).NewAsyncWriter(double)
 	idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
-	shuffle(len(idxs), func(i int, j int) {
+	rand.Shuffle(len(idxs), func(i int, j int) {
 		idxs[i], idxs[j] = idxs[j], idxs[i]
 	})
 
@@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
 
 // benchmarks 100 concurrent bmt hashes with pool capacity
 func benchmarkPool(t *testing.B, poolsize, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	pool := NewTreePool(hasher, segmentCount, poolsize)
 	cycles := 100
@@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) {
 
 // benchmarks the reference hasher
 func benchmarkRefHasher(t *testing.B, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	rbmt := NewRefHasher(hasher, 128)
 
@@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) {
 	}
 }
 
-func newData(bufferSize int) []byte {
-	data := make([]byte, bufferSize)
-	_, err := io.ReadFull(crand.Reader, data)
-	if err != nil {
-		panic(err.Error())
-	}
-	return data
-}
-
 // Hash hashes the data and the span using the bmt hasher
 func syncHash(h *Hasher, span, data []byte) []byte {
 	h.ResetWithLength(span)
@@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) {
 		section := data[i*secsize : end]
 		segments = append(segments, section)
 	}
-	shuffle(n, func(i int, j int) {
+	rand.Shuffle(n, func(i int, j int) {
 		idxs[i], idxs[j] = idxs[j], idxs[i]
 	})
 	return idxs, segments
@@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s
 	}
 	return <-c
 }
-
-// this is also in swarm/network_test.go
-// shuffle pseudo-randomizes the order of elements.
-// n is the number of elements. Shuffle panics if n < 0.
-// swap swaps the elements with indexes i and j.
-func shuffle(n int, swap func(i, j int)) {
-	if n < 0 {
-		panic("invalid argument to Shuffle")
-	}
-
-	// Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
-	// Shuffle really ought not be called with n that doesn't fit in 32 bits.
-	// Not only will it take a very long time, but with 2³¹! possible permutations,
-	// there's no way that any PRNG can have a big enough internal state to
-	// generate even a minuscule percentage of the possible permutations.
-	// Nevertheless, the right API signature accepts an int n, so handle it as best we can.
-	i := n - 1
-	for ; i > 1<<31-1-1; i-- {
-		j := int(rand.Int63n(int64(i + 1)))
-		swap(i, j)
-	}
-	for ; i > 0; i-- {
-		j := int(rand.Int31n(int32(i + 1)))
-		swap(i, j)
-	}
-}

swarm/fuse/swarmfs_test.go

@@ -20,20 +20,19 @@ package fuse
 
 import (
 	"bytes"
-	"crypto/rand"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-
+	"github.com/ethereum/go-ethereum/swarm/testutil"
-	"github.com/ethereum/go-ethereum/log"
-
 	colorable "github.com/mattn/go-colorable"
 )
 
@@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) {
 	}
 }
 
-func getRandomBytes(size int) []byte {
-	contents := make([]byte, size)
-	rand.Read(contents)
-	return contents
-}
-
 func isDirEmpty(name string) bool {
 	f, err := os.Open(name)
 	if err != nil {
@@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) {
 	dat.testMountDir = filepath.Join(dat.testDir, "testMountDir")
 	dat.files = make(map[string]fileInfo)
 
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)}
+	dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)}
+	dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)}
-	dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)}
+	dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)}
-	dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)}
+	dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)}
-	dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)}
+	dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)}
-	dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)}
+	dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)}
-	dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)}
+	dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)}
-	dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)}
-	dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)}
-	dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)}
-	dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)}
-	dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)}
+	dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)}
-	dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)}
+	dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)}
-	dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)}
+	dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)}
-	dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount1")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount2")
-	dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount3")
-	dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount4")
-	dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount5")
-	dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 	if err != nil {
 		t.Fatalf("Couldn't create upload dir 6: %v", err)
 	}
-	dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 	testMountDir6 := filepath.Join(dat.testDir, "max-mount6")
 	err = os.MkdirAll(testMountDir6, 0777)
 	if err != nil {
@@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
 	dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1")
 	dat.files = make(map[string]fileInfo)
 
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
 	}
 
 	// mount a different hash in already mounted point
-	dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 	testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2")
 	if err3 != nil {
 		t.Fatalf("Error creating second upload dir: %v", err3)
@@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
 	//we need to manually close the file before mount for this test
 	//but let's defer too in case of errors
 	defer d.Close()
-	_, err = d.Write(getRandomBytes(10))
+	_, err = d.Write(testutil.RandomBytes(1, 10))
 	if err != nil {
 		t.Fatalf("Couldn't write to file: %v", err)
 	}
@@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "seek-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "create-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
 	}
 	defer d.Close()
 	log.Debug("Opened file")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Could not rand read contents %v", err)
-	}
 	log.Debug("content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
 	}
 	defer d.Close()
 	log.Debug("File opened")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Error filling random bytes into byte array %v", err)
-	}
 	log.Debug("Content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
 	dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
 	}
 	defer d.Close()
 	log.Debug("File opened")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 	log.Debug("content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
 	}
 	defer d.Close()
 	log.Debug("file opened")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 	log.Debug("content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
-	dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)}
-	dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)}
-	dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
 	dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount")
 	dat.files = make(map[string]fileInfo)
 
-	line1 := make([]byte, 10)
+	line1 := testutil.RandomBytes(1, 10)
-	_, err = rand.Read(line1)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 
 	dat.files["1.txt"] = fileInfo{0700, 333, 444, line1}
 
@@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
 	}
 	defer fd.Close()
 	log.Debug("file opened")
-	line2 := make([]byte, 5)
+	line2 := testutil.RandomBytes(1, 5)
-	_, err = rand.Read(line2)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 	log.Debug("line read")
 	_, err = fd.Seek(int64(len(line1)), 0)
 	if err != nil {

swarm/network/hive_test.go

@@ -70,6 +70,9 @@ func TestHiveStatePersistance(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	store, err := state.NewDBStore(dir) //start the hive with an empty dbstore
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	params := NewHiveParams()
 	s, pp := newHiveTester(t, params, 5, store)
@@ -90,6 +93,9 @@ func TestHiveStatePersistance(t *testing.T) {
 	store.Close()
 
 	persistedStore, err := state.NewDBStore(dir) //start the hive with an empty dbstore
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	s1, pp := newHiveTester(t, params, 1, persistedStore)
 

swarm/network/kademlia.go

@@ -261,7 +261,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) {
 		// found among live peers, do nothing
 		return v
 	})
-	if ins {
+	if ins && !p.BzzPeer.LightNode {
 		a := newEntry(p.BzzAddr)
 		a.conn = p
 		// insert new online peer into addrs
@@ -329,6 +329,7 @@ func (k *Kademlia) Off(p *Peer) {
 	k.lock.Lock()
 	defer k.lock.Unlock()
 	var del bool
+	if !p.BzzPeer.LightNode {
 		k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val {
 			// v cannot be nil, must check otherwise we overwrite entry
 			if v == nil {
@@ -337,6 +338,9 @@ func (k *Kademlia) Off(p *Peer) {
 			del = true
 			return newEntry(p.BzzAddr)
 		})
+	} else {
+		del = true
+	}
 
 	if del {
 		k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val {

swarm/network/kademlia_test.go

@@ -46,19 +46,19 @@ func newTestKademlia(b string) *Kademlia {
 	return NewKademlia(base, params)
 }
 
-func newTestKadPeer(k *Kademlia, s string) *Peer {
+func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer {
-	return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k)
+	return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k)
 }
 
 func On(k *Kademlia, ons ...string) {
 	for _, s := range ons {
-		k.On(newTestKadPeer(k, s))
+		k.On(newTestKadPeer(k, s, false))
 	}
 }
 
 func Off(k *Kademlia, offs ...string) {
 	for _, s := range offs {
-		k.Off(newTestKadPeer(k, s))
+		k.Off(newTestKadPeer(k, s, false))
 	}
 }
 
@@ -254,6 +254,56 @@ func TestSuggestPeerFindPeers(t *testing.T) {
 
 }
 
+// a node should stay in the address book if it's removed from the kademlia
+func TestOffEffectingAddressBookNormalNode(t *testing.T) {
+	k := newTestKademlia("00000000")
+	// peer added to kademlia
+	k.On(newTestKadPeer(k, "01000000", false))
+	// peer should be in the address book
+	if k.addrs.Size() != 1 {
+		t.Fatal("known peer addresses should contain 1 entry")
+	}
+	// peer should be among live connections
+	if k.conns.Size() != 1 {
+		t.Fatal("live peers should contain 1 entry")
+	}
+	// remove peer from kademlia
+	k.Off(newTestKadPeer(k, "01000000", false))
+	// peer should be in the address book
+	if k.addrs.Size() != 1 {
+		t.Fatal("known peer addresses should contain 1 entry")
+	}
+	// peer should not be among live connections
+	if k.conns.Size() != 0 {
+		t.Fatal("live peers should contain 0 entry")
+	}
+}
+
+// a light node should not be in the address book
+func TestOffEffectingAddressBookLightNode(t *testing.T) {
+	k := newTestKademlia("00000000")
+	// light node peer added to kademlia
+	k.On(newTestKadPeer(k, "01000000", true))
+	// peer should not be in the address book
+	if k.addrs.Size() != 0 {
+		t.Fatal("known peer addresses should contain 0 entry")
+	}
+	// peer should be among live connections
+	if k.conns.Size() != 1 {
+		t.Fatal("live peers should contain 1 entry")
+	}
+	// remove peer from kademlia
+	k.Off(newTestKadPeer(k, "01000000", true))
+	// peer should not be in the address book
+	if k.addrs.Size() != 0 {
+		t.Fatal("known peer addresses should contain 0 entry")
+	}
+	// peer should not be among live connections
+	if k.conns.Size() != 0 {
+		t.Fatal("live peers should contain 0 entry")
+	}
+}
+
 func TestSuggestPeerRetries(t *testing.T) {
 	k := newTestKademlia("00000000")
 	k.RetryInterval = int64(300 * time.Millisecond) // cycle

swarm/network/protocol_test.go

@@ -153,17 +153,7 @@ func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *
 
 // should test handshakes in one exchange? parallelisation
 func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptest.Disconnect) error {
-	var peers []enode.ID
+	if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, rhs.Addr.ID())...); err != nil {
-	id := rhs.Addr.ID()
-	if len(disconnects) > 0 {
-		for _, d := range disconnects {
-			peers = append(peers, d.Peer)
-		}
-	} else {
-		peers = []enode.ID{id}
-	}
-
-	if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, id)...); err != nil {
 		return err
 	}