Merge branch 'master' of github.com:ethereum/go-ethereum into ulc/master_2

.travis.yml

@@ -29,6 +29,14 @@ matrix:
     - os: osx
       go: 1.11.x
       script:
+        - echo "Increase the maximum number of open file descriptors on macOS"
+        - NOFILE=20480
+        - sudo sysctl -w kern.maxfiles=$NOFILE
+        - sudo sysctl -w kern.maxfilesperproc=$NOFILE
+        - sudo launchctl limit maxfiles $NOFILE $NOFILE
+        - sudo launchctl limit maxfiles
+        - ulimit -S -n $NOFILE
+        - ulimit -n
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES

README.md

@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 

accounts/keystore/account_cache.go

@@ -265,7 +265,10 @@ func (ac *accountCache) scanAccounts() error {
 		case (addr == common.Address{}):
 			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
 		default:
-			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+			return &accounts.Account{
+				Address: addr,
+				URL:     accounts.URL{Scheme: KeyStoreScheme, Path: path},
+			}
 		}
 		return nil
 	}

accounts/keystore/key.go

@@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	if err != nil {
 		return nil, accounts.Account{}, err
 	}
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))},
+	}
 	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
@@ -224,5 +227,6 @@ func toISO8601(t time.Time) string {
 	} else {
 		tz = fmt.Sprintf("%03d00", offset/3600)
 	}
-	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
+	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s",
+		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
 }

accounts/keystore/passphrase.go

@@ -233,6 +233,7 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 		PrivateKey: key,
 	}, nil
 }
+
 func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
 	if cryptoJson.Cipher != "aes-128-ctr" {
 		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)

accounts/keystore/presale.go

@@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou
 		return accounts.Account{}, nil, err
 	}
 	key.Id = uuid.NewRandom()
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL: accounts.URL{
+			Scheme: KeyStoreScheme,
+			Path:   keyStore.JoinPath(keyFileName(key.Address)),
+		},
+	}
 	err = keyStore.StoreKey(a.URL.Path, key, password)
 	return a, key, err
 }

cmd/evm/runner.go

@@ -206,6 +206,7 @@ func runCmd(ctx *cli.Context) error {
 	execTime := time.Since(tstart)
 
 	if ctx.GlobalBool(DumpFlag.Name) {
+		statedb.Commit(true)
 		statedb.IntermediateRoot(true)
 		fmt.Println(string(statedb.Dump()))
 	}

cmd/geth/main.go

@@ -93,6 +93,7 @@ var (
 		utils.LightKDFFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
+		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
 		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,

cmd/geth/usage.go

@@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
 			utils.TrieCacheGenFlag,
 		},

cmd/swarm/access_test.go

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !windows
-
 package main
 
 import (
@@ -28,6 +26,7 @@ import (
 	gorand "math/rand"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -37,7 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -48,22 +47,41 @@ const (
 
 var DefaultCurve = crypto.S256()
 
-// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
+func TestACT(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"Password", testPassword},
+		{"PK", testPK},
+		{"ACTWithoutBogus", testACTWithoutBogus},
+		{"ACTWithBogus", testACTWithBogus},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testPassword tests for the correct creation of an ACT manifest protected by a password.
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
 // is then fetched through 2nd node. since the tested code is not key-aware - we can just
 // fetch from the 2nd node using HTTP BasicAuth
-func TestAccessPassword(t *testing.T) {
+func testPassword(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
-	defer srv.Close()
-
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
 		"--bzzapi",
-		srv.URL, //it doesn't matter through which node we upload content
+		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
 		dataFilename)
@@ -137,16 +155,17 @@ func TestAccessPassword(t *testing.T) {
 	if a.Publisher != "" {
 		t.Fatal("should be empty")
 	}
-	client := swarm.NewClient(srv.URL)
+
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	httpClient := &http.Client{}
+	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
 
-	url := srv.URL + "/" + "bzz:/" + hash
+	httpClient := &http.Client{}
 	response, err := httpClient.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -188,7 +207,7 @@ func TestAccessPassword(t *testing.T) {
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
 		"--bzzapi",
-		srv.URL,
+		cluster.Nodes[0].URL,
 		"down",
 		"bzz:/"+hash,
 		tmp,
@@ -202,16 +221,12 @@ func TestAccessPassword(t *testing.T) {
 	up.ExpectExit()
 }
 
-// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
+// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
 // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
 // the test will fail if the proxy's given private key is not granted on the ACT.
-func TestAccessPK(t *testing.T) {
+func testPK(t *testing.T) {
-	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 2)
-	defer cluster.Shutdown()
-
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
@@ -317,7 +332,7 @@ func TestAccessPK(t *testing.T) {
 	if a.Publisher != pkComp {
 		t.Fatal("publisher key did not match")
 	}
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
@@ -343,29 +358,24 @@ func TestAccessPK(t *testing.T) {
 	}
 }
 
-// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
+// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
-func TestAccessACT(t *testing.T) {
+func testACTWithoutBogus(t *testing.T) {
-	testAccessACT(t, 0)
+	testACT(t, 0)
 }
 
-// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
+// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
-func TestAccessACTScale(t *testing.T) {
+func testACTWithBogus(t *testing.T) {
-	testAccessACT(t, 1000)
+	testACT(t, 100)
 }
 
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
 // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
 // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
 // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
 // the publisher uploads through one of the nodes then disappears.
-func testAccessACT(t *testing.T, bogusEntries int) {
+func testACT(t *testing.T, bogusEntries int) {
-	// Setup Swarm and upload a test file to it
-	const clusterSize = 3
-	cluster := newTestCluster(t, clusterSize)
-	defer cluster.Shutdown()
-
 	var uploadThroughNode = cluster.Nodes[0]
-	client := swarm.NewClient(uploadThroughNode.URL)
+	client := swarmapi.NewClient(uploadThroughNode.URL)
 
 	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
 	nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)

cmd/swarm/config_test.go

@@ -26,14 +26,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	"github.com/ethereum/go-ethereum/swarm/api"
-
-	"github.com/docker/docker/pkg/reexec"
 )
 
-func TestDumpConfig(t *testing.T) {
+func TestConfigDump(t *testing.T) {
 	swarm := runSwarm(t, "dumpconfig")
 	defaultConf := api.NewConfig()
 	out, err := tomlSettings.Marshal(&defaultConf)
@@ -91,8 +91,8 @@ func TestConfigCmdLineOverrides(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
 		fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name),
 		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--datadir", dir,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -189,9 +189,9 @@ func TestConfigFileOverrides(t *testing.T) {
 	flags := []string{
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
-		"--ens-api", "",
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
-		"--datadir", dir,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -407,9 +407,9 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
-		"--ens-api", "",
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--datadir", dir,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -466,7 +466,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 	node.Shutdown()
 }
 
-func TestValidateConfig(t *testing.T) {
+func TestConfigValidate(t *testing.T) {
 	for _, c := range []struct {
 		cfg *api.Config
 		err string

cmd/swarm/export_test.go

@@ -19,9 +19,7 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"runtime"
@@ -29,6 +27,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // TestCLISwarmExportImport perform the following test:
@@ -44,12 +43,13 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 	cluster := newTestCluster(t, 1)
 
-	// generate random 10mb file
+	// generate random 1mb file
-	f, cleanup := generateRandomFile(t, 10000000)
+	content := testutil.RandomBytes(1, 1000000)
-	defer cleanup()
+	fileName := testutil.TempFileWithContent(t, string(content))
+	defer os.Remove(fileName)
 
 	// upload the file with 'swarm up' and expect a hash
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
 	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
 	up.ExpectExit()
 	hash := matches[0]
@@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 
 	// compare downloaded file with the generated random file
-	mustEqualFiles(t, f, res.Body)
+	mustEqualFiles(t, bytes.NewReader(content), res.Body)
 }
 
 func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
 		t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
 	}
 }
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	// write 10mb random data to file
-	buf := make([]byte, 10000000)
-	_, err = rand.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
-}

cmd/swarm/feeds.go

@@ -169,7 +169,6 @@ func feedUpdate(ctx *cli.Context) {
 		query = new(feed.Query)
 		query.User = signer.Address()
 		query.Topic = getTopic(ctx)
-
 	}
 
 	// Retrieve a feed update request
@@ -178,6 +177,11 @@ func feedUpdate(ctx *cli.Context) {
 		utils.Fatalf("Error retrieving feed status: %s", err.Error())
 	}
 
+	// Check that the provided signer matches the request to sign
+	if updateRequest.User != signer.Address() {
+		utils.Fatalf("Signer address does not match the update request")
+	}
+
 	// set the new data
 	updateRequest.SetData(data)
 

cmd/swarm/feeds_test.go

@@ -19,50 +19,35 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestCLIFeedUpdate(t *testing.T) {
-
+	srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
-	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
 		return swarmhttp.NewServer(api, "")
 	}, nil)
 	log.Info("starting a test swarm server")
 	defer srv.Close()
 
 	// create a private key file for signing
-	pkfile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer pkfile.Close()
-	defer os.Remove(pkfile.Name())
-
 	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
 	privKey, _ := crypto.HexToECDSA(privkeyHex)
 	address := crypto.PubkeyToAddress(privKey.PublicKey)
 
-	// save the private key to a file
+	pkFileName := testutil.TempFileWithContent(t, privkeyHex)
-	_, err = io.WriteString(pkfile, privkeyHex)
+	defer os.Remove(pkFileName)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// compose a topic. We'll be doing quotes about Miguel de Cervantes
 	var topic feed.Topic
@@ -76,26 +61,23 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	flags := []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "update",
 		"--topic", topic.Hex(),
 		"--name", name,
 		hexData}
 
 	// create an update and expect an exit without errors
-	log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'"))
+	log.Info("updating a feed with 'swarm feed update'")
 	cmd := runSwarm(t, flags...)
 	cmd.ExpectExit()
 
 	// now try to get the update using the client
 	client := swarm.NewClient(srv.URL)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// build the same topic as before, this time
 	// we use NewTopic to create a topic automatically.
-	topic, err = feed.NewTopic(name, subject)
+	topic, err := feed.NewTopic(name, subject)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -133,7 +115,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 		"--user", address.Hex(),
 	}
 
-	log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'"))
+	log.Info("getting feed info with 'swarm feed info'")
 	cmd = runSwarm(t, flags...)
 	_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
 	cmd.ExpectExit()
@@ -153,14 +135,14 @@ func TestCLIFeedUpdate(t *testing.T) {
 	// test publishing a manifest
 	flags = []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "create",
 		"--topic", topic.Hex(),
 	}
 
-	log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'"))
+	log.Info("Publishing manifest with 'swarm feed create'")
 	cmd = runSwarm(t, flags...)
-	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
 	cmd.ExpectExit()
 
 	manifestAddress := matches[0] // read the received feed manifest
@@ -179,4 +161,36 @@ func TestCLIFeedUpdate(t *testing.T) {
 	if !bytes.Equal(data, retrieved) {
 		t.Fatalf("Received %s, expected %s", retrieved, data)
 	}
+
+	// test publishing a manifest for a different user
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"feed", "create",
+		"--topic", topic.Hex(),
+		"--user", "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // different user
+	}
+
+	log.Info("Publishing manifest with 'swarm feed create' for a different user")
+	cmd = runSwarm(t, flags...)
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
+	cmd.ExpectExit()
+
+	manifestAddress = matches[0] // read the received feed manifest
+
+	// now let's try to update that user's manifest which we don't have the private key for
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkFileName,
+		"feed", "update",
+		"--manifest", manifestAddress,
+		hexData}
+
+	// create an update and expect an error given there is a user mismatch
+	log.Info("updating a feed with 'swarm feed update'")
+	cmd = runSwarm(t, flags...)
+	cmd.ExpectRegexp("Fatal:.*") // best way so far to detect a failure.
+	cmd.ExpectExit()
+	if cmd.ExitStatus() == 0 {
+		t.Fatal("Expected nonzero exit code when updating a manifest with the wrong user. Got 0.")
+	}
 }

cmd/swarm/fs.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/fuse"
 	"gopkg.in/urfave/cli.v1"
@@ -41,27 +41,24 @@ var fsCommand = cli.Command{
 			Action:             mount,
 			CustomHelpTemplate: helpTemplate,
 			Name:               "mount",
-			Flags:              []cli.Flag{utils.IPCPathFlag},
 			Usage:              "mount a swarm hash to a mount point",
-			ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
+			ArgsUsage:          "swarm fs mount <manifest hash> <mount point>",
 			Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
 		},
 		{
 			Action:             unmount,
 			CustomHelpTemplate: helpTemplate,
 			Name:               "unmount",
-			Flags:              []cli.Flag{utils.IPCPathFlag},
 			Usage:              "unmount a swarmfs mount",
-			ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
+			ArgsUsage:          "swarm fs unmount <mount point>",
 			Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
 		},
 		{
 			Action:             listMounts,
 			CustomHelpTemplate: helpTemplate,
 			Name:               "list",
-			Flags:              []cli.Flag{utils.IPCPathFlag},
 			Usage:              "list swarmfs mounts",
-			ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
+			ArgsUsage:          "swarm fs list",
 			Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
 		},
 	},
@@ -70,7 +67,7 @@ var fsCommand = cli.Command{
 func mount(cliContext *cli.Context) {
 	args := cliContext.Args()
 	if len(args) < 2 {
-		utils.Fatalf("Usage: swarm fs mount --ipcpath <path to bzzd.ipc> <manifestHash> <file name>")
+		utils.Fatalf("Usage: swarm fs mount <manifestHash> <file name>")
 	}
 
 	client, err := dialRPC(cliContext)
@@ -97,7 +94,7 @@ func unmount(cliContext *cli.Context) {
 	args := cliContext.Args()
 
 	if len(args) < 1 {
-		utils.Fatalf("Usage: swarm fs unmount --ipcpath <path to bzzd.ipc> <mount path>")
+		utils.Fatalf("Usage: swarm fs unmount <mount path>")
 	}
 	client, err := dialRPC(cliContext)
 	if err != nil {
@@ -145,20 +142,21 @@ func listMounts(cliContext *cli.Context) {
 }
 
 func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
-	var endpoint string
+	endpoint := getIPCEndpoint(ctx)
+	log.Info("IPC endpoint", "path", endpoint)
+	return rpc.Dial(endpoint)
+}
 
-	if ctx.IsSet(utils.IPCPathFlag.Name) {
+func getIPCEndpoint(ctx *cli.Context) string {
-		endpoint = ctx.String(utils.IPCPathFlag.Name)
+	cfg := defaultNodeConfig
-	} else {
+	utils.SetNodeConfig(ctx, &cfg)
-		utils.Fatalf("swarm ipc endpoint not specified")
-	}
 
-	if endpoint == "" {
+	endpoint := cfg.IPCEndpoint()
-		endpoint = node.DefaultIPCEndpoint(clientIdentifier)
+
-	} else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
+	if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
 		// Backwards compatibility with geth < 1.5 which required
 		// these prefixes.
 		endpoint = endpoint[4:]
 	}
-	return rpc.Dial(endpoint)
+	return endpoint
 }

cmd/swarm/fs_test.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -28,20 +29,35 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 )
 
-func init() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
-}
-
 type testFile struct {
 	filePath string
 	content  string
 }
 
+// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list
+// can find and correctly connect to a running Swarm node on the default
+// IPCPath.
+func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	handlingNode := cluster.Nodes[0]
+	list := runSwarm(t, []string{
+		"--datadir", handlingNode.Dir,
+		"fs",
+		"list",
+	}...)
+
+	list.WaitExit()
+	if list.Err != nil {
+		t.Fatal(list.Err)
+	}
+}
+
 // TestCLISwarmFs is a high-level test of swarmfs
 //
 // This test fails on travis for macOS as this executable exits with code 1
@@ -65,9 +81,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	mount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"mount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		mhash,
 		mountPoint,
 	}...)
@@ -107,9 +123,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	unmount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"unmount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		mountPoint,
 	}...)
 	_, matches := unmount.ExpectRegexp(hashRegexp)
@@ -142,9 +158,9 @@ func TestCLISwarmFs(t *testing.T) {
 
 	//remount, check files
 	newMount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"mount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		hash, // the latest hash
 		secondMountPoint,
 	}...)
@@ -178,9 +194,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	unmountSec := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"unmount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		secondMountPoint,
 	}...)
 

cmd/swarm/manifest_test.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 // TestManifestChange tests manifest add, update and remove
@@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) {
 // Argument encrypt controls whether to use encryption or not.
 func testManifestChange(t *testing.T, encrypt bool) {
 	t.Parallel()
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
@@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
 
 func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 	t.Parallel()
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")

cmd/swarm/run_test.go

@@ -42,7 +42,6 @@ import (
 	"github.com/ethereum/go-ethereum/swarm"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@@ -58,7 +57,18 @@ func init() {
 	})
 }
 
-func serverFunc(api *api.API) testutil.TestServer {
+const clusterSize = 3
+
+var clusteronce sync.Once
+var cluster *testCluster
+
+func initCluster(t *testing.T) {
+	clusteronce.Do(func() {
+		cluster = newTestCluster(t, clusterSize)
+	})
+}
+
+func serverFunc(api *api.API) swarmhttp.TestServer {
 	return swarmhttp.NewServer(api, "")
 }
 func TestMain(m *testing.M) {

cmd/swarm/swarm-smoke/feed_upload_and_sync.go

@@ -16,7 +16,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
 	colorable "github.com/mattn/go-colorable"
 	"github.com/pborman/uuid"
@@ -36,7 +35,7 @@ func cliFeedUploadAndSync(c *cli.Context) error {
 
 	generateEndpoints(scheme, cluster, from, to)
 
-	log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing")
+	log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing")
 
 	// create a random private key to sign updates with and derive the address
 	pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
@@ -218,8 +217,7 @@ func cliFeedUploadAndSync(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes))
+	multihashHex := hexutil.Encode(hashBytes)
-
 	fileHash, err := digest(f)
 	if err != nil {
 		return err

cmd/swarm/swarm-smoke/main.go

@@ -45,8 +45,8 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "cluster-endpoint",
-			Value:       "testing",
+			Value:       "prod",
-			Usage:       "cluster to point to (local, open or testing)",
+			Usage:       "cluster to point to (prod or a given namespace)",
 			Destination: &cluster,
 		},
 		cli.IntFlag{

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	crand "crypto/rand"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -32,6 +33,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
+	colorable "github.com/mattn/go-colorable"
 	"github.com/pborman/uuid"
 
 	cli "gopkg.in/urfave/cli.v1"
@@ -39,18 +41,13 @@ import (
 
 func generateEndpoints(scheme string, cluster string, from int, to int) {
 	if cluster == "prod" {
-		cluster = ""
-	} else if cluster == "local" {
 		for port := from; port <= to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
+			endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
 		}
-		return
 	} else {
-		cluster = cluster + "."
-	}
-
 		for port := from; port <= to; port++ {
-		endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
+			endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster))
+		}
 	}
 
 	if includeLocalhost {
@@ -59,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
 }
 
 func cliUploadAndSync(c *cli.Context) error {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+
 	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
 
 	generateEndpoints(scheme, cluster, from, to)
@@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
 	time.Sleep(3 * time.Second)
 
 	log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
-	res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}}
+	res, err := client.Get(endpoint + "/bzz:/" + hash + "/")
 	if err != nil {
 		log.Warn(err.Error(), "ruid", ruid)
 		return err

cmd/swarm/upload_test.go

@@ -31,7 +31,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 	"github.com/mattn/go-colorable"
 )
@@ -41,69 +41,66 @@ func init() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 }
 
-// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
+func TestSwarmUp(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"NoEncryption", testNoEncryption},
+		{"Encrypted", testEncrypted},
+		{"RecursiveNoEncryption", testRecursiveNoEncryption},
+		{"RecursiveEncrypted", testRecursiveEncrypted},
+		{"DefaultPathAll", testDefaultPathAll},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testNoEncryption tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUp(t *testing.T) {
+func testNoEncryption(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	testDefault(false, t)
-		t.Skip()
-	}
-
-	testCLISwarmUp(false, t)
-}
-func TestCLISwarmUpRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(false, t)
 }
 
-// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
+// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUpEncrypted(t *testing.T) {
+func testEncrypted(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	testDefault(true, t)
-		t.Skip()
-	}
-	testCLISwarmUp(true, t)
-}
-func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(true, t)
 }
 
-func testCLISwarmUp(toEncrypt bool, t *testing.T) {
+func testRecursiveNoEncryption(t *testing.T) {
-	log.Info("starting 3 node cluster")
+	testRecursive(false, t)
-	cluster := newTestCluster(t, 3)
+}
-	defer cluster.Shutdown()
 
-	// create a tmp file
+func testRecursiveEncrypted(t *testing.T) {
-	tmp, err := ioutil.TempFile("", "swarm-test")
+	testRecursive(true, t)
-	if err != nil {
+}
-		t.Fatal(err)
+
-	}
+func testDefault(toEncrypt bool, t *testing.T) {
-	defer tmp.Close()
+	tmpFileName := testutil.TempFileWithContent(t, data)
-	defer os.Remove(tmp.Name())
+	defer os.Remove(tmpFileName)
 
 	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	hashRegexp := `[a-f\d]{64}`
 	flags := []string{
 		"--bzzapi", cluster.Nodes[0].URL,
 		"up",
-		tmp.Name()}
+		tmpFileName}
 	if toEncrypt {
 		hashRegexp = `[a-f\d]{128}`
 		flags = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"up",
 			"--encrypt",
-			tmp.Name()}
+			tmpFileName}
 	}
 	// upload the file with 'swarm up' and expect a hash
 	log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@@ -192,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
 	}
 }
 
-func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
+func testRecursive(toEncrypt bool, t *testing.T) {
-	fmt.Println("starting 3 node cluster")
-	cluster := newTestCluster(t, 3)
-	defer cluster.Shutdown()
-
 	tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpUploadDir)
 	// create tmp files
-	data := "notsorandomdata"
 	for _, path := range []string{"tmp1", "tmp2"} {
 		if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
 			t.Fatal(err)
@@ -264,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 
 			switch mode := fi.Mode(); {
 			case mode.IsRegular():
-				if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
+				if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
 					t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
 				} else {
 					ff := make([]byte, len(data))
@@ -285,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 	}
 }
 
-// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// testDefaultPathAll tests swarm recursive upload with relative and absolute
 // default paths and with encryption.
-func TestCLISwarmUpDefaultPath(t *testing.T) {
+func testDefaultPathAll(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	testDefaultPath(false, false, t)
-		t.Skip()
+	testDefaultPath(false, true, t)
-	}
+	testDefaultPath(true, false, t)
-	testCLISwarmUpDefaultPath(false, false, t)
+	testDefaultPath(true, true, t)
-	testCLISwarmUpDefaultPath(false, true, t)
-	testCLISwarmUpDefaultPath(true, false, t)
-	testCLISwarmUpDefaultPath(true, true, t)
 }
 
-func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
+func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
-	defer srv.Close()
-
 	tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
 	if err != nil {
 		t.Fatal(err)
@@ -323,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 
 	args := []string{
 		"--bzzapi",
-		srv.URL,
+		cluster.Nodes[0].URL,
 		"--recursive",
 		"--defaultpath",
 		defaultPath,
@@ -340,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 	up.ExpectExit()
 	hash := matches[0]
 
-	client := swarm.NewClient(srv.URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	m, isEncrypted, err := client.DownloadManifest(hash)
 	if err != nil {

cmd/utils/flags.go

@@ -313,7 +313,12 @@ var (
 	CacheDatabaseFlag = cli.IntFlag{
 		Name:  "cache.database",
 		Usage: "Percentage of cache memory allowance to use for database io",
-		Value: 75,
+		Value: 50,
+	}
+	CacheTrieFlag = cli.IntFlag{
+		Name:  "cache.trie",
+		Usage: "Percentage of cache memory allowance to use for trie caching",
+		Value: 25,
 	}
 	CacheGCFlag = cli.IntFlag{
 		Name:  "cache.gc",
@@ -1025,16 +1030,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setWS(ctx, cfg)
 	setNodeUserIdent(ctx, cfg)
 
-	switch {
+	setDataDir(ctx, cfg)
-	case ctx.GlobalIsSet(DataDirFlag.Name):
-		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
-	case ctx.GlobalBool(DeveloperFlag.Name):
-		cfg.DataDir = "" // unless explicitly requested, use memory databases
-	case ctx.GlobalBool(TestnetFlag.Name):
-		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
-	case ctx.GlobalBool(RinkebyFlag.Name):
-		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
-	}
 
 	if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
 		cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
@@ -1047,6 +1043,19 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	}
 }
 
+func setDataDir(ctx *cli.Context, cfg *node.Config) {
+	switch {
+	case ctx.GlobalIsSet(DataDirFlag.Name):
+		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
+	case ctx.GlobalBool(DeveloperFlag.Name):
+		cfg.DataDir = "" // unless explicitly requested, use memory databases
+	case ctx.GlobalBool(TestnetFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
+	case ctx.GlobalBool(RinkebyFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
+	}
+}
+
 func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
 	if ctx.GlobalIsSet(GpoBlocksFlag.Name) {
 		cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name)
@@ -1212,8 +1221,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	}
 	cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
 
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
 		cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
@@ -1449,11 +1461,15 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	}
 	cache := &core.CacheConfig{
 		Disabled:       ctx.GlobalString(GCModeFlag.Name) == "archive",
-		TrieNodeLimit: eth.DefaultConfig.TrieCache,
+		TrieCleanLimit: eth.DefaultConfig.TrieCleanCache,
+		TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
 		TrieTimeLimit:  eth.DefaultConfig.TrieTimeout,
 	}
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
 	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)

consensus/clique/clique.go

@@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash {
 	return sigHash(header)
 }
 
-// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
+// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
 func (c *Clique) Close() error {
 	return nil
 }

core/blockchain.go

@@ -48,6 +48,9 @@ import (
 
 var (
 	blockInsertTimer     = metrics.NewRegisteredTimer("chain/inserts", nil)
+	blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
+	blockExecutionTimer  = metrics.NewRegisteredTimer("chain/execution", nil)
+	blockWriteTimer      = metrics.NewRegisteredTimer("chain/write", nil)
 
 	ErrNoGenesis = errors.New("Genesis not found in chain")
 )
@@ -69,7 +72,8 @@ const (
 // that's resident in a blockchain.
 type CacheConfig struct {
 	Disabled       bool          // Whether to disable trie write caching (archive node)
-	TrieNodeLimit int           // Memory limit (MB) at which to flush the current in-memory trie to disk
+	TrieCleanLimit int           // Memory allowance (MB) to use for caching trie nodes in memory
+	TrieDirtyLimit int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
 	TrieTimeLimit  time.Duration // Time limit after which to flush the current in-memory trie to disk
 }
 
@@ -140,7 +144,8 @@ type BlockChain struct {
 func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = &CacheConfig{
-			TrieNodeLimit: 256 * 1024 * 1024,
+			TrieCleanLimit: 256,
+			TrieDirtyLimit: 256,
 			TrieTimeLimit:  5 * time.Minute,
 		}
 	}
@@ -156,7 +161,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		cacheConfig:    cacheConfig,
 		db:             db,
 		triegc:         prque.New(nil),
-		stateCache:     state.NewDatabase(db),
+		stateCache:     state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
 		quit:           make(chan struct{}),
 		shouldPreserve: shouldPreserve,
 		bodyCache:      bodyCache,
@@ -393,6 +398,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
 	return state.New(root, bc.stateCache)
 }
 
+// StateCache returns the caching database underpinning the blockchain instance.
+func (bc *BlockChain) StateCache() state.Database {
+	return bc.stateCache
+}
+
 // Reset purges the entire blockchain, restoring it to its genesis state.
 func (bc *BlockChain) Reset() error {
 	return bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -438,7 +448,11 @@ func (bc *BlockChain) repair(head **types.Block) error {
 			return nil
 		}
 		// Otherwise rewind one block and recheck state availability there
-		(*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+		block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+		if block == nil {
+			return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
+		}
+		(*head) = block
 	}
 }
 
@@ -554,6 +568,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
 	return rawdb.HasBody(bc.db, hash, number)
 }
 
+// HasFastBlock checks if a fast block is fully present in the database or not.
+func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
+	if !bc.HasBlock(hash, number) {
+		return false
+	}
+	if bc.receiptsCache.Contains(hash) {
+		return true
+	}
+	return rawdb.HasReceipts(bc.db, hash, number)
+}
+
 // HasState checks if state trie is fully present in the database or not.
 func (bc *BlockChain) HasState(hash common.Hash) bool {
 	_, err := bc.stateCache.OpenTrie(hash)
@@ -611,12 +636,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
 		return receipts.(types.Receipts)
 	}
-
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
-
 	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
 	bc.receiptsCache.Add(hash, receipts)
 	return receipts
@@ -938,7 +961,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 			// If we exceeded our memory allowance, flush matured singleton nodes to disk
 			var (
 				nodes, imgs = triedb.Size()
-				limit       = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
+				limit       = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
 			)
 			if nodes > limit || imgs > 4*1024*1024 {
 				triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1020,6 +1043,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	return status, nil
 }
 
+// addFutureBlock checks if the block is within the max allowed window to get
+// accepted for future processing, and returns an error if the block is too far
+// ahead and was not added.
+func (bc *BlockChain) addFutureBlock(block *types.Block) error {
+	max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
+	if block.Time().Cmp(max) > 0 {
+		return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
+	}
+	bc.futureBlocks.Add(block.Hash(), block)
+	return nil
+}
+
 // InsertChain attempts to insert the given batch of blocks in to the canonical
 // chain or, otherwise, create a fork. If an error is returned it will return
 // the index number of the failing block as well an error describing what went
@@ -1027,18 +1062,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 //
 // After insertion is done, all accumulated events will be fired.
 func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
-	n, events, logs, err := bc.insertChain(chain)
-	bc.PostChainEvents(events, logs)
-	return n, err
-}
-
-// insertChain will execute the actual chain insertion and event aggregation. The
-// only reason this method exists as a separate one is to make locking cleaner
-// with deferred statements.
-func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) {
 	// Sanity check that we have something meaningful to import
 	if len(chain) == 0 {
-		return 0, nil, nil, nil
+		return 0, nil
 	}
 	// Do a sanity check that the provided chain is actually ordered and linked
 	for i := 1; i < len(chain); i++ {
@@ -1047,16 +1073,36 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
 				"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
 
-			return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
+			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
 				chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
 		}
 	}
 	// Pre-checks passed, start the full block imports
 	bc.wg.Add(1)
-	defer bc.wg.Done()
-
 	bc.chainmu.Lock()
-	defer bc.chainmu.Unlock()
+	n, events, logs, err := bc.insertChain(chain, true)
+	bc.chainmu.Unlock()
+	bc.wg.Done()
+
+	bc.PostChainEvents(events, logs)
+	return n, err
+}
+
+// insertChain is the internal implementation of insertChain, which assumes that
+// 1) chains are contiguous, and 2) The chain mutex is held.
+//
+// This method is split out so that import batches that require re-injecting
+// historical blocks can do so without releasing the lock, which could lead to
+// racey behaviour. If a sidechain import is in progress, and the historic state
+// is imported, but then new canon-head is added before the actual sidechain
+// completes, then the historic state could be pruned again
+func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) {
+	// If the chain is terminating, don't even bother starting u
+	if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+		return 0, nil, nil, nil
+	}
+	// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
+	senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
 
 	// A queued approach to delivering events. This is generally
 	// faster than direct delivery and requires much less mutex
@@ -1073,16 +1119,56 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 
 	for i, block := range chain {
 		headers[i] = block.Header()
-		seals[i] = true
+		seals[i] = verifySeals
 	}
 	abort, results := bc.engine.VerifyHeaders(bc, headers, seals)
 	defer close(abort)
 
-	// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
+	// Peek the error for the first block to decide the directing import logic
-	senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
+	it := newInsertIterator(chain, results, bc.Validator())
 
-	// Iterate over the blocks and insert when the verifier permits
+	block, err := it.next()
-	for i, block := range chain {
+	switch {
+	// First block is pruned, insert as sidechain and reorg only if TD grows enough
+	case err == consensus.ErrPrunedAncestor:
+		return bc.insertSidechain(it)
+
+	// First block is future, shove it (and all children) to the future queue (unknown ancestor)
+	case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
+		for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) {
+			if err := bc.addFutureBlock(block); err != nil {
+				return it.index, events, coalescedLogs, err
+			}
+			block, err = it.next()
+		}
+		stats.queued += it.processed()
+		stats.ignored += it.remaining()
+
+		// If there are any still remaining, mark as ignored
+		return it.index, events, coalescedLogs, err
+
+	// First block (and state) is known
+	//   1. We did a roll-back, and should now do a re-import
+	//   2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
+	// 	    from the canonical chain, which has not been verified.
+	case err == ErrKnownBlock:
+		// Skip all known blocks that behind us
+		current := bc.CurrentBlock().NumberU64()
+
+		for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
+			stats.ignored++
+			block, err = it.next()
+		}
+		// Falls through to the block import
+
+	// Some other error occurred, abort
+	case err != nil:
+		stats.ignored += len(it.chain)
+		bc.reportBlock(block, nil, err)
+		return it.index, events, coalescedLogs, err
+	}
+	// No validation errors for the first block (or chain prefix skipped)
+	for ; block != nil && err == nil; block, err = it.next() {
 		// If the chain is terminating, stop processing blocks
 		if atomic.LoadInt32(&bc.procInterrupt) == 1 {
 			log.Debug("Premature abort during blocks processing")
@@ -1091,115 +1177,53 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		// If the header is a banned one, straight out abort
 		if BadHashes[block.Hash()] {
 			bc.reportBlock(block, nil, ErrBlacklistedHash)
-			return i, events, coalescedLogs, ErrBlacklistedHash
+			return it.index, events, coalescedLogs, ErrBlacklistedHash
 		}
-		// Wait for the block's verification to complete
+		// Retrieve the parent block and it's state to execute on top
-		bstart := time.Now()
+		start := time.Now()
 
-		err := <-results
+		parent := it.previous()
-		if err == nil {
+		if parent == nil {
-			err = bc.Validator().ValidateBody(block)
-		}
-		switch {
-		case err == ErrKnownBlock:
-			// Block and state both already known. However if the current block is below
-			// this number we did a rollback and we should reimport it nonetheless.
-			if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
-				stats.ignored++
-				continue
-			}
-
-		case err == consensus.ErrFutureBlock:
-			// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
-			// the chain is discarded and processed at a later time if given.
-			max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
-			if block.Time().Cmp(max) > 0 {
-				return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
-			}
-			bc.futureBlocks.Add(block.Hash(), block)
-			stats.queued++
-			continue
-
-		case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
-			bc.futureBlocks.Add(block.Hash(), block)
-			stats.queued++
-			continue
-
-		case err == consensus.ErrPrunedAncestor:
-			// Block competing with the canonical chain, store in the db, but don't process
-			// until the competitor TD goes above the canonical TD
-			currentBlock := bc.CurrentBlock()
-			localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
-			externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
-			if localTd.Cmp(externTd) > 0 {
-				if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
-					return i, events, coalescedLogs, err
-				}
-				continue
-			}
-			// Competitor chain beat canonical, gather all blocks from the common ancestor
-			var winner []*types.Block
-
-			parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-			for !bc.HasState(parent.Root()) {
-				winner = append(winner, parent)
-				parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
-			}
-			for j := 0; j < len(winner)/2; j++ {
-				winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
-			}
-			// Import all the pruned blocks to make the state available
-			bc.chainmu.Unlock()
-			_, evs, logs, err := bc.insertChain(winner)
-			bc.chainmu.Lock()
-			events, coalescedLogs = evs, logs
-
-			if err != nil {
-				return i, events, coalescedLogs, err
-			}
-
-		case err != nil:
-			bc.reportBlock(block, nil, err)
-			return i, events, coalescedLogs, err
-		}
-		// Create a new statedb using the parent block and report an
-		// error if it fails.
-		var parent *types.Block
-		if i == 0 {
 			parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-		} else {
-			parent = chain[i-1]
 		}
 		state, err := state.New(parent.Root(), bc.stateCache)
 		if err != nil {
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		// Process block using the parent state as reference point.
+		t0 := time.Now()
 		receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig)
+		t1 := time.Now()
 		if err != nil {
 			bc.reportBlock(block, receipts, err)
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		// Validate the state using the default validator
-		err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas)
+		if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil {
-		if err != nil {
 			bc.reportBlock(block, receipts, err)
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
-		proctime := time.Since(bstart)
+		t2 := time.Now()
+		proctime := time.Since(start)
 
 		// Write the block to the chain and get the status.
 		status, err := bc.WriteBlockWithState(block, receipts, state)
+		t3 := time.Now()
 		if err != nil {
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
+		blockInsertTimer.UpdateSince(start)
+		blockExecutionTimer.Update(t1.Sub(t0))
+		blockValidationTimer.Update(t2.Sub(t1))
+		blockWriteTimer.Update(t3.Sub(t2))
 		switch status {
 		case CanonStatTy:
-			log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()),
+			log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(),
-				"txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart)))
+				"uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(),
+				"elapsed", common.PrettyDuration(time.Since(start)),
+				"root", block.Root())
 
 			coalescedLogs = append(coalescedLogs, logs...)
-			blockInsertTimer.UpdateSince(bstart)
 			events = append(events, ChainEvent{block, block.Hash(), logs})
 			lastCanon = block
 
@@ -1207,78 +1231,153 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			bc.gcproc += proctime
 
 		case SideStatTy:
-			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
+			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(),
-				common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
+				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
-
+				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
-			blockInsertTimer.UpdateSince(bstart)
+				"root", block.Root())
 			events = append(events, ChainSideEvent{block})
 		}
+		blockInsertTimer.UpdateSince(start)
 		stats.processed++
 		stats.usedGas += usedGas
 
 		cache, _ := bc.stateCache.TrieDB().Size()
-		stats.report(chain, i, cache)
+		stats.report(chain, it.index, cache)
 	}
+	// Any blocks remaining here? The only ones we care about are the future ones
+	if block != nil && err == consensus.ErrFutureBlock {
+		if err := bc.addFutureBlock(block); err != nil {
+			return it.index, events, coalescedLogs, err
+		}
+		block, err = it.next()
+
+		for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() {
+			if err := bc.addFutureBlock(block); err != nil {
+				return it.index, events, coalescedLogs, err
+			}
+			stats.queued++
+		}
+	}
+	stats.ignored += it.remaining()
+
 	// Append a single chain head event if we've progressed the chain
 	if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
 		events = append(events, ChainHeadEvent{lastCanon})
 	}
-	return 0, events, coalescedLogs, nil
+	return it.index, events, coalescedLogs, err
 }
 
-// insertStats tracks and reports on block insertion.
+// insertSidechain is called when an import batch hits upon a pruned ancestor
-type insertStats struct {
+// error, which happens when a sidechain with a sufficiently old fork-block is
-	queued, processed, ignored int
+// found.
-	usedGas                    uint64
+//
-	lastIndex                  int
+// The method writes all (header-and-body-valid) blocks to disk, then tries to
-	startTime                  mclock.AbsTime
+// switch over to the new chain if the TD exceeded the current chain.
-}
+func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
-
-// statsReportLimit is the time limit during import and export after which we
-// always print out progress. This avoids the user wondering what's going on.
-const statsReportLimit = 8 * time.Second
-
-// report prints statistics if some number of blocks have been processed
-// or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
-	// Fetch the timings for the batch
 	var (
-		now     = mclock.Now()
+		externTd *big.Int
-		elapsed = time.Duration(now) - time.Duration(st.startTime)
+		current  = bc.CurrentBlock().NumberU64()
 	)
-	// If we're at the last block of the batch or report period reached, log
+	// The first sidechain block error is already verified to be ErrPrunedAncestor.
-	if index == len(chain)-1 || elapsed >= statsReportLimit {
+	// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
+	// ones. Any other errors means that the block is invalid, and should not be written
+	// to disk.
+	block, err := it.current(), consensus.ErrPrunedAncestor
+	for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
+		// Check the canonical state root for that number
+		if number := block.NumberU64(); current >= number {
+			if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
+				// This is most likely a shadow-state attack. When a fork is imported into the
+				// database, and it eventually reaches a block height which is not pruned, we
+				// just found that the state already exist! This means that the sidechain block
+				// refers to a state which already exists in our canon chain.
+				//
+				// If left unchecked, we would now proceed importing the blocks, without actually
+				// having verified the state of the previous blocks.
+				log.Warn("Sidechain ghost-state attack detected", "number", block.NumberU64(), "sideroot", block.Root(), "canonroot", canonical.Root())
+
+				// If someone legitimately side-mines blocks, they would still be imported as usual. However,
+				// we cannot risk writing unverified blocks to disk when they obviously target the pruning
+				// mechanism.
+				return it.index, nil, nil, errors.New("sidechain ghost-state attack")
+			}
+		}
+		if externTd == nil {
+			externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1)
+		}
+		externTd = new(big.Int).Add(externTd, block.Difficulty())
+
+		if !bc.HasBlock(block.Hash(), block.NumberU64()) {
+			start := time.Now()
+			if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
+				return it.index, nil, nil, err
+			}
+			log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
+				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
+				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
+				"root", block.Root())
+		}
+	}
+	// At this point, we've written all sidechain blocks to database. Loop ended
+	// either on some other error or all were processed. If there was some other
+	// error, we can ignore the rest of those blocks.
+	//
+	// If the externTd was larger than our local TD, we now need to reimport the previous
+	// blocks to regenerate the required state
+	localTd := bc.GetTd(bc.CurrentBlock().Hash(), current)
+	if localTd.Cmp(externTd) > 0 {
+		log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd)
+		return it.index, nil, nil, err
+	}
+	// Gather all the sidechain hashes (full blocks may be memory heavy)
 	var (
-			end = chain[index]
+		hashes  []common.Hash
-			txs = countTransactions(chain[st.lastIndex : index+1])
+		numbers []uint64
 	)
-		context := []interface{}{
+	parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64())
-			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
+	for parent != nil && !bc.HasState(parent.Root) {
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+		hashes = append(hashes, parent.Hash())
-			"number", end.Number(), "hash", end.Hash(),
+		numbers = append(numbers, parent.Number.Uint64())
-		}
-		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
-			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
-		}
-		context = append(context, []interface{}{"cache", cache}...)
 
-		if st.queued > 0 {
+		parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
-			context = append(context, []interface{}{"queued", st.queued}...)
 	}
-		if st.ignored > 0 {
+	if parent == nil {
-			context = append(context, []interface{}{"ignored", st.ignored}...)
+		return it.index, nil, nil, errors.New("missing parent")
 	}
-		log.Info("Imported new chain segment", context...)
+	// Import all the pruned blocks to make the state available
+	var (
+		blocks []*types.Block
+		memory common.StorageSize
+	)
+	for i := len(hashes) - 1; i >= 0; i-- {
+		// Append the next block to our batch
+		block := bc.GetBlock(hashes[i], numbers[i])
 
-		*st = insertStats{startTime: now, lastIndex: index + 1}
+		blocks = append(blocks, block)
-	}
+		memory += block.Size()
-}
 
-func countTransactions(chain []*types.Block) (c int) {
+		// If memory use grew too large, import and continue. Sadly we need to discard
-	for _, b := range chain {
+		// all raised events and logs from notifications since we're too heavy on the
-		c += len(b.Transactions())
+		// memory here.
+		if len(blocks) >= 2048 || memory > 64*1024*1024 {
+			log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64())
+			if _, _, _, err := bc.insertChain(blocks, false); err != nil {
+				return 0, nil, nil, err
 			}
-	return c
+			blocks, memory = blocks[:0], 0
+
+			// If the chain is terminating, stop processing blocks
+			if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+				log.Debug("Premature abort during blocks processing")
+				return 0, nil, nil, nil
+			}
+		}
+	}
+	if len(blocks) > 0 {
+		log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64())
+		return bc.insertChain(blocks, false)
+	}
+	return 0, nil, nil, nil
 }
 
 // reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them
@@ -1453,8 +1552,10 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e
 	bc.addBadBlock(block)
 
 	var receiptString string
-	for _, receipt := range receipts {
+	for i, receipt := range receipts {
-		receiptString += fmt.Sprintf("\t%v\n", receipt)
+		receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n",
+			i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(),
+			receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState)
 	}
 	log.Error(fmt.Sprintf(`
 ########## BAD BLOCK #########

core/blockchain_insert.go

@@ -0,0 +1,143 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// insertStats tracks and reports on block insertion.
+type insertStats struct {
+	queued, processed, ignored int
+	usedGas                    uint64
+	lastIndex                  int
+	startTime                  mclock.AbsTime
+}
+
+// statsReportLimit is the time limit during import and export after which we
+// always print out progress. This avoids the user wondering what's going on.
+const statsReportLimit = 8 * time.Second
+
+// report prints statistics if some number of blocks have been processed
+// or more than a few seconds have passed since the last message.
+func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
+	// Fetch the timings for the batch
+	var (
+		now     = mclock.Now()
+		elapsed = time.Duration(now) - time.Duration(st.startTime)
+	)
+	// If we're at the last block of the batch or report period reached, log
+	if index == len(chain)-1 || elapsed >= statsReportLimit {
+		// Count the number of transactions in this segment
+		var txs int
+		for _, block := range chain[st.lastIndex : index+1] {
+			txs += len(block.Transactions())
+		}
+		end := chain[index]
+
+		// Assemble the log context and send it to the logger
+		context := []interface{}{
+			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"number", end.Number(), "hash", end.Hash(),
+		}
+		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
+			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
+		}
+		context = append(context, []interface{}{"cache", cache}...)
+
+		if st.queued > 0 {
+			context = append(context, []interface{}{"queued", st.queued}...)
+		}
+		if st.ignored > 0 {
+			context = append(context, []interface{}{"ignored", st.ignored}...)
+		}
+		log.Info("Imported new chain segment", context...)
+
+		// Bump the stats reported to the next section
+		*st = insertStats{startTime: now, lastIndex: index + 1}
+	}
+}
+
+// insertIterator is a helper to assist during chain import.
+type insertIterator struct {
+	chain     types.Blocks
+	results   <-chan error
+	index     int
+	validator Validator
+}
+
+// newInsertIterator creates a new iterator based on the given blocks, which are
+// assumed to be a contiguous chain.
+func newInsertIterator(chain types.Blocks, results <-chan error, validator Validator) *insertIterator {
+	return &insertIterator{
+		chain:     chain,
+		results:   results,
+		index:     -1,
+		validator: validator,
+	}
+}
+
+// next returns the next block in the iterator, along with any potential validation
+// error for that block. When the end is reached, it will return (nil, nil).
+func (it *insertIterator) next() (*types.Block, error) {
+	if it.index+1 >= len(it.chain) {
+		it.index = len(it.chain)
+		return nil, nil
+	}
+	it.index++
+	if err := <-it.results; err != nil {
+		return it.chain[it.index], err
+	}
+	return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
+}
+
+// current returns the current block that's being processed.
+func (it *insertIterator) current() *types.Block {
+	if it.index < 0 || it.index+1 >= len(it.chain) {
+		return nil
+	}
+	return it.chain[it.index]
+}
+
+// previous returns the previous block was being processed, or nil
+func (it *insertIterator) previous() *types.Block {
+	if it.index < 1 {
+		return nil
+	}
+	return it.chain[it.index-1]
+}
+
+// first returns the first block in the it.
+func (it *insertIterator) first() *types.Block {
+	return it.chain[0]
+}
+
+// remaining returns the number of remaining blocks.
+func (it *insertIterator) remaining() int {
+	return len(it.chain) - it.index
+}
+
+// processed returns the number of processed blocks.
+func (it *insertIterator) processed() int {
+	return it.index + 1
+}

core/blockchain_test.go

@@ -579,11 +579,11 @@ func testInsertNonceError(t *testing.T, full bool) {
 			blockchain.hc.engine = blockchain.engine
 			failRes, err = blockchain.InsertHeaderChain(headers, 1)
 		}
-		// Check that the returned error indicates the failure.
+		// Check that the returned error indicates the failure
 		if failRes != failAt {
-			t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
+			t.Errorf("test %d: failure (%v) index mismatch: have %d, want %d", i, err, failRes, failAt)
 		}
-		// Check that all no blocks after the failing block have been inserted.
+		// Check that all blocks after the failing block have been inserted
 		for j := 0; j < i-failAt; j++ {
 			if full {
 				if block := blockchain.GetBlockByNumber(failNum + uint64(j)); block != nil {
@@ -1345,7 +1345,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		t.Fatalf("failed to insert shared chain: %v", err)
 	}
 	if _, err := chain.InsertChain(original); err != nil {
-		t.Fatalf("failed to insert shared chain: %v", err)
+		t.Fatalf("failed to insert original chain: %v", err)
 	}
 	// Ensure that the state associated with the forking point is pruned away
 	if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {

core/rawdb/accessors_chain.go

@@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
 	}
 }
 
+// HasReceipts verifies the existence of all the transaction receipts belonging
+// to a block.
+func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
+	if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
+		return false
+	}
+	return true
+}
+
 // ReadReceipts retrieves all the transaction receipts belonging to a block.
 func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
 	// Retrieve the flattened receipt slice

core/state/database.go

@@ -72,13 +72,19 @@ type Trie interface {
 }
 
 // NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains cached trie nodes in memory. The pool is an optional
+// concurrent use and retains a few recent expanded trie nodes in memory. To keep
-// intermediate trie-node memory pool between the low level storage layer and the
+// more historical state in memory, use the NewDatabaseWithCache constructor.
-// high level trie abstraction.
 func NewDatabase(db ethdb.Database) Database {
+	return NewDatabaseWithCache(db, 0)
+}
+
+// NewDatabase creates a backing store for state. The returned database is safe for
+// concurrent use and retains both a few recent expanded trie nodes in memory, as
+// well as a lot of collapsed RLP trie nodes in a large memory cache.
+func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	return &cachingDB{
-		db:            trie.NewDatabase(db),
+		db:            trie.NewDatabaseWithCache(db, cache),
 		codeSizeCache: csc,
 	}
 }

core/tx_pool.go

@@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
 // addTxsLocked attempts to queue a batch of transactions if they are valid,
 // whilst assuming the transaction pool lock is already held.
 func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
-	// Add the batch of transaction, tracking the accepted ones
+	// Add the batch of transactions, tracking the accepted ones
 	dirty := make(map[common.Address]struct{})
 	errs := make([]error, len(txs))
 

core/types/block.go

@@ -81,8 +81,8 @@ type Header struct {
 	GasUsed     uint64         `json:"gasUsed"          gencodec:"required"`
 	Time        *big.Int       `json:"timestamp"        gencodec:"required"`
 	Extra       []byte         `json:"extraData"        gencodec:"required"`
-	MixDigest   common.Hash    `json:"mixHash"          gencodec:"required"`
+	MixDigest   common.Hash    `json:"mixHash"`
-	Nonce       BlockNonce     `json:"nonce"            gencodec:"required"`
+	Nonce       BlockNonce     `json:"nonce"`
 }
 
 // field type overrides for gencodec

core/types/gen_header_json.go

@@ -13,6 +13,7 @@ import (
 
 var _ = (*headerMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (h Header) MarshalJSON() ([]byte, error) {
 	type Header struct {
 		ParentHash  common.Hash    `json:"parentHash"       gencodec:"required"`
@@ -28,8 +29,8 @@ func (h Header) MarshalJSON() ([]byte, error) {
 		GasUsed     hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
 		Time        *hexutil.Big   `json:"timestamp"        gencodec:"required"`
 		Extra       hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest   common.Hash    `json:"mixHash"          gencodec:"required"`
+		MixDigest   common.Hash    `json:"mixHash"`
-		Nonce       BlockNonce     `json:"nonce"            gencodec:"required"`
+		Nonce       BlockNonce     `json:"nonce"`
 		Hash        common.Hash    `json:"hash"`
 	}
 	var enc Header
@@ -52,6 +53,7 @@ func (h Header) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (h *Header) UnmarshalJSON(input []byte) error {
 	type Header struct {
 		ParentHash  *common.Hash    `json:"parentHash"       gencodec:"required"`
@@ -67,8 +69,8 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 		GasUsed     *hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
 		Time        *hexutil.Big    `json:"timestamp"        gencodec:"required"`
 		Extra       *hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest   *common.Hash    `json:"mixHash"          gencodec:"required"`
+		MixDigest   *common.Hash    `json:"mixHash"`
-		Nonce       *BlockNonce     `json:"nonce"            gencodec:"required"`
+		Nonce       *BlockNonce     `json:"nonce"`
 	}
 	var dec Header
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -126,13 +128,11 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 		return errors.New("missing required field 'extraData' for Header")
 	}
 	h.Extra = *dec.Extra
-	if dec.MixDigest == nil {
+	if dec.MixDigest != nil {
-		return errors.New("missing required field 'mixHash' for Header")
-	}
 		h.MixDigest = *dec.MixDigest
-	if dec.Nonce == nil {
-		return errors.New("missing required field 'nonce' for Header")
 	}
+	if dec.Nonce != nil {
 		h.Nonce = *dec.Nonce
+	}
 	return nil
 }

core/vm/evm.go

@@ -339,6 +339,12 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 	contract := NewContract(caller, to, new(big.Int), gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
+	// We do an AddBalance of zero here, just in order to trigger a touch.
+	// This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium,
+	// but is the correct thing to do and matters on other networks, in tests, and potential
+	// future scenarios
+	evm.StateDB.AddBalance(addr, bigZero)
+
 	// When an error was returned by the EVM or when setting the creation code
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in Homestead this also counts for code storage gas errors.

eth/api.go

@@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
 	if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
 		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
 	}
+	triedb := api.eth.BlockChain().StateCache().TrieDB()
 
-	oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+	oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0)
 	if err != nil {
 		return nil, err
 	}
-	newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+	newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0)
 	if err != nil {
 		return nil, err
 	}
-
 	diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}))
 	iter := trie.NewIterator(diff)
 

eth/api_tracer.go

@@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 
 	// Ensure we have a valid starting state before doing any work
 	origin := start.NumberU64()
-	database := state.NewDatabase(api.eth.ChainDb())
+	database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis
 
 	if number := start.NumberU64(); number > 0 {
 		start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
@@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 	}
 	// Otherwise try to reexec blocks until we find a state or reach our limit
 	origin := block.NumberU64()
-	database := state.NewDatabase(api.eth.ChainDb())
+	database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16)
 
 	for i := uint64(0); i < reexec; i++ {
 		block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)

eth/backend.go

@@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 			EWASMInterpreter:        config.EWASMInterpreter,
 			EVMInterpreter:          config.EVMInterpreter,
 		}
-		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
+		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout}
 	)
 	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
 	if err != nil {

eth/config.go

@@ -45,8 +45,9 @@ var DefaultConfig = Config{
 	},
 	NetworkId:      1,
 	LightPeers:     100,
-	DatabaseCache: 768,
+	DatabaseCache:  512,
-	TrieCache:     256,
+	TrieCleanCache: 256,
+	TrieDirtyCache: 256,
 	TrieTimeout:    60 * time.Minute,
 	MinerGasFloor:  8000000,
 	MinerGasCeil:   8000000,
@@ -98,7 +99,8 @@ type Config struct {
 	SkipBcVersionCheck bool `toml:"-"`
 	DatabaseHandles    int  `toml:"-"`
 	DatabaseCache      int
-	TrieCache          int
+	TrieCleanCache     int
+	TrieDirtyCache     int
 	TrieTimeout        time.Duration
 
 	// Mining-related options

eth/downloader/downloader.go

@@ -99,6 +99,7 @@ type Downloader struct {
 	mode SyncMode       // Synchronisation mode defining the strategy used (per sync cycle)
 	mux  *event.TypeMux // Event multiplexer to announce sync operation events
 
+	genesis uint64   // Genesis block number to limit sync to (e.g. light client CHT)
 	queue   *queue   // Scheduler for selecting the hashes to download
 	peers   *peerSet // Set of active peers from which download can proceed
 	stateDB ethdb.Database
@@ -181,6 +182,9 @@ type BlockChain interface {
 	// HasBlock verifies a block's presence in the local chain.
 	HasBlock(common.Hash, uint64) bool
 
+	// HasFastBlock verifies a fast block's presence in the local chain.
+	HasFastBlock(common.Hash, uint64) bool
+
 	// GetBlockByHash retrieves a block from the local chain.
 	GetBlockByHash(common.Hash) *types.Block
 
@@ -430,7 +434,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
 	}
 	height := latest.Number.Uint64()
 
-	origin, err := d.findAncestor(p, height)
+	origin, err := d.findAncestor(p, latest)
 	if err != nil {
 		return err
 	}
@@ -587,41 +591,107 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) {
 	}
 }
 
+// calculateRequestSpan calculates what headers to request from a peer when trying to determine the
+// common ancestor.
+// It returns parameters to be used for peer.RequestHeadersByNumber:
+//  from - starting block number
+//  count - number of headers to request
+//  skip - number of headers to skip
+// and also returns 'max', the last block which is expected to be returned by the remote peers,
+// given the (from,count,skip)
+func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) {
+	var (
+		from     int
+		count    int
+		MaxCount = MaxHeaderFetch / 16
+	)
+	// requestHead is the highest block that we will ask for. If requestHead is not offset,
+	// the highest block that we will get is 16 blocks back from head, which means we
+	// will fetch 14 or 15 blocks unnecessarily in the case the height difference
+	// between us and the peer is 1-2 blocks, which is most common
+	requestHead := int(remoteHeight) - 1
+	if requestHead < 0 {
+		requestHead = 0
+	}
+	// requestBottom is the lowest block we want included in the query
+	// Ideally, we want to include just below own head
+	requestBottom := int(localHeight - 1)
+	if requestBottom < 0 {
+		requestBottom = 0
+	}
+	totalSpan := requestHead - requestBottom
+	span := 1 + totalSpan/MaxCount
+	if span < 2 {
+		span = 2
+	}
+	if span > 16 {
+		span = 16
+	}
+
+	count = 1 + totalSpan/span
+	if count > MaxCount {
+		count = MaxCount
+	}
+	if count < 2 {
+		count = 2
+	}
+	from = requestHead - (count-1)*span
+	if from < 0 {
+		from = 0
+	}
+	max := from + (count-1)*span
+	return int64(from), count, span - 1, uint64(max)
+}
+
 // findAncestor tries to locate the common ancestor link of the local chain and
 // a remote peers blockchain. In the general case when our node was in sync and
 // on the correct chain, checking the top N links should already get us a match.
 // In the rare scenario when we ended up on a long reorganisation (i.e. none of
 // the head links match), we do a binary search to find the common ancestor.
-func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) {
+func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) {
 	// Figure out the valid ancestor range to prevent rewrite attacks
-	floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
+	var (
+		floor        = int64(-1)
+		localHeight  uint64
+		remoteHeight = remoteHeader.Number.Uint64()
+	)
+	switch d.mode {
+	case FullSync:
+		localHeight = d.blockchain.CurrentBlock().NumberU64()
+	case FastSync:
+		localHeight = d.blockchain.CurrentFastBlock().NumberU64()
+	default:
+		localHeight = d.lightchain.CurrentHeader().Number.Uint64()
+	}
+	p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight)
+	if localHeight >= MaxForkAncestry {
+		// We're above the max reorg threshold, find the earliest fork point
+		floor = int64(localHeight - MaxForkAncestry)
 
-	if d.mode == FullSync {
+		// If we're doing a light sync, ensure the floor doesn't go below the CHT, as
-		ceil = d.blockchain.CurrentBlock().NumberU64()
+		// all headers before that point will be missing.
-	} else if d.mode == FastSync {
+		if d.mode == LightSync {
-		ceil = d.blockchain.CurrentFastBlock().NumberU64()
+			// If we dont know the current CHT position, find it
+			if d.genesis == 0 {
+				header := d.lightchain.CurrentHeader()
+				for header != nil {
+					d.genesis = header.Number.Uint64()
+					if floor >= int64(d.genesis)-1 {
+						break
 					}
-	if ceil >= MaxForkAncestry {
+					header = d.lightchain.GetHeaderByHash(header.ParentHash)
-		floor = int64(ceil - MaxForkAncestry)
 				}
-	p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
+			}
+			// We already know the "genesis" block number, cap floor to that
+			if floor < int64(d.genesis)-1 {
+				floor = int64(d.genesis) - 1
+			}
+		}
+	}
+	from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight)
 
-	// Request the topmost blocks to short circuit binary ancestor lookup
+	p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip)
-	head := ceil
+	go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false)
-	if head > height {
-		head = height
-	}
-	from := int64(head) - int64(MaxHeaderFetch)
-	if from < 0 {
-		from = 0
-	}
-	// Span out with 15 block gaps into the future to catch bad head reports
-	limit := 2 * MaxHeaderFetch / 16
-	count := 1 + int((int64(ceil)-from)/16)
-	if count > limit {
-		count = limit
-	}
-	go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false)
 
 	// Wait for the remote response to the head fetch
 	number, hash := uint64(0), common.Hash{}
@@ -647,9 +717,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				return 0, errEmptyHeaderSet
 			}
 			// Make sure the peer's reply conforms to the request
-			for i := 0; i < len(headers); i++ {
+			for i, header := range headers {
-				if number := headers[i].Number.Int64(); number != from+int64(i)*16 {
+				expectNumber := from + int64(i)*int64((skip+1))
-					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number)
+				if number := header.Number.Int64(); number != expectNumber {
+					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
 					return 0, errInvalidChain
 				}
 			}
@@ -657,20 +728,24 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 			finished = true
 			for i := len(headers) - 1; i >= 0; i-- {
 				// Skip any headers that underflow/overflow our requested set
-				if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil {
+				if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max {
 					continue
 				}
 				// Otherwise check if we already know the header or not
 				h := headers[i].Hash()
 				n := headers[i].Number.Uint64()
-				if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) {
-					number, hash = n, h
 
-					// If every header is known, even future ones, the peer straight out lied about its head
+				var known bool
-					if number > height && i == limit-1 {
+				switch d.mode {
-						p.log.Warn("Lied about chain head", "reported", height, "found", number)
+				case FullSync:
-						return 0, errStallingPeer
+					known = d.blockchain.HasBlock(h, n)
+				case FastSync:
+					known = d.blockchain.HasFastBlock(h, n)
+				default:
+					known = d.lightchain.HasHeader(h, n)
 				}
+				if known {
+					number, hash = n, h
 					break
 				}
 			}
@@ -694,10 +769,12 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 		return number, nil
 	}
 	// Ancestor not found, we need to binary search over our chain
-	start, end := uint64(0), head
+	start, end := uint64(0), remoteHeight
 	if floor > 0 {
 		start = uint64(floor)
 	}
+	p.log.Trace("Binary searching for common ancestor", "start", start, "end", end)
+
 	for start+1 < end {
 		// Split our chain interval in two, and request the hash to cross check
 		check := (start + end) / 2
@@ -730,7 +807,17 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				// Modify the search interval based on the response
 				h := headers[0].Hash()
 				n := headers[0].Number.Uint64()
-				if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) {
+
+				var known bool
+				switch d.mode {
+				case FullSync:
+					known = d.blockchain.HasBlock(h, n)
+				case FastSync:
+					known = d.blockchain.HasFastBlock(h, n)
+				default:
+					known = d.lightchain.HasHeader(h, n)
+				}
+				if !known {
 					end = check
 					break
 				}

eth/downloader/downloader_test.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -114,6 +115,15 @@ func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool {
 	return dl.GetBlockByHash(hash) != nil
 }
 
+// HasFastBlock checks if a block is present in the testers canonical chain.
+func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool {
+	dl.lock.RLock()
+	defer dl.lock.RUnlock()
+
+	_, ok := dl.ownReceipts[hash]
+	return ok
+}
+
 // GetHeader retrieves a header from the testers canonical chain.
 func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header {
 	dl.lock.RLock()
@@ -234,6 +244,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) {
 			dl.ownHeaders[block.Hash()] = block.Header()
 		}
 		dl.ownBlocks[block.Hash()] = block
+		dl.ownReceipts[block.Hash()] = make(types.Receipts, 0)
 		dl.stateDb.Put(block.Root().Bytes(), []byte{0x00})
 		dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty())
 	}
@@ -374,28 +385,28 @@ func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error {
 // assertOwnChain checks if the local chain contains the correct number of items
 // of the various chain components.
 func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
+	// Mark this method as a helper to report errors at callsite, not in here
+	t.Helper()
+
 	assertOwnForkedChain(t, tester, 1, []int{length})
 }
 
 // assertOwnForkedChain checks if the local forked chain contains the correct
 // number of items of the various chain components.
 func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
-	// Initialize the counters for the first fork
+	// Mark this method as a helper to report errors at callsite, not in here
-	headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks
+	t.Helper()
+
+	// Initialize the counters for the first fork
+	headers, blocks, receipts := lengths[0], lengths[0], lengths[0]
 
-	if receipts < 0 {
-		receipts = 1
-	}
 	// Update the counters for each subsequent fork
 	for _, length := range lengths[1:] {
 		headers += length - common
 		blocks += length - common
-		receipts += length - common - fsMinFullBlocks
+		receipts += length - common
 	}
-	switch tester.downloader.mode {
+	if tester.downloader.mode == LightSync {
-	case FullSync:
-		receipts = 1
-	case LightSync:
 		blocks, receipts = 1, 1
 	}
 	if hs := len(tester.ownHeaders); hs != headers {
@@ -1149,7 +1160,9 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 }
 
 func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) {
+	// Mark this method as a helper to report errors at callsite, not in here
 	t.Helper()
+
 	p := d.Progress()
 	p.KnownStates, p.PulledStates = 0, 0
 	want.KnownStates, want.PulledStates = 0, 0
@@ -1479,3 +1492,78 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int
 	}
 	return nil
 }
+
+func TestRemoteHeaderRequestSpan(t *testing.T) {
+	testCases := []struct {
+		remoteHeight uint64
+		localHeight  uint64
+		expected     []int
+	}{
+		// Remote is way higher. We should ask for the remote head and go backwards
+		{1500, 1000,
+			[]int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499},
+		},
+		{15000, 13006,
+			[]int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999},
+		},
+		//Remote is pretty close to us. We don't have to fetch as many
+		{1200, 1150,
+			[]int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199},
+		},
+		// Remote is equal to us (so on a fork with higher td)
+		// We should get the closest couple of ancestors
+		{1500, 1500,
+			[]int{1497, 1499},
+		},
+		// We're higher than the remote! Odd
+		{1000, 1500,
+			[]int{997, 999},
+		},
+		// Check some weird edgecases that it behaves somewhat rationally
+		{0, 1500,
+			[]int{0, 2},
+		},
+		{6000000, 0,
+			[]int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999},
+		},
+		{0, 0,
+			[]int{0, 2},
+		},
+	}
+	reqs := func(from, count, span int) []int {
+		var r []int
+		num := from
+		for len(r) < count {
+			r = append(r, num)
+			num += span + 1
+		}
+		return r
+	}
+	for i, tt := range testCases {
+		from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight)
+		data := reqs(int(from), count, span)
+
+		if max != uint64(data[len(data)-1]) {
+			t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max)
+		}
+		failed := false
+		if len(data) != len(tt.expected) {
+			failed = true
+			t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data))
+		} else {
+			for j, n := range data {
+				if n != tt.expected[j] {
+					failed = true
+					break
+				}
+			}
+		}
+		if failed {
+			res := strings.Replace(fmt.Sprint(data), " ", ",", -1)
+			exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1)
+			fmt.Printf("got: %v\n", res)
+			fmt.Printf("exp: %v\n", exp)
+			t.Errorf("test %d: wrong values", i)
+		}
+	}
+}

eth/gen_config.go

@@ -30,7 +30,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		SkipBcVersionCheck      bool       `toml:"-"`
 		DatabaseHandles         int        `toml:"-"`
 		DatabaseCache           int
-		TrieCache               int
+		TrieCleanCache          int
+		TrieDirtyCache          int
 		TrieTimeout             time.Duration
 		Etherbase               common.Address `toml:",omitempty"`
 		MinerNotify             []string       `toml:",omitempty"`
@@ -45,6 +46,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		GPO                     gasprice.Config
 		EnablePreimageRecording bool
 		DocRoot                 string `toml:"-"`
+		EWASMInterpreter        string
+		EVMInterpreter          string
 	}
 	var enc Config
 	enc.Genesis = c.Genesis
@@ -58,7 +61,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.SkipBcVersionCheck = c.SkipBcVersionCheck
 	enc.DatabaseHandles = c.DatabaseHandles
 	enc.DatabaseCache = c.DatabaseCache
-	enc.TrieCache = c.TrieCache
+	enc.TrieCleanCache = c.TrieCleanCache
+	enc.TrieDirtyCache = c.TrieDirtyCache
 	enc.TrieTimeout = c.TrieTimeout
 	enc.Etherbase = c.Etherbase
 	enc.MinerNotify = c.MinerNotify
@@ -74,6 +78,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 
 	enc.EnablePreimageRecording = c.EnablePreimageRecording
 	enc.DocRoot = c.DocRoot
+	enc.EWASMInterpreter = c.EWASMInterpreter
+	enc.EVMInterpreter = c.EVMInterpreter
 	return &enc, nil
 }
 
@@ -91,7 +97,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		SkipBcVersionCheck      *bool      `toml:"-"`
 		DatabaseHandles         *int       `toml:"-"`
 		DatabaseCache           *int
-		TrieCache               *int
+		TrieCleanCache          *int
+		TrieDirtyCache          *int
 		TrieTimeout             *time.Duration
 		Etherbase               *common.Address `toml:",omitempty"`
 		MinerNotify             []string        `toml:",omitempty"`
@@ -106,6 +113,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		GPO                     *gasprice.Config
 		EnablePreimageRecording *bool
 		DocRoot                 *string `toml:"-"`
+		EWASMInterpreter        *string
+		EVMInterpreter          *string
 	}
 	var dec Config
 	if err := unmarshal(&dec); err != nil {
@@ -144,8 +153,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.DatabaseCache != nil {
 		c.DatabaseCache = *dec.DatabaseCache
 	}
-	if dec.TrieCache != nil {
+	if dec.TrieCleanCache != nil {
-		c.TrieCache = *dec.TrieCache
+		c.TrieCleanCache = *dec.TrieCleanCache
+	}
+	if dec.TrieDirtyCache != nil {
+		c.TrieDirtyCache = *dec.TrieDirtyCache
 	}
 	if dec.TrieTimeout != nil {
 		c.TrieTimeout = *dec.TrieTimeout
@@ -189,5 +201,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.DocRoot != nil {
 		c.DocRoot = *dec.DocRoot
 	}
+	if dec.EWASMInterpreter != nil {
+		c.EWASMInterpreter = *dec.EWASMInterpreter
+	}
+	if dec.EVMInterpreter != nil {
+		c.EVMInterpreter = *dec.EVMInterpreter
+	}
 	return nil
 }

eth/handler.go

@@ -653,12 +653,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			trueHead = request.Block.ParentHash()
 			trueTD   = new(big.Int).Sub(request.TD, request.Block.Difficulty())
 		)
-		// Update the peers total difficulty if better than the previous
+		// Update the peer's total difficulty if better than the previous
 		if _, td := p.Head(); trueTD.Cmp(td) > 0 {
 			p.SetHead(trueHead, trueTD)
 
 			// Schedule a sync if above ours. Note, this will not fire a sync for a gap of
-			// a singe block (as the true TD is below the propagated block), however this
+			// a single block (as the true TD is below the propagated block), however this
 			// scenario should easily be covered by the fetcher.
 			currentBlock := pm.blockchain.CurrentBlock()
 			if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 {

eth/handler_test.go

@@ -585,7 +585,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
 			}
 		}(peer)
 	}
-	timeoutCh := time.NewTimer(time.Millisecond * 100).C
+	timeout := time.After(300 * time.Millisecond)
 	var receivedCount int
 outer:
 	for {
@@ -597,7 +597,7 @@ outer:
 			if receivedCount == totalPeers {
 				break outer
 			}
-		case <-timeoutCh:
+		case <-timeout:
 			break outer
 		}
 	}

internal/cmdtest/test_cmd.go

@@ -27,6 +27,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"text/template"
 	"time"
@@ -50,6 +51,8 @@ type TestCmd struct {
 	stdout *bufio.Reader
 	stdin  io.WriteCloser
 	stderr *testlogger
+	// Err will contain the process exit error or interrupt signal error
+	Err error
 }
 
 // Run exec's the current binary using name as argv[0] which will trigger the
@@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() {
 }
 
 func (tt *TestCmd) WaitExit() {
-	tt.cmd.Wait()
+	tt.Err = tt.cmd.Wait()
 }
 
 func (tt *TestCmd) Interrupt() {
-	tt.cmd.Process.Signal(os.Interrupt)
+	tt.Err = tt.cmd.Process.Signal(os.Interrupt)
+}
+
+// ExitStatus exposes the process' OS exit code
+// It will only return a valid value after the process has finished.
+func (tt *TestCmd) ExitStatus() int {
+	if tt.Err != nil {
+		exitErr := tt.Err.(*exec.ExitError)
+		if exitErr != nil {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+				return status.ExitStatus()
+			}
+		}
+	}
+	return 0
 }
 
 // StderrText returns any stderr output written so far.

internal/ethapi/api.go

@@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
 	return fetchKeystore(s.am).Lock(addr) == nil
 }
 
-// signTransactions sets defaults and signs the given transaction
+// signTransaction sets defaults and signs the given transaction
 // NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
 // and release it after the transaction has been submitted to the tx pool
 func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) {

les/fetcher.go

@@ -151,23 +151,22 @@ func (f *lightFetcher) syncLoop() {
 			var (
 				rq      *distReq
 				reqID   uint64
+				syncing bool
 			)
 
 			if !f.syncing && !(newAnnounce && s) {
-				rq, reqID = f.nextRequest()
+				rq, reqID, syncing = f.nextRequest()
 			}
-
-			syncing := f.syncing
 			f.lock.Unlock()
 
 			if rq != nil {
 				requesting = true
-				_, ok := <-f.pm.reqDist.queue(rq)
+				if _, ok := <-f.pm.reqDist.queue(rq); ok {
-				if !ok {
+					if syncing {
-					f.requestChn <- false
+						f.lock.Lock()
-				}
+						f.syncing = true
-
+						f.lock.Unlock()
-				if !syncing {
+					} else {
 						go func() {
 							time.Sleep(softRequestTimeout)
 							f.reqMu.Lock()
@@ -181,6 +180,9 @@ func (f *lightFetcher) syncLoop() {
 							f.requestChn <- false
 						}()
 					}
+				} else {
+					f.requestChn <- false
+				}
 			}
 		case reqID := <-f.timeoutChn:
 			f.reqMu.Lock()
@@ -219,6 +221,7 @@ func (f *lightFetcher) syncLoop() {
 			f.checkSyncedHeaders(p)
 			f.syncing = false
 			f.lock.Unlock()
+			f.requestChn <- false
 		}
 	}
 }
@@ -417,7 +420,7 @@ func (f *lightFetcher) requestedID(reqID uint64) bool {
 
 // nextRequest selects the peer and announced head to be requested next, amount
 // to be downloaded starting from the head backwards is also returned
-func (f *lightFetcher) nextRequest() (*distReq, uint64) {
+func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) {
 	var (
 		bestHash    common.Hash
 		bestAmount  uint64
@@ -427,19 +430,17 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
 	bestHash, bestAmount, bestTd, bestSyncing = f.findBestRequest()
 
 	if bestTd == f.maxConfirmedTd {
-		return nil, 0
+		return nil, 0, false
 	}
 
-	f.syncing = bestSyncing
-
 	var rq *distReq
 	reqID := genReqID()
-	if f.syncing {
+	if bestSyncing {
 		rq = f.newFetcherDistReqForSync(bestHash)
 	} else {
 		rq = f.newFetcherDistReq(bestHash, reqID, bestAmount)
 	}
-	return rq, reqID
+	return rq, reqID, bestSyncing
 }
 
 // findBestRequest finds the best head to request that has been announced by but not yet requested from a known peer.
@@ -496,6 +497,9 @@ func (f *lightFetcher) newFetcherDistReqForSync(bestHash common.Hash) *distReq {
 		},
 		canSend: func(dp distPeer) bool {
 			p := dp.(*peer)
+			f.lock.Lock()
+			defer f.lock.Unlock()
+
 			if p.isOnlyAnnounce {
 				return false
 			}
@@ -528,6 +532,9 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes
 		},
 		canSend: func(dp distPeer) bool {
 			p := dp.(*peer)
+			f.lock.Lock()
+			defer f.lock.Unlock()
+
 			if p.isOnlyAnnounce {
 				return false
 			}
@@ -541,7 +548,7 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes
 		},
 		request: func(dp distPeer) func() {
 			p := dp.(*peer)
-
+			f.lock.Lock()
 			fp := f.peers[p]
 			if fp != nil {
 				n := fp.nodeByHash[bestHash]
@@ -549,6 +556,7 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes
 					n.requested = true
 				}
 			}
+			f.lock.Unlock()
 
 			cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount))
 			p.fcServer.QueueRequest(reqID, cost)

les/flowcontrol/control.go

@@ -82,7 +82,6 @@ func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) {
 	time := mclock.Now()
 	peer.recalcBV(time)
 	peer.bufValue -= cost
-	peer.recalcBV(time)
 	rcValue, rcost := peer.cm.processed(peer.cmNode, time)
 	if rcValue < peer.params.BufLimit {
 		bv := peer.params.BufLimit - rcValue

les/serverpool.go

@@ -729,7 +729,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
 }
 
 func encodePubkey64(pub *ecdsa.PublicKey) []byte {
-	return crypto.FromECDSAPub(pub)[:1]
+	return crypto.FromECDSAPub(pub)[1:]
 }
 
 func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {

les/ulc_test.go

@@ -81,15 +81,7 @@ func TestULCReceiveAnnounce(t *testing.T) {
 		Td:     td.Add(td, big.NewInt(1)),
 	}
 	announce.sign(f.Key)
-
 	lPeer.SendAnnounce(announce)
-	time.Sleep(time.Millisecond)
-
-	l.PM.peers.lock.Lock()
-	if len(l.PM.peers.peers) == 0 {
-		t.Fatal("peer list after receiving message should not be empty")
-	}
-	l.PM.peers.lock.Unlock()
 }
 
 func TestULCShouldNotSyncWithTwoPeersOneHaveEmptyChain(t *testing.T) {

light/postprocess.go

@@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co
 		diskdb:      db,
 		odr:         odr,
 		trieTable:   trieTable,
-		triedb:      trie.NewDatabase(trieTable),
+		triedb:      trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
 		sectionSize: size,
 	}
 	return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht")
@@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
 		diskdb:     db,
 		odr:        odr,
 		trieTable:  trieTable,
-		triedb:     trie.NewDatabase(trieTable),
+		triedb:     trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
 		parentSize: parentSize,
 		size:       size,
 	}

light/trie.go

@@ -108,7 +108,7 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) {
 func (t *odrTrie) TryUpdate(key, value []byte) error {
 	key = crypto.Keccak256(key)
 	return t.do(key, func() error {
-		return t.trie.TryDelete(key)
+		return t.trie.TryUpdate(key, value)
 	})
 }
 

mobile/big.go

@@ -84,6 +84,13 @@ func (bi *BigInt) SetString(x string, base int) {
 // BigInts represents a slice of big ints.
 type BigInts struct{ bigints []*big.Int }
 
+// NewBigInts creates a slice of uninitialized big numbers.
+func NewBigInts(size int) *BigInts {
+	return &BigInts{
+		bigints: make([]*big.Int, size),
+	}
+}
+
 // Size returns the number of big ints in the slice.
 func (bi *BigInts) Size() int {
 	return len(bi.bigints)

node/node.go

@@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error {
 		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 			return err
 		}
-		n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace)
+		n.log.Debug("InProc registered", "namespace", api.Namespace)
 	}
 	n.inprocHandler = handler
 	return nil

p2p/discover/table.go

@@ -434,7 +434,7 @@ func (tab *Table) loadSeedNodes() {
 	for i := range seeds {
 		seed := seeds[i]
 		age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID())) }}
-		log.Debug("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
+		log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
 		tab.add(seed)
 	}
 }

p2p/discv5/net.go

@@ -567,12 +567,11 @@ loop:
 			net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte {
 				if n.state != nil && n.state.canQuery {
 					return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration
-				} else {
+				}
 				if n.state == unknown {
 					net.ping(n, n.addr())
 				}
 				return nil
-				}
 			})
 
 		case <-statsDump.C:

p2p/protocols/accounting.go

@@ -16,29 +16,32 @@
 
 package protocols
 
-import "github.com/ethereum/go-ethereum/metrics"
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/metrics"
+)
 
 //define some metrics
 var (
-	//NOTE: these metrics just define the interfaces and are currently *NOT persisted* over sessions
 	//All metrics are cumulative
 
 	//total amount of units credited
-	mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", nil)
+	mBalanceCredit metrics.Counter
 	//total amount of units debited
-	mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", nil)
+	mBalanceDebit metrics.Counter
 	//total amount of bytes credited
-	mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", nil)
+	mBytesCredit metrics.Counter
 	//total amount of bytes debited
-	mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", nil)
+	mBytesDebit metrics.Counter
 	//total amount of credited messages
-	mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", nil)
+	mMsgCredit metrics.Counter
 	//total amount of debited messages
-	mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", nil)
+	mMsgDebit metrics.Counter
 	//how many times local node had to drop remote peers
-	mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", nil)
+	mPeerDrops metrics.Counter
 	//how many times local node overdrafted and dropped
-	mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", nil)
+	mSelfDrops metrics.Counter
 )
 
 //Prices defines how prices are being passed on to the accounting instance
@@ -105,6 +108,26 @@ func NewAccounting(balance Balance, po Prices) *Accounting {
 	return ah
 }
 
+//SetupAccountingMetrics creates a separate registry for p2p accounting metrics;
+//this registry should be independent of any other metrics as it persists at different endpoints.
+//It also instantiates the given metrics and starts the persisting go-routine which
+//at the passed interval writes the metrics to a LevelDB
+func SetupAccountingMetrics(reportInterval time.Duration, path string) *AccountingMetrics {
+	//create an empty registry
+	registry := metrics.NewRegistry()
+	//instantiate the metrics
+	mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", registry)
+	mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", registry)
+	mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", registry)
+	mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", registry)
+	mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", registry)
+	mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", registry)
+	mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", registry)
+	mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", registry)
+	//create the DB and start persisting
+	return NewAccountingMetrics(registry, reportInterval, path)
+}
+
 //Implement Hook.Send
 // Send takes a peer, a size and a msg and
 // - calculates the cost for the local node sending a msg of size to peer using the Prices interface

p2p/protocols/accounting_simulation_test.go

@@ -20,7 +20,10 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
+	"os"
+	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
@@ -66,6 +69,13 @@ func init() {
 func TestAccountingSimulation(t *testing.T) {
 	//setup the balances objects for every node
 	bal := newBalances(*nodes)
+	//setup the metrics system or tests will fail trying to write metrics
+	dir, err := ioutil.TempDir("", "account-sim")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	SetupAccountingMetrics(1*time.Second, filepath.Join(dir, "metrics.db"))
 	//define the node.Service for this test
 	services := adapters.Services{
 		"accounting": func(ctx *adapters.ServiceContext) (node.Service, error) {

p2p/protocols/protocol.go

@@ -381,7 +381,7 @@ func (p *Peer) handleIncoming(handle func(ctx context.Context, msg interface{})
 // * arguments
 //   * context
 //   * the local handshake to be sent to the remote peer
-//   * funcion to be called on the remote handshake (can be nil)
+//   * function to be called on the remote handshake (can be nil)
 // * expects a remote handshake back of the same type
 // * the dialing peer needs to send the handshake first and then waits for remote
 // * the listening peer waits for the remote handshake and then sends it

p2p/protocols/reporter.go

@@ -0,0 +1,147 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package protocols
+
+import (
+	"encoding/binary"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+//AccountMetrics abstracts away the metrics DB and
+//the reporter to persist metrics
+type AccountingMetrics struct {
+	reporter *reporter
+}
+
+//Close will be called when the node is being shutdown
+//for a graceful cleanup
+func (am *AccountingMetrics) Close() {
+	close(am.reporter.quit)
+	am.reporter.db.Close()
+}
+
+//reporter is an internal structure used to write p2p accounting related
+//metrics to a LevelDB. It will periodically write the accrued metrics to the DB.
+type reporter struct {
+	reg      metrics.Registry //the registry for these metrics (independent of other metrics)
+	interval time.Duration    //duration at which the reporter will persist metrics
+	db       *leveldb.DB      //the actual DB
+	quit     chan struct{}    //quit the reporter loop
+}
+
+//NewMetricsDB creates a new LevelDB instance used to persist metrics defined
+//inside p2p/protocols/accounting.go
+func NewAccountingMetrics(r metrics.Registry, d time.Duration, path string) *AccountingMetrics {
+	var val = make([]byte, 8)
+	var err error
+
+	//Create the LevelDB
+	db, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	//Check for all defined metrics that there is a value in the DB
+	//If there is, assign it to the metric. This means that the node
+	//has been running before and that metrics have been persisted.
+	metricsMap := map[string]metrics.Counter{
+		"account.balance.credit": mBalanceCredit,
+		"account.balance.debit":  mBalanceDebit,
+		"account.bytes.credit":   mBytesCredit,
+		"account.bytes.debit":    mBytesDebit,
+		"account.msg.credit":     mMsgCredit,
+		"account.msg.debit":      mMsgDebit,
+		"account.peerdrops":      mPeerDrops,
+		"account.selfdrops":      mSelfDrops,
+	}
+	//iterate the map and get the values
+	for key, metric := range metricsMap {
+		val, err = db.Get([]byte(key), nil)
+		//until the first time a value is being written,
+		//this will return an error.
+		//it could be beneficial though to log errors later,
+		//but that would require a different logic
+		if err == nil {
+			metric.Inc(int64(binary.BigEndian.Uint64(val)))
+		}
+	}
+
+	//create the reporter
+	rep := &reporter{
+		reg:      r,
+		interval: d,
+		db:       db,
+		quit:     make(chan struct{}),
+	}
+
+	//run the go routine
+	go rep.run()
+
+	m := &AccountingMetrics{
+		reporter: rep,
+	}
+
+	return m
+}
+
+//run is the goroutine which periodically sends the metrics to the configured LevelDB
+func (r *reporter) run() {
+	intervalTicker := time.NewTicker(r.interval)
+
+	for {
+		select {
+		case <-intervalTicker.C:
+			//at each tick send the metrics
+			if err := r.save(); err != nil {
+				log.Error("unable to send metrics to LevelDB", "err", err)
+				//If there is an error in writing, exit the routine; we assume here that the error is
+				//severe and don't attempt to write again.
+				//Also, this should prevent leaking when the node is stopped
+				return
+			}
+		case <-r.quit:
+			//graceful shutdown
+			return
+		}
+	}
+}
+
+//send the metrics to the DB
+func (r *reporter) save() error {
+	//create a LevelDB Batch
+	batch := leveldb.Batch{}
+	//for each metric in the registry (which is independent)...
+	r.reg.Each(func(name string, i interface{}) {
+		metric, ok := i.(metrics.Counter)
+		if ok {
+			//assuming every metric here to be a Counter (separate registry)
+			//...create a snapshot...
+			ms := metric.Snapshot()
+			byteVal := make([]byte, 8)
+			binary.BigEndian.PutUint64(byteVal, uint64(ms.Count()))
+			//...and save the value to the DB
+			batch.Put([]byte(name), byteVal)
+		}
+	})
+	return r.db.Write(&batch, nil)
+}

p2p/protocols/reporter_test.go

@@ -0,0 +1,77 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package protocols
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+//TestReporter tests that the metrics being collected for p2p accounting
+//are being persisted and available after restart of a node.
+//It simulates restarting by just recreating the DB as if the node had restarted.
+func TestReporter(t *testing.T) {
+	//create a test directory
+	dir, err := ioutil.TempDir("", "reporter-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	//setup the metrics
+	log.Debug("Setting up metrics first time")
+	reportInterval := 5 * time.Millisecond
+	metrics := SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db"))
+	log.Debug("Done.")
+
+	//do some metrics
+	mBalanceCredit.Inc(12)
+	mBytesCredit.Inc(34)
+	mMsgDebit.Inc(9)
+
+	//give the reporter time to write the metrics to DB
+	time.Sleep(20 * time.Millisecond)
+
+	//set the metrics to nil - this effectively simulates the node having shut down...
+	mBalanceCredit = nil
+	mBytesCredit = nil
+	mMsgDebit = nil
+	//close the DB also, or we can't create a new one
+	metrics.Close()
+
+	//setup the metrics again
+	log.Debug("Setting up metrics second time")
+	metrics = SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db"))
+	defer metrics.Close()
+	log.Debug("Done.")
+
+	//now check the metrics, they should have the same value as before "shutdown"
+	if mBalanceCredit.Count() != 12 {
+		t.Fatalf("Expected counter to be %d, but is %d", 12, mBalanceCredit.Count())
+	}
+	if mBytesCredit.Count() != 34 {
+		t.Fatalf("Expected counter to be %d, but is %d", 23, mBytesCredit.Count())
+	}
+	if mMsgDebit.Count() != 9 {
+		t.Fatalf("Expected counter to be %d, but is %d", 9, mMsgDebit.Count())
+	}
+}

params/config.go

@@ -49,10 +49,10 @@ var (
 	// MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network.
 	MainnetTrustedCheckpoint = &TrustedCheckpoint{
 		Name:         "mainnet",
-		SectionIndex: 195,
+		SectionIndex: 206,
-		SectionHead:  common.HexToHash("0x1cdd2a84cf6c1261ffccc88f6bcefb513abd7934a96c1e909fbf74767560f16b"),
+		SectionHead:  common.HexToHash("0x9fa677c7c0580136f5a86d9b2fd29b112e531f0284396298b8809bcb6787b538"),
-		CHTRoot:      common.HexToHash("0xe453333c20391d16b91b6fe11c104704f62c8dba15f69db73b4cdf7e100105eb"),
+		CHTRoot:      common.HexToHash("0x7f32dfb29e341b4c8c10ea2e06a812bcea470366f635b7a8b3d0856684cd76f4"),
-		BloomRoot:    common.HexToHash("0x47f30069473072e00d2cdca146dce40f0aad243dfc8221bf810822c091674efe"),
+		BloomRoot:    common.HexToHash("0x0169e174f0a8172aec217556d8a25c7ba7ca52aacff170325230a75740ff1eaf"),
 	}
 
 	// TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network.
@@ -73,10 +73,10 @@ var (
 	// TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network.
 	TestnetTrustedCheckpoint = &TrustedCheckpoint{
 		Name:         "testnet",
-		SectionIndex: 126,
+		SectionIndex: 136,
-		SectionHead:  common.HexToHash("0x48f7dd4c9c60be04bf15fd4d0bcac46ddd8caf6b01d6fb8f8e1f7955cdd1337a"),
+		SectionHead:  common.HexToHash("0xe5d80bb08d92bbc12dfe510c64cba01eafcbb4ba585e7c7ab7f8a93c6f295ab3"),
-		CHTRoot:      common.HexToHash("0x6e54cb80a1884881ea1a114243af9012c95e0296b47f103b5ab124313968508e"),
+		CHTRoot:      common.HexToHash("0xe3ca77ab0cb51eec74f4f7458e36aee207c68768387b39cb0bcff0940a6264d8"),
-		BloomRoot:    common.HexToHash("0xb55accf6dce6455b47db8510d15eff38d0ed7378829f3036d26b48e7d15da3f6"),
+		BloomRoot:    common.HexToHash("0x30c8eeadac5539d3dcd6e88915d1a07cb2f3a1d6ebe7e553e3ee783c04c68c2d"),
 	}
 
 	// RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network.
@@ -100,10 +100,10 @@ var (
 	// RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network.
 	RinkebyTrustedCheckpoint = &TrustedCheckpoint{
 		Name:         "rinkeby",
-		SectionIndex: 93,
+		SectionIndex: 103,
-		SectionHead:  common.HexToHash("0xdefb94aa217ab38f2919f7318d1d5476bd2aabf1ec9148047fe03e555615e0b4"),
+		SectionHead:  common.HexToHash("0x9f38b903852831bf4fa7992f7fd43d8b26da2deb82b421fb845cf6faee54e056"),
-		CHTRoot:      common.HexToHash("0x52c98c2fe508a8332c27dc10538f3fead43306e2b22b597587763c2fe6586da6"),
+		CHTRoot:      common.HexToHash("0x2d710c2cea468d2e604838000d658ee213e4abb07f90c4f71f5cd7f8510aa708"),
-		BloomRoot:    common.HexToHash("0x93d83be0c1b12f732b1a027ecdfb16f39b0d020b8c10bfb90e76f3b01adfc5b6"),
+		BloomRoot:    common.HexToHash("0xcc401060280c2cc82697ea5ecef8cac61e52063c37533a2e9609332419704d5f"),
 	}
 
 	// AllEthashProtocolChanges contains every protocol change (EIPs) introduced
@@ -111,16 +111,16 @@ var (
 	//
 	// This configuration is intentionally not using keyed fields to force anyone
 	// adding flags to the config to also have to set these fields.
-	AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
+	AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
 
 	// AllCliqueProtocolChanges contains every protocol change (EIPs) introduced
 	// and accepted by the Ethereum core developers into the Clique consensus.
 	//
 	// This configuration is intentionally not using keyed fields to force anyone
 	// adding flags to the config to also have to set these fields.
-	AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
+	AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
 
-	TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
+	TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
 	TestRules       = TestChainConfig.Rules(new(big.Int))
 )
 

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1          // Major version component of the current release
 	VersionMinor = 8          // Minor version component of the current release
-	VersionPatch = 18         // Patch version component of the current release
+	VersionPatch = 20         // Patch version component of the current release
 	VersionMeta  = "unstable" // Version metadata to append to the version string
 )
 

rpc/client_example_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-// In this example, our client whishes to track the latest 'block number'
+// In this example, our client wishes to track the latest 'block number'
 // known to the server. The server supports two methods:
 //
 // eth_getBlockByNumber("latest", {})

rpc/doc.go

@@ -32,7 +32,7 @@ An example method:
  func (s *CalcService) Add(a, b int) (int, error)
 
 When the returned error isn't nil the returned integer is ignored and the error is
-send back to the client. Otherwise the returned integer is send back to the client.
+sent back to the client. Otherwise the returned integer is sent back to the client.
 
 Optional arguments are supported by accepting pointer values as arguments. E.g.
 if we want to do the addition in an optional finite field we can accept a mod

swarm/OWNERS

@@ -7,7 +7,6 @@ swarm
 ├── fuse ────────────────── @jmozah, @holisticode
 ├── grafana_dashboards ──── @nonsense
 ├── metrics ─────────────── @nonsense, @holisticode
-├── multihash ───────────── @nolash
 ├── network ─────────────── ethersphere
 │   ├── bitvector ───────── @zelig, @janos, @gbalint
 │   ├── priorityqueue ───── @zelig, @janos, @gbalint

swarm/api/api.go

@@ -42,7 +42,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/swarm/log"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
@@ -417,7 +416,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 				return reader, mimeType, status, nil, err
 			}
 			// get the data of the update
-			_, rsrcData, err := a.feed.GetContent(entry.Feed)
+			_, contentAddr, err := a.feed.GetContent(entry.Feed)
 			if err != nil {
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
@@ -425,23 +424,23 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 				return reader, mimeType, status, nil, err
 			}
 
-			// extract multihash
+			// extract content hash
-			decodedMultihash, err := multihash.FromMultihash(rsrcData)
+			if len(contentAddr) != storage.AddressLength {
-			if err != nil {
 				apiGetInvalid.Inc(1)
 				status = http.StatusUnprocessableEntity
-				log.Warn("invalid multihash in feed update", "err", err)
+				errorMessage := fmt.Sprintf("invalid swarm hash in feed update. Expected %d bytes. Got %d", storage.AddressLength, len(contentAddr))
-				return reader, mimeType, status, nil, err
+				log.Warn(errorMessage)
+				return reader, mimeType, status, nil, errors.New(errorMessage)
 			}
-			manifestAddr = storage.Address(decodedMultihash)
+			manifestAddr = storage.Address(contentAddr)
-			log.Trace("feed update contains multihash", "key", manifestAddr)
+			log.Trace("feed update contains swarm hash", "key", manifestAddr)
 
-			// get the manifest the multihash digest points to
+			// get the manifest the swarm hash points to
 			trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
 			if err != nil {
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
-				log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err))
+				log.Warn(fmt.Sprintf("loadManifestTrie (feed update) error: %v", err))
 				return reader, mimeType, status, nil, err
 			}
 
@@ -451,8 +450,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 			if entry == nil {
 				status = http.StatusNotFound
 				apiGetNotFound.Inc(1)
-				err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path)
+				err = fmt.Errorf("manifest (feed update) entry for '%s' not found", path)
-				log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path)
+				log.Trace("manifest (feed update) entry not found", "key", manifestAddr, "path", path)
 				return reader, mimeType, status, nil, err
 			}
 		}
@@ -472,7 +471,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 		// no entry found
 		status = http.StatusNotFound
 		apiGetNotFound.Inc(1)
-		err = fmt.Errorf("manifest entry for '%s' not found", path)
+		err = fmt.Errorf("Not found: could not find resource '%s'", path)
 		log.Trace("manifest entry not found", "key", contentAddr, "path", path)
 	}
 	return

swarm/api/client/client_test.go

@@ -25,18 +25,17 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
-func serverFunc(api *api.API) testutil.TestServer {
+func serverFunc(api *api.API) swarmhttp.TestServer {
 	return swarmhttp.NewServer(api, "")
 }
 
@@ -49,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) {
 }
 
 func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	client := NewClient(srv.URL)
@@ -90,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
 }
 
 func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	client := NewClient(srv.URL)
@@ -188,7 +187,7 @@ func newTestDirectory(t *testing.T) string {
 // TestClientUploadDownloadDirectory tests uploading and downloading a
 // directory of files to a swarm manifest
 func TestClientUploadDownloadDirectory(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	dir := newTestDirectory(t)
@@ -254,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) {
 }
 
 func testClientFileList(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	dir := newTestDirectory(t)
@@ -312,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
 // TestClientMultipartUpload tests uploading files to swarm using a multipart
 // upload
 func TestClientMultipartUpload(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	// define an uploader which uploads testDirFiles with some data
@@ -369,58 +368,99 @@ func newTestSigner() (*feed.GenericSigner, error) {
 	return feed.NewGenericSigner(privKey), nil
 }
 
-// test the transparent resolving of multihash feed updates with bzz:// scheme
+// Test the transparent resolving of feed updates with bzz:// scheme
 //
-// first upload data, and store the multihash to the resulting manifest in a feed update
+// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
-// retrieving the update with the multihash should return the manifest pointing directly to the data
+// This effectively uses a feed to store a pointer to content rather than the content itself
+// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
 // and raw retrieve of that hash should return the data
-func TestClientCreateFeedMultihash(t *testing.T) {
+func TestClientBzzWithFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	// Initialize a Swarm test server
-	client := NewClient(srv.URL)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	swarmClient := NewClient(srv.URL)
 	defer srv.Close()
 
-	// add the data our multihash aliased manifest will point to
+	// put together some data for our test:
-	databytes := []byte("bar")
+	dataBytes := []byte(`
+	//
+	// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
+	// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
+	//
+	// MANIFEST HASH --> DATA
+	//
+	// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
+	// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
+	//
+	// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
+	//
+	// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
+	// stays constant, we have effectively created a fixed address to changing content. (Applause)
+	//
+	// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2)
+	//
+	`)
 
-	swarmHash, err := client.UploadRaw(bytes.NewReader(databytes), int64(len(databytes)), false)
+	// Create a virtual File out of memory containing the above data
-	if err != nil {
+	f := &File{
-		t.Fatalf("Error uploading raw test data: %s", err)
+		ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
+		ManifestEntry: api.ManifestEntry{
+			ContentType: "text/plain",
+			Mode:        0660,
+			Size:        int64(len(dataBytes)),
+		},
 	}
 
-	s := common.FromHex(swarmHash)
+	// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
-	mh := multihash.ToMultihash(s)
+	manifestAddressHex, err := swarmClient.Upload(f, "", false)
+	if err != nil {
+		t.Fatalf("Error creating manifest: %s", err)
+	}
 
-	// our feed topic
+	// convert the hex-encoded manifest hash to a 32-byte slice
-	topic, _ := feed.NewTopic("foo.eth", nil)
+	manifestAddress := common.FromHex(manifestAddressHex)
 
-	createRequest := feed.NewFirstRequest(topic)
+	if len(manifestAddress) != storage.AddressLength {
+		t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress))
+	}
 
-	createRequest.SetData(mh)
+	// Now create a **feed manifest**. For that, we need a topic:
-	if err := createRequest.Sign(signer); err != nil {
+	topic, _ := feed.NewTopic("interesting topic indeed", nil)
+
+	// Build a feed request to update data
+	request := feed.NewFirstRequest(topic)
+
+	// Put the 32-byte address of the manifest into the feed update
+	request.SetData(manifestAddress)
+
+	// Sign the update
+	if err := request.Sign(signer); err != nil {
 		t.Fatalf("Error signing update: %s", err)
 	}
 
-	feedManifestHash, err := client.CreateFeedWithManifest(createRequest)
+	// Publish the update and at the same time request a **feed manifest** to be created
-
+	feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request)
 	if err != nil {
 		t.Fatalf("Error creating feed manifest: %s", err)
 	}
 
-	correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
+	// Check we have received the exact **feed manifest** to be expected
-	if feedManifestHash != correctManifestAddrHex {
+	// given the topic and user signing the updates:
-		t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash)
+	correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
+	if feedManifestAddressHex != correctFeedManifestAddrHex {
+		t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex)
 	}
 
 	// Check we get a not found error when trying to get feed updates with a made-up manifest
-	_, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
+	_, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
 	if err != ErrNoFeedUpdatesFound {
 		t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err)
 	}
 
-	reader, err := client.QueryFeed(nil, correctManifestAddrHex)
+	// If we query the feed directly we should get **manifest hash** back:
+	reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex)
 	if err != nil {
 		t.Fatalf("Error retrieving feed updates: %s", err)
 	}
@@ -429,10 +469,27 @@ func TestClientCreateFeedMultihash(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(mh, gotData) {
+
-		t.Fatalf("Expected: %v, got %v", mh, gotData)
+	//Check that indeed the **manifest hash** is retrieved
+	if !bytes.Equal(manifestAddress, gotData) {
+		t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
 	}
 
+	// Now the final test we were looking for: Use bzz://<feed-manifest> and that should resolve all manifests
+	// and return the original data directly:
+	f, err = swarmClient.Download(feedManifestAddressHex, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	gotData, err = ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that we get back the original data:
+	if !bytes.Equal(dataBytes, gotData) {
+		t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
+	}
 }
 
 // TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client.
@@ -440,7 +497,7 @@ func TestClientCreateUpdateFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	client := NewClient(srv.URL)
 	defer srv.Close()
 

swarm/api/http/middleware.go

@@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		writer := newLoggingResponseWriter(w)
 		h.ServeHTTP(writer, r)
-		log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
+		log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
 	})
 }
 

swarm/api/http/response_test.go

@@ -24,12 +24,10 @@ import (
 	"testing"
 
 	"golang.org/x/net/html"
-
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestError(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -55,7 +53,7 @@ func TestError(t *testing.T) {
 }
 
 func Test404Page(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -81,7 +79,7 @@ func Test404Page(t *testing.T) {
 }
 
 func Test500Page(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -106,7 +104,7 @@ func Test500Page(t *testing.T) {
 	}
 }
 func Test500PageWith0xHashPrefix(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response
@@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) {
 }
 
 func TestJsonResponse(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	var resp *http.Response

swarm/api/http/server_test.go

@@ -20,7 +20,6 @@ import (
 	"archive/tar"
 	"bytes"
 	"context"
-	"crypto/rand"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -46,7 +45,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
@@ -58,7 +56,7 @@ func init() {
 	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
 }
 
-func serverFunc(api *api.API) testutil.TestServer {
+func serverFunc(api *api.API) TestServer {
 	return NewServer(api, "")
 }
 
@@ -70,60 +68,91 @@ func newTestSigner() (*feed.GenericSigner, error) {
 	return feed.NewGenericSigner(privKey), nil
 }
 
-// test the transparent resolving of multihash-containing feed updates with bzz:// scheme
+// Test the transparent resolving of feed updates with bzz:// scheme
 //
-// first upload data, and store the multihash to the resulting manifest in a feed update
+// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
-// retrieving the update with the multihash should return the manifest pointing directly to the data
+// This effectively uses a feed to store a pointer to content rather than the content itself
+// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
 // and raw retrieve of that hash should return the data
-func TestBzzFeedMultihash(t *testing.T) {
+func TestBzzWithFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	// Initialize Swarm test server
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
-	// add the data our multihash aliased manifest will point to
+	// put together some data for our test:
-	databytes := "bar"
+	dataBytes := []byte(`
-	testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL)
+	//
-	resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes)))
+	// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
+	// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
+	//
+	// MANIFEST HASH --> DATA
+	//
+	// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
+	// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
+	//
+	// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
+	//
+	// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
+	// stays constant, we have effectively created a fixed address to changing content. (Applause)
+	//
+	// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) ...
+	//
+	`)
+
+	// POST data to bzz and get back a content-addressed **manifest hash** pointing to it.
+	resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader([]byte(dataBytes)))
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)
 	}
-	b, err := ioutil.ReadAll(resp.Body)
+	manifestAddressHex, err := ioutil.ReadAll(resp.Body)
 
 	if err != nil {
 		t.Fatal(err)
 	}
-	s := common.FromHex(string(b))
-	mh := multihash.ToMultihash(s)
 
-	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
+	manifestAddress := common.FromHex(string(manifestAddressHex))
 
-	topic, _ := feed.NewTopic("foo.eth", nil)
+	log.Info("added data", "manifest", string(manifestAddressHex))
+
+	// At this point we have uploaded the data and have a manifest pointing to it
+	// Now store that manifest address in a feed update.
+	// We also want a feed manifest, so we can use it to refer to the feed.
+
+	// First, create a topic for our feed:
+	topic, _ := feed.NewTopic("interesting topic indeed", nil)
+
+	// Create a feed update request:
 	updateRequest := feed.NewFirstRequest(topic)
 
-	updateRequest.SetData(mh)
+	// Store the **manifest address** as data into the feed update.
+	updateRequest.SetData(manifestAddress)
 
+	// Sign the update
 	if err := updateRequest.Sign(signer); err != nil {
 		t.Fatal(err)
 	}
-	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
+	log.Info("added data", "data", common.ToHex(manifestAddress))
 
-	testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL))
+	// Build the feed update http request:
+	feedUpdateURL, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL))
 	if err != nil {
 		t.Fatal(err)
 	}
-	query := testUrl.Query()
+	query := feedUpdateURL.Query()
 	body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted
-	query.Set("manifest", "1")                // indicate we want a manifest back
+	query.Set("manifest", "1")                // indicate we want a feed manifest back
-	testUrl.RawQuery = query.Encode()
+	feedUpdateURL.RawQuery = query.Encode()
 
-	// create the multihash update
+	// submit the feed update request to Swarm
-	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
+	resp, err = http.Post(feedUpdateURL.String(), "application/octet-stream", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -131,24 +160,25 @@ func TestBzzFeedMultihash(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)
 	}
-	b, err = ioutil.ReadAll(resp.Body)
+
+	feedManifestAddressHex, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}
-	rsrcResp := &storage.Address{}
+	feedManifestAddress := &storage.Address{}
-	err = json.Unmarshal(b, rsrcResp)
+	err = json.Unmarshal(feedManifestAddressHex, feedManifestAddress)
 	if err != nil {
-		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
+		t.Fatalf("data %s could not be unmarshaled: %v", feedManifestAddressHex, err)
 	}
 
-	correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
+	correctManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
-	if rsrcResp.Hex() != correctManifestAddrHex {
+	if feedManifestAddress.Hex() != correctManifestAddrHex {
-		t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
+		t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestAddress.Hex())
 	}
 
 	// get bzz manifest transparent feed update resolve
-	testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
+	getBzzURL := fmt.Sprintf("%s/bzz:/%s", srv.URL, feedManifestAddress)
-	resp, err = http.Get(testBzzUrl)
+	resp, err = http.Get(getBzzURL)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -156,37 +186,30 @@ func TestBzzFeedMultihash(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)
 	}
-	b, err = ioutil.ReadAll(resp.Body)
+	retrievedData, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(b, []byte(databytes)) {
+	if !bytes.Equal(retrievedData, []byte(dataBytes)) {
-		t.Fatalf("retrieved data mismatch, expected %x, got %x", databytes, b)
+		t.Fatalf("retrieved data mismatch, expected %x, got %x", dataBytes, retrievedData)
 	}
 }
 
 // Test Swarm feeds using the raw update methods
 func TestBzzFeed(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	signer, _ := newTestSigner()
 
 	defer srv.Close()
 
 	// data of update 1
-	update1Data := make([]byte, 666)
+	update1Data := testutil.RandomBytes(1, 666)
 	update1Timestamp := srv.CurrentTime
-	_, err := rand.Read(update1Data)
-	if err != nil {
-		t.Fatal(err)
-	}
 	//data for update 2
 	update2Data := []byte("foo")
 
 	topic, _ := feed.NewTopic("foo.eth", nil)
 	updateRequest := feed.NewFirstRequest(topic)
-	if err != nil {
-		t.Fatal(err)
-	}
 	updateRequest.SetData(update1Data)
 
 	if err := updateRequest.Sign(signer); err != nil {
@@ -253,7 +276,8 @@ func TestBzzFeed(t *testing.T) {
 		t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex())
 	}
 
-	// get bzz manifest transparent feed update resolve
+	// take the chance to have bzz: crash on resolving a feed update that does not contain
+	// a swarm hash:
 	testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
 	resp, err = http.Get(testBzzUrl)
 	if err != nil {
@@ -261,7 +285,7 @@ func TestBzzFeed(t *testing.T) {
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == http.StatusOK {
-		t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK")
+		t.Fatal("Expected error status since feed update does not contain a Swarm hash. Received 200 OK")
 	}
 	_, err = ioutil.ReadAll(resp.Body)
 	if err != nil {
@@ -450,7 +474,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
 
 	addr := [3]storage.Address{}
 
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	for i, mf := range testmanifest {
@@ -688,7 +712,7 @@ func TestBzzTar(t *testing.T) {
 }
 
 func testBzzTar(encrypted bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 	fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"}
 	fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
@@ -823,7 +847,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) {
 }
 
 func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	// create a manifest with some data at the root path
@@ -878,7 +902,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
 }
 
 func TestMethodsNotAllowed(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 	databytes := "bar"
 	for _, c := range []struct {
@@ -937,7 +961,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string
 }
 
 func TestGet(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	for _, testCase := range []struct {
@@ -1020,7 +1044,7 @@ func TestGet(t *testing.T) {
 }
 
 func TestModify(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	swarmClient := swarm.NewClient(srv.URL)
@@ -1121,7 +1145,7 @@ func TestMultiPartUpload(t *testing.T) {
 	// POST /bzz:/ Content-Type: multipart/form-data
 	verbose := false
 	// Setup Swarm
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	url := fmt.Sprintf("%s/bzz:/", srv.URL)
@@ -1152,7 +1176,7 @@ func TestMultiPartUpload(t *testing.T) {
 // TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver
 func TestBzzGetFileWithResolver(t *testing.T) {
 	resolver := newTestResolveValidator("")
-	srv := testutil.NewTestSwarmServer(t, serverFunc, resolver)
+	srv := NewTestSwarmServer(t, serverFunc, resolver)
 	defer srv.Close()
 	fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"}
 	fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}

swarm/api/http/test_server.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package testutil
+package http
 
 import (
 	"io/ioutil"

swarm/bmt/bmt_test.go

@@ -18,10 +18,8 @@ package bmt
 
 import (
 	"bytes"
-	crand "crypto/rand"
 	"encoding/binary"
 	"fmt"
-	"io"
 	"math/rand"
 	"sync"
 	"sync/atomic"
@@ -29,6 +27,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // the actual data length generated (could be longer than max datalength of the BMT)
@@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) {
 	})
 
 	// run the tests
-	for _, x := range tests {
+	for i, x := range tests {
 		for segmentCount := x.from; segmentCount <= x.to; segmentCount++ {
 			for length := 1; length <= segmentCount*32; length++ {
 				t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) {
-					data := make([]byte, length)
+					data := testutil.RandomBytes(i, length)
-					if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF {
-						t.Fatal(err)
-					}
 					expected := x.expected(data)
 					actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data)
 					if !bytes.Equal(actual, expected) {
@@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) {
 
 // tests sequential write with entire max size written in one go
 func TestSyncHasherCorrectness(t *testing.T) {
-	data := newData(BufferSize)
+	data := testutil.RandomBytes(1, BufferSize)
 	hasher := sha3.NewKeccak256
 	size := hasher().Size()
 
@@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) {
 
 // tests order-neutral concurrent writes with entire max size written in one go
 func TestAsyncCorrectness(t *testing.T) {
-	data := newData(BufferSize)
+	data := testutil.RandomBytes(1, BufferSize)
 	hasher := sha3.NewKeccak256
 	size := hasher().Size()
 	whs := []whenHash{first, last, random}
@@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) {
 	bmt := New(pool)
 
 	for i := 0; i < 100; i++ {
-		data := newData(BufferSize)
+		data := testutil.RandomBytes(1, BufferSize)
 		n := rand.Intn(bmt.Size())
 		err := testHasherCorrectness(bmt, hasher, data, n, segmentCount)
 		if err != nil {
@@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) {
 	for i := 0; i < cycles; i++ {
 		go func() {
 			bmt := New(pool)
-			data := newData(BufferSize)
+			data := testutil.RandomBytes(1, BufferSize)
 			n := rand.Intn(bmt.Size())
 			errc <- testHasherCorrectness(bmt, hasher, data, n, 128)
 		}()
@@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) {
 			defer pool.Drain(0)
 			n := count * 32
 			bmt := New(pool)
-			data := newData(n)
+			data := testutil.RandomBytes(1, n)
 			rbmt := NewRefHasher(hasher, count)
 			refHash := rbmt.Hash(data)
 			expHash := syncHash(bmt, nil, data)
@@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) {
 
 // benchmarks simple sha3 hash on chunks
 func benchmarkSHA3(t *testing.B, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	h := hasher()
 
@@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) {
 func benchmarkBMTBaseline(t *testing.B, n int) {
 	hasher := sha3.NewKeccak256
 	hashSize := hasher().Size()
-	data := newData(hashSize)
+	data := testutil.RandomBytes(1, hashSize)
 
 	t.ReportAllocs()
 	t.ResetTimer()
@@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) {
 
 // benchmarks BMT Hasher
 func benchmarkBMT(t *testing.B, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	bmt := New(pool)
@@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) {
 
 // benchmarks BMT hasher with asynchronous concurrent segment/section writes
 func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	bmt := New(pool).NewAsyncWriter(double)
 	idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
-	shuffle(len(idxs), func(i int, j int) {
+	rand.Shuffle(len(idxs), func(i int, j int) {
 		idxs[i], idxs[j] = idxs[j], idxs[i]
 	})
 
@@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
 
 // benchmarks 100 concurrent bmt hashes with pool capacity
 func benchmarkPool(t *testing.B, poolsize, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	pool := NewTreePool(hasher, segmentCount, poolsize)
 	cycles := 100
@@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) {
 
 // benchmarks the reference hasher
 func benchmarkRefHasher(t *testing.B, n int) {
-	data := newData(n)
+	data := testutil.RandomBytes(1, n)
 	hasher := sha3.NewKeccak256
 	rbmt := NewRefHasher(hasher, 128)
 
@@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) {
 	}
 }
 
-func newData(bufferSize int) []byte {
-	data := make([]byte, bufferSize)
-	_, err := io.ReadFull(crand.Reader, data)
-	if err != nil {
-		panic(err.Error())
-	}
-	return data
-}
-
 // Hash hashes the data and the span using the bmt hasher
 func syncHash(h *Hasher, span, data []byte) []byte {
 	h.ResetWithLength(span)
@@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) {
 		section := data[i*secsize : end]
 		segments = append(segments, section)
 	}
-	shuffle(n, func(i int, j int) {
+	rand.Shuffle(n, func(i int, j int) {
 		idxs[i], idxs[j] = idxs[j], idxs[i]
 	})
 	return idxs, segments
@@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s
 	}
 	return <-c
 }
-
-// this is also in swarm/network_test.go
-// shuffle pseudo-randomizes the order of elements.
-// n is the number of elements. Shuffle panics if n < 0.
-// swap swaps the elements with indexes i and j.
-func shuffle(n int, swap func(i, j int)) {
-	if n < 0 {
-		panic("invalid argument to Shuffle")
-	}
-
-	// Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
-	// Shuffle really ought not be called with n that doesn't fit in 32 bits.
-	// Not only will it take a very long time, but with 2³¹! possible permutations,
-	// there's no way that any PRNG can have a big enough internal state to
-	// generate even a minuscule percentage of the possible permutations.
-	// Nevertheless, the right API signature accepts an int n, so handle it as best we can.
-	i := n - 1
-	for ; i > 1<<31-1-1; i-- {
-		j := int(rand.Int63n(int64(i + 1)))
-		swap(i, j)
-	}
-	for ; i > 0; i-- {
-		j := int(rand.Int31n(int32(i + 1)))
-		swap(i, j)
-	}
-}

swarm/fuse/swarmfs_test.go

@@ -20,20 +20,19 @@ package fuse
 
 import (
 	"bytes"
-	"crypto/rand"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-
+	"github.com/ethereum/go-ethereum/swarm/testutil"
-	"github.com/ethereum/go-ethereum/log"
-
 	colorable "github.com/mattn/go-colorable"
 )
 
@@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) {
 	}
 }
 
-func getRandomBytes(size int) []byte {
-	contents := make([]byte, size)
-	rand.Read(contents)
-	return contents
-}
-
 func isDirEmpty(name string) bool {
 	f, err := os.Open(name)
 	if err != nil {
@@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) {
 	dat.testMountDir = filepath.Join(dat.testDir, "testMountDir")
 	dat.files = make(map[string]fileInfo)
 
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)}
+	dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)}
+	dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)}
-	dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)}
+	dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)}
-	dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)}
+	dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)}
-	dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)}
+	dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)}
-	dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)}
+	dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)}
-	dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)}
+	dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)}
-	dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)}
-	dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)}
-	dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)}
-	dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)}
-	dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)}
+	dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)}
-	dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)}
+	dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)}
-	dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)}
+	dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)}
-	dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+	dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount1")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount2")
-	dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount3")
-	dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount4")
-	dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 
 	dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5")
 	dat.testMountDir = filepath.Join(dat.testDir, "max-mount5")
-	dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
 	if err != nil {
 		t.Fatalf("Couldn't create upload dir 6: %v", err)
 	}
-	dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 	testMountDir6 := filepath.Join(dat.testDir, "max-mount6")
 	err = os.MkdirAll(testMountDir6, 0777)
 	if err != nil {
@@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
 	dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1")
 	dat.files = make(map[string]fileInfo)
 
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
 	}
 
 	// mount a different hash in already mounted point
-	dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 	testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2")
 	if err3 != nil {
 		t.Fatalf("Error creating second upload dir: %v", err3)
@@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
 	//we need to manually close the file before mount for this test
 	//but let's defer too in case of errors
 	defer d.Close()
-	_, err = d.Write(getRandomBytes(10))
+	_, err = d.Write(testutil.RandomBytes(1, 10))
 	if err != nil {
 		t.Fatalf("Couldn't write to file: %v", err)
 	}
@@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "seek-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1")
 	dat.testMountDir = filepath.Join(dat.testDir, "create-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
 	}
 	defer d.Close()
 	log.Debug("Opened file")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Could not rand read contents %v", err)
-	}
 	log.Debug("content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
 	}
 	defer d.Close()
 	log.Debug("File opened")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Error filling random bytes into byte array %v", err)
-	}
 	log.Debug("Content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
 	dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
 	}
 	defer d.Close()
 	log.Debug("File opened")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 	log.Debug("content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
 	}
 	defer d.Close()
 	log.Debug("file opened")
-	contents := make([]byte, 11)
+	contents := testutil.RandomBytes(1, 11)
-	_, err = rand.Read(contents)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 	log.Debug("content read")
 	_, err = d.Write(contents)
 	if err != nil {
@@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) {
 	dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload")
 	dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount")
 	dat.files = make(map[string]fileInfo)
-	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
-	dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
-	dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
-	dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)}
-	dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)}
-	dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+	dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)}
 
 	dat, err = ta.uploadAndMount(dat, t)
 	if err != nil {
@@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
 	dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount")
 	dat.files = make(map[string]fileInfo)
 
-	line1 := make([]byte, 10)
+	line1 := testutil.RandomBytes(1, 10)
-	_, err = rand.Read(line1)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 
 	dat.files["1.txt"] = fileInfo{0700, 333, 444, line1}
 
@@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
 	}
 	defer fd.Close()
 	log.Debug("file opened")
-	line2 := make([]byte, 5)
+	line2 := testutil.RandomBytes(1, 5)
-	_, err = rand.Read(line2)
-	if err != nil {
-		t.Fatalf("Error writing random bytes to byte array: %v", err)
-	}
 	log.Debug("line read")
 	_, err = fd.Seek(int64(len(line1)), 0)
 	if err != nil {

swarm/multihash/multihash.go

@@ -1,92 +0,0 @@
-// Copyright 2018 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package multihash
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-)
-
-const (
-	defaultMultihashLength   = 32
-	defaultMultihashTypeCode = 0x1b
-)
-
-var (
-	multihashTypeCode uint8
-	MultihashLength   = defaultMultihashLength
-)
-
-func init() {
-	multihashTypeCode = defaultMultihashTypeCode
-	MultihashLength = defaultMultihashLength
-}
-
-// check if valid swarm multihash
-func isSwarmMultihashType(code uint8) bool {
-	return code == multihashTypeCode
-}
-
-// GetMultihashLength returns the digest length of the provided multihash
-// It will fail if the multihash is not a valid swarm mulithash
-func GetMultihashLength(data []byte) (int, int, error) {
-	cursor := 0
-	typ, c := binary.Uvarint(data)
-	if c <= 0 {
-		return 0, 0, errors.New("unreadable hashtype field")
-	}
-	if !isSwarmMultihashType(uint8(typ)) {
-		return 0, 0, fmt.Errorf("hash code %x is not a swarm hashtype", typ)
-	}
-	cursor += c
-	hashlength, c := binary.Uvarint(data[cursor:])
-	if c <= 0 {
-		return 0, 0, errors.New("unreadable length field")
-	}
-	cursor += c
-
-	// we cheekily assume hashlength < maxint
-	inthashlength := int(hashlength)
-	if len(data[c:]) < inthashlength {
-		return 0, 0, errors.New("length mismatch")
-	}
-	return inthashlength, cursor, nil
-}
-
-// FromMulithash returns the digest portion of the multihash
-// It will fail if the multihash is not a valid swarm multihash
-func FromMultihash(data []byte) ([]byte, error) {
-	hashLength, _, err := GetMultihashLength(data)
-	if err != nil {
-		return nil, err
-	}
-	return data[len(data)-hashLength:], nil
-}
-
-// ToMulithash wraps the provided digest data with a swarm mulithash header
-func ToMultihash(hashData []byte) []byte {
-	buf := bytes.NewBuffer(nil)
-	b := make([]byte, 8)
-	c := binary.PutUvarint(b, uint64(multihashTypeCode))
-	buf.Write(b[:c])
-	c = binary.PutUvarint(b, uint64(len(hashData)))
-	buf.Write(b[:c])
-	buf.Write(hashData)
-	return buf.Bytes()
-}

swarm/multihash/multihash_test.go

@@ -1,53 +0,0 @@
-// Copyright 2018 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package multihash
-
-import (
-	"bytes"
-	"math/rand"
-	"testing"
-)
-
-// parse multihash, and check that invalid multihashes fail
-func TestCheckMultihash(t *testing.T) {
-	hashbytes := make([]byte, 32)
-	c, err := rand.Read(hashbytes)
-	if err != nil {
-		t.Fatal(err)
-	} else if c < 32 {
-		t.Fatal("short read")
-	}
-
-	expected := ToMultihash(hashbytes)
-
-	l, hl, _ := GetMultihashLength(expected)
-	if l != 32 {
-		t.Fatalf("expected length %d, got %d", 32, l)
-	} else if hl != 2 {
-		t.Fatalf("expected header length %d, got %d", 2, hl)
-	}
-	if _, _, err := GetMultihashLength(expected[1:]); err == nil {
-		t.Fatal("expected failure on corrupt header")
-	}
-	if _, _, err := GetMultihashLength(expected[:len(expected)-2]); err == nil {
-		t.Fatal("expected failure on short content")
-	}
-	dh, _ := FromMultihash(expected)
-	if !bytes.Equal(dh, hashbytes) {
-		t.Fatalf("expected content hash %x, got %x", hashbytes, dh)
-	}
-}

swarm/network/kademlia.go

@@ -89,6 +89,7 @@ type Kademlia struct {
 	nDepth     int                                     // stores the last neighbourhood depth
 	nDepthC    chan int                                // returned by DepthC function to signal neighbourhood depth change
 	addrCountC chan int                                // returned by AddrCountC function to signal peer count change
+	Pof        func(pot.Val, pot.Val, int) (int, bool) // function for calculating kademlia routing distance between two addresses
 }
 
 // NewKademlia creates a Kademlia table for base address addr
@@ -103,6 +104,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia {
 		KadParams: params,
 		addrs:     pot.NewPot(nil, 0),
 		conns:     pot.NewPot(nil, 0),
+		Pof:       pof,
 	}
 }
 
@@ -175,7 +177,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) {
 	k.lock.Lock()
 	defer k.lock.Unlock()
 	minsize := k.MinBinSize
-	depth := k.neighbourhoodDepth()
+	depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 	// if there is a callable neighbour within the current proxBin, connect
 	// this makes sure nearest neighbour set is fully connected
 	var ppo int
@@ -289,6 +291,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) {
 // neighbourhood depth on each change.
 // Not receiving from the returned channel will block On function
 // when the neighbourhood depth is changed.
+// TODO: Why is this exported, and if it should be; why can't we have more subscribers than one?
 func (k *Kademlia) NeighbourhoodDepthC() <-chan int {
 	k.lock.Lock()
 	defer k.lock.Unlock()
@@ -305,7 +308,7 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() {
 	// It provides signaling of neighbourhood depth change.
 	// This part of the code is sending new neighbourhood depth to nDepthC if that condition is met.
 	if k.nDepthC != nil {
-		nDepth := k.neighbourhoodDepth()
+		nDepth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 		if nDepth != k.nDepth {
 			k.nDepth = nDepth
 			k.nDepthC <- nDepth
@@ -361,7 +364,7 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con
 
 	var startPo int
 	var endPo int
-	kadDepth := k.neighbourhoodDepth()
+	kadDepth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 
 	k.conns.EachBin(base, pof, o, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
 		if startPo > 0 && endPo != k.MaxProxDisplay {
@@ -395,7 +398,7 @@ func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int, bool) bool) {
 	if len(base) == 0 {
 		base = k.base
 	}
-	depth := k.neighbourhoodDepth()
+	depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 	k.conns.EachNeighbour(base, pof, func(val pot.Val, po int) bool {
 		if po > o {
 			return true
@@ -417,7 +420,7 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool
 	if len(base) == 0 {
 		base = k.base
 	}
-	depth := k.neighbourhoodDepth()
+	depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 	k.addrs.EachNeighbour(base, pof, func(val pot.Val, po int) bool {
 		if po > o {
 			return true
@@ -426,21 +429,72 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool
 	})
 }
 
-// neighbourhoodDepth returns the proximity order that defines the distance of
+func (k *Kademlia) NeighbourhoodDepth() (depth int) {
+	k.lock.RLock()
+	defer k.lock.RUnlock()
+	return depthForPot(k.conns, k.MinProxBinSize, k.base)
+}
+
+// depthForPot returns the proximity order that defines the distance of
 // the nearest neighbour set with cardinality >= MinProxBinSize
 // if there is altogether less than MinProxBinSize peers it returns 0
 // caller must hold the lock
-func (k *Kademlia) neighbourhoodDepth() (depth int) {
+func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) {
-	if k.conns.Size() < k.MinProxBinSize {
+	if p.Size() <= minProxBinSize {
 		return 0
 	}
+
+	// total number of peers in iteration
 	var size int
+
+	// true if iteration has all prox peers
+	var b bool
+
+	// last po recorded in iteration
+	var lastPo int
+
 	f := func(v pot.Val, i int) bool {
-		size++
+		// po == 256 means that addr is the pivot address(self)
-		depth = i
+		if i == 256 {
-		return size < k.MinProxBinSize
+			return true
+		}
+		size++
+
+		// this means we have all nn-peers.
+		// depth is by default set to the bin of the farthest nn-peer
+		if size == minProxBinSize {
+			b = true
+			depth = i
+			return true
+		}
+
+		// if there are empty bins between farthest nn and current node,
+		// the depth should recalculated to be
+		// the farthest of those empty bins
+		//
+		// 0   abac ccde
+		// 1   2a2a
+		// 2   589f       <--- nearest non-nn
+		// ============ DEPTH 3  ===========
+		// 3              <--- don't count as empty bins
+		// 4              <--- don't count as empty bins
+		// 5  cbcb cdcd    <---- furthest nn
+		// 6  a1a2 b3c4
+		if b && i < depth {
+			depth = i + 1
+			lastPo = i
+			return false
+		}
+		lastPo = i
+		return true
+	}
+	p.EachNeighbour(pivotAddr, pof, f)
+
+	// cover edge case where more than one farthest nn
+	// AND we only have nn-peers
+	if lastPo == depth {
+		depth = 0
 	}
-	k.conns.EachNeighbour(k.base, pof, f)
 	return depth
 }
 
@@ -500,7 +554,7 @@ func (k *Kademlia) string() string {
 	liverows := make([]string, k.MaxProxDisplay)
 	peersrows := make([]string, k.MaxProxDisplay)
 
-	depth := k.neighbourhoodDepth()
+	depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 	rest := k.conns.Size()
 	k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
 		var rowlen int
@@ -570,6 +624,7 @@ type PeerPot struct {
 // as hexadecimal representations of the address.
 // used for testing only
 func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot {
+
 	// create a table of all nodes for health check
 	np := pot.NewPot(nil, 0)
 	for _, addr := range addrs {
@@ -578,34 +633,47 @@ func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot {
 	ppmap := make(map[string]*PeerPot)
 
 	for i, a := range addrs {
-		pl := 256
+
-		prev := 256
+		// actual kademlia depth
+		depth := depthForPot(np, kadMinProxSize, a)
+
+		// upon entering a new iteration
+		// this will hold the value the po should be
+		// if it's one higher than the po in the last iteration
+		prevPo := 256
+
+		// all empty bins which are outside neighbourhood depth
 		var emptyBins []int
+
+		// all nn-peers
 		var nns [][]byte
-		np.EachNeighbour(addrs[i], pof, func(val pot.Val, po int) bool {
+
-			a := val.([]byte)
+		np.EachNeighbour(a, pof, func(val pot.Val, po int) bool {
+			addr := val.([]byte)
+			// po == 256 means that addr is the pivot address(self)
 			if po == 256 {
 				return true
 			}
-			if pl == 256 || pl == po {
+
-				nns = append(nns, a)
+			// iterate through the neighbours, going from the closest to the farthest
+			// we calculate the nearest neighbours that should be in the set
+			// depth in this case equates to:
+			// 1.  Within all bins that are higher or equal than depth there are
+			//     at least minProxBinSize peers connected
+			// 2.  depth-1 bin is not empty
+			if po >= depth {
+				nns = append(nns, addr)
+				prevPo = depth - 1
+				return true
 			}
-			if pl == 256 && len(nns) >= kadMinProxSize {
+			for j := prevPo; j > po; j-- {
-				pl = po
-				prev = po
-			}
-			if prev < pl {
-				for j := prev; j > po; j-- {
 				emptyBins = append(emptyBins, j)
 			}
-			}
+			prevPo = po - 1
-			prev = po - 1
 			return true
 		})
-		for j := prev; j >= 0; j-- {
+
-			emptyBins = append(emptyBins, j)
+		log.Trace(fmt.Sprintf("%x NNS: %s, emptyBins: %s", addrs[i][:4], LogAddrs(nns), logEmptyBins(emptyBins)))
-		}
-		log.Trace(fmt.Sprintf("%x NNS: %s", addrs[i][:4], LogAddrs(nns)))
 		ppmap[common.Bytes2Hex(a)] = &PeerPot{nns, emptyBins}
 	}
 	return ppmap
@@ -620,7 +688,7 @@ func (k *Kademlia) saturation(n int) int {
 		prev++
 		return prev == po && size >= n
 	})
-	depth := k.neighbourhoodDepth()
+	depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 	if depth < prev {
 		return depth
 	}
@@ -633,8 +701,11 @@ func (k *Kademlia) full(emptyBins []int) (full bool) {
 	prev := 0
 	e := len(emptyBins)
 	ok := true
-	depth := k.neighbourhoodDepth()
+	depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
 	k.conns.EachBin(k.base, pof, 0, func(po, _ int, _ func(func(val pot.Val, i int) bool) bool) bool {
+		if po >= depth {
+			return false
+		}
 		if prev == depth+1 {
 			return true
 		}

swarm/network/kademlia_test.go

@@ -25,6 +25,9 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/p2p/protocols"
 	"github.com/ethereum/go-ethereum/swarm/pot"
 )
 
@@ -73,6 +76,76 @@ func Register(k *Kademlia, regs ...string) {
 	}
 }
 
+// tests the validity of neighborhood depth calculations
+//
+// in particular, it tests that if there are one or more consecutive
+// empty bins above the farthest "nearest neighbor-peer" then
+// the depth should be set at the farthest of those empty bins
+//
+// TODO: Make test adapt to change in MinProxBinSize
+func TestNeighbourhoodDepth(t *testing.T) {
+	baseAddressBytes := RandomAddr().OAddr
+	kad := NewKademlia(baseAddressBytes, NewKadParams())
+
+	baseAddress := pot.NewAddressFromBytes(baseAddressBytes)
+
+	closerAddress := pot.RandomAddressAt(baseAddress, 7)
+	closerPeer := newTestDiscoveryPeer(closerAddress, kad)
+	kad.On(closerPeer)
+	depth := kad.NeighbourhoodDepth()
+	if depth != 0 {
+		t.Fatalf("expected depth 0, was %d", depth)
+	}
+
+	sameAddress := pot.RandomAddressAt(baseAddress, 7)
+	samePeer := newTestDiscoveryPeer(sameAddress, kad)
+	kad.On(samePeer)
+	depth = kad.NeighbourhoodDepth()
+	if depth != 0 {
+		t.Fatalf("expected depth 0, was %d", depth)
+	}
+
+	midAddress := pot.RandomAddressAt(baseAddress, 4)
+	midPeer := newTestDiscoveryPeer(midAddress, kad)
+	kad.On(midPeer)
+	depth = kad.NeighbourhoodDepth()
+	if depth != 5 {
+		t.Fatalf("expected depth 5, was %d", depth)
+	}
+
+	kad.Off(midPeer)
+	depth = kad.NeighbourhoodDepth()
+	if depth != 0 {
+		t.Fatalf("expected depth 0, was %d", depth)
+	}
+
+	fartherAddress := pot.RandomAddressAt(baseAddress, 1)
+	fartherPeer := newTestDiscoveryPeer(fartherAddress, kad)
+	kad.On(fartherPeer)
+	depth = kad.NeighbourhoodDepth()
+	if depth != 2 {
+		t.Fatalf("expected depth 2, was %d", depth)
+	}
+
+	midSameAddress := pot.RandomAddressAt(baseAddress, 4)
+	midSamePeer := newTestDiscoveryPeer(midSameAddress, kad)
+	kad.Off(closerPeer)
+	kad.On(midPeer)
+	kad.On(midSamePeer)
+	depth = kad.NeighbourhoodDepth()
+	if depth != 2 {
+		t.Fatalf("expected depth 2, was %d", depth)
+	}
+
+	kad.Off(fartherPeer)
+	log.Trace(kad.string())
+	time.Sleep(time.Millisecond)
+	depth = kad.NeighbourhoodDepth()
+	if depth != 0 {
+		t.Fatalf("expected depth 0, was %d", depth)
+	}
+}
+
 func testSuggestPeer(k *Kademlia, expAddr string, expPo int, expWant bool) error {
 	addr, o, want := k.SuggestPeer()
 	if binStr(addr) != expAddr {
@@ -376,7 +449,7 @@ func TestKademliaHiveString(t *testing.T) {
 	Register(k, "10000000", "10000001")
 	k.MaxProxDisplay = 8
 	h := k.String()
-	expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000  0                              |  2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001  1 4000                         |  1 4000 (0)\n002  1 2000                         |  1 2000 (0)\n003  0                              |  0\n004  0                              |  0\n005  0                              |  0\n006  0                              |  0\n007  0                              |  0\n========================================================================="
+	expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000  0                              |  2 8100 (0) 8000 (0)\n001  1 4000                         |  1 4000 (0)\n002  1 2000                         |  1 2000 (0)\n003  0                              |  0\n004  0                              |  0\n005  0                              |  0\n006  0                              |  0\n007  0                              |  0\n========================================================================="
 	if expH[104:] != h[104:] {
 		t.Fatalf("incorrect hive output. expected %v, got %v", expH, h)
 	}
@@ -644,3 +717,17 @@ func TestKademliaCase5(t *testing.T) {
 		"78fafa0809929a1279ece089a51d12457c2d8416dff859aeb2ccc24bb50df5ec", "1dd39b1257e745f147cbbc3cadd609ccd6207c41056dbc4254bba5d2527d3ee5", "5f61dd66d4d94aec8fcc3ce0e7885c7edf30c43143fa730e2841c5d28e3cd081", "8aa8b0472cb351d967e575ad05c4b9f393e76c4b01ef4b3a54aac5283b78abc9", "4502f385152a915b438a6726ce3ea9342e7a6db91a23c2f6bee83a885ed7eb82", "718677a504249db47525e959ef1784bed167e1c46f1e0275b9c7b588e28a3758", "7c54c6ed1f8376323896ed3a4e048866410de189e9599dd89bf312ca4adb96b5", "18e03bd3378126c09e799a497150da5c24c895aedc84b6f0dbae41fc4bac081a", "23db76ac9e6e58d9f5395ca78252513a7b4118b4155f8462d3d5eec62486cadc", "40ae0e8f065e96c7adb7fa39505136401f01780481e678d718b7f6dbb2c906ec", "c1539998b8bae19d339d6bbb691f4e9daeb0e86847545229e80fe0dffe716e92", "ed139d73a2699e205574c08722ca9f030ad2d866c662f1112a276b91421c3cb9", "5bdb19584b7a36d09ca689422ef7e6bb681b8f2558a6b2177a8f7c812f631022", "636c9de7fe234ffc15d67a504c69702c719f626c17461d3f2918e924cd9d69e2", "de4455413ff9335c440d52458c6544191bd58a16d85f700c1de53b62773064ea", "de1963310849527acabc7885b6e345a56406a8f23e35e436b6d9725e69a79a83", "a80a50a467f561210a114cba6c7fb1489ed43a14d61a9edd70e2eb15c31f074d", "7804f12b8d8e6e4b375b242058242068a3809385e05df0e64973cde805cf729c", "60f9aa320c02c6f2e6370aa740cf7cea38083fa95fca8c99552cda52935c1520", "d8da963602390f6c002c00ce62a84b514edfce9ebde035b277a957264bb54d21", "8463d93256e026fe436abad44697152b9a56ac8e06a0583d318e9571b83d073c", "9a3f78fcefb9a05e40a23de55f6153d7a8b9d973ede43a380bf46bb3b3847de1", "e3bb576f4b3760b9ca6bff59326f4ebfc4a669d263fb7d67ab9797adea54ed13", "4d5cdbd6dcca5bdf819a0fe8d175dc55cc96f088d37462acd5ea14bc6296bdbe", "5a0ed28de7b5258c727cb85447071c74c00a5fbba9e6bc0393bc51944d04ab2a", "61e4ddb479c283c638f4edec24353b6cc7a3a13b930824aad016b0996ca93c47", "7e3610868acf714836cafaaa7b8c009a9ac6e3a6d443e5586cf661530a204ee2", "d74b244d4345d2c86e30a097105e4fb133d53c578320285132a952cdaa64416e", "cfeed57d0f935bfab89e3f630a7c97e0b1605f0724d85a008bbfb92cb47863a8", "580837af95055670e20d494978f60c7f1458dc4b9e389fc7aa4982b2aca3bce3", "df55c0c49e6c8a83d82dfa1c307d3bf6a20e18721c80d8ec4f1f68dc0a137ced", "5f149c51ce581ba32a285439a806c063ced01ccd4211cd024e6a615b8f216f95", "1eb76b00aeb127b10dd1b7cd4c3edeb4d812b5a658f0feb13e85c4d2b7c6fe06", "7a56ba7c3fb7cbfb5561a46a75d95d7722096b45771ec16e6fa7bbfab0b35dfe", "4bae85ad88c28470f0015246d530adc0cd1778bdd5145c3c6b538ee50c4e04bd", "afd1892e2a7145c99ec0ebe9ded0d3fec21089b277a68d47f45961ec5e39e7e0", "953138885d7b36b0ef79e46030f8e61fd7037fbe5ce9e0a94d728e8c8d7eab86", "de761613ef305e4f628cb6bf97d7b7dc69a9d513dc233630792de97bcda777a6", "3f3087280063d09504c084bbf7fdf984347a72b50d097fd5b086ffabb5b3fb4c", "7d18a94bb1ebfdef4d3e454d2db8cb772f30ca57920dd1e402184a9e598581a0", "a7d6fbdc9126d9f10d10617f49fb9f5474ffe1b229f76b7dd27cebba30eccb5d", "fad0246303618353d1387ec10c09ee991eb6180697ed3470ed9a6b377695203d", "1cf66e09ea51ee5c23df26615a9e7420be2ac8063f28f60a3bc86020e94fe6f3", "8269cdaa153da7c358b0b940791af74d7c651cd4d3f5ed13acfe6d0f2c539e7f", "90d52eaaa60e74bf1c79106113f2599471a902d7b1c39ac1f55b20604f453c09", "9788fd0c09190a3f3d0541f68073a2f44c2fcc45bb97558a7c319f36c25a75b3", "10b68fc44157ecfdae238ee6c1ce0333f906ad04d1a4cb1505c8e35c3c87fbb0", "e5284117fdf3757920475c786e0004cb00ba0932163659a89b36651a01e57394", "403ad51d911e113dcd5f9ff58c94f6d278886a2a4da64c3ceca2083282c92de3",
 	)
 }
+
+func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer {
+	rw := &p2p.MsgPipeRW{}
+	p := p2p.NewPeer(enode.ID{}, "foo", []p2p.Cap{})
+	pp := protocols.NewPeer(p, rw, &protocols.Spec{})
+	bp := &BzzPeer{
+		Peer: pp,
+		BzzAddr: &BzzAddr{
+			OAddr: addr.Bytes(),
+			UAddr: []byte(fmt.Sprintf("%x", addr[:])),
+		},
+	}
+	return NewPeer(bp, kad)
+}

swarm/network/protocol.go

@@ -44,7 +44,7 @@ const (
 // BzzSpec is the spec of the generic swarm handshake
 var BzzSpec = &protocols.Spec{
 	Name:       "bzz",
-	Version:    7,
+	Version:    8,
 	MaxMsgSize: 10 * 1024 * 1024,
 	Messages: []interface{}{
 		HandshakeMsg{},
@@ -54,7 +54,7 @@ var BzzSpec = &protocols.Spec{
 // DiscoverySpec is the spec for the bzz discovery subprotocols
 var DiscoverySpec = &protocols.Spec{
 	Name:       "hive",
-	Version:    6,
+	Version:    8,
 	MaxMsgSize: 10 * 1024 * 1024,
 	Messages: []interface{}{
 		peersMsg{},

swarm/network/protocol_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 const (
-	TestProtocolVersion   = 7
+	TestProtocolVersion   = 8
 	TestProtocolNetworkID = 3
 )
 

swarm/network/simulation/events.go

@@ -20,16 +20,18 @@ import (
 	"context"
 	"sync"
 
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
 )
 
 // PeerEvent is the type of the channel returned by Simulation.PeerEvents.
 type PeerEvent struct {
 	// NodeID is the ID of node that the event is caught on.
 	NodeID enode.ID
+	// PeerID is the ID of the peer node that the event is caught on.
+	PeerID enode.ID
 	// Event is the event that is caught.
-	Event *p2p.PeerEvent
+	Event *simulations.Event
 	// Error is the error that may have happened during event watching.
 	Error error
 }
@@ -37,7 +39,11 @@ type PeerEvent struct {
 // PeerEventsFilter defines a filter on PeerEvents to exclude messages with
 // defined properties. Use PeerEventsFilter methods to set required options.
 type PeerEventsFilter struct {
-	t        *p2p.PeerEventType
+	eventType simulations.EventType
+
+	connUp *bool
+
+	msgReceive *bool
 	protocol   *string
 	msgCode    *uint64
 }
@@ -47,20 +53,48 @@ func NewPeerEventsFilter() *PeerEventsFilter {
 	return &PeerEventsFilter{}
 }
 
-// Type sets the filter to only one peer event type.
+// Connect sets the filter to events when two nodes connect.
-func (f *PeerEventsFilter) Type(t p2p.PeerEventType) *PeerEventsFilter {
+func (f *PeerEventsFilter) Connect() *PeerEventsFilter {
-	f.t = &t
+	f.eventType = simulations.EventTypeConn
+	b := true
+	f.connUp = &b
+	return f
+}
+
+// Drop sets the filter to events when two nodes disconnect.
+func (f *PeerEventsFilter) Drop() *PeerEventsFilter {
+	f.eventType = simulations.EventTypeConn
+	b := false
+	f.connUp = &b
+	return f
+}
+
+// ReceivedMessages sets the filter to only messages that are received.
+func (f *PeerEventsFilter) ReceivedMessages() *PeerEventsFilter {
+	f.eventType = simulations.EventTypeMsg
+	b := true
+	f.msgReceive = &b
+	return f
+}
+
+// SentMessages sets the filter to only messages that are sent.
+func (f *PeerEventsFilter) SentMessages() *PeerEventsFilter {
+	f.eventType = simulations.EventTypeMsg
+	b := false
+	f.msgReceive = &b
 	return f
 }
 
 // Protocol sets the filter to only one message protocol.
 func (f *PeerEventsFilter) Protocol(p string) *PeerEventsFilter {
+	f.eventType = simulations.EventTypeMsg
 	f.protocol = &p
 	return f
 }
 
 // MsgCode sets the filter to only one msg code.
 func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter {
+	f.eventType = simulations.EventTypeMsg
 	f.msgCode = &c
 	return f
 }
@@ -80,19 +114,8 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ...
 		go func(id enode.ID) {
 			defer s.shutdownWG.Done()
 
-			client, err := s.Net.GetNode(id).Client()
+			events := make(chan *simulations.Event)
-			if err != nil {
+			sub := s.Net.Events().Subscribe(events)
-				subsWG.Done()
-				eventC <- PeerEvent{NodeID: id, Error: err}
-				return
-			}
-			events := make(chan *p2p.PeerEvent)
-			sub, err := client.Subscribe(ctx, "admin", events, "peerEvents")
-			if err != nil {
-				subsWG.Done()
-				eventC <- PeerEvent{NodeID: id, Error: err}
-				return
-			}
 			defer sub.Unsubscribe()
 
 			subsWG.Done()
@@ -110,28 +133,55 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ...
 				case <-s.Done():
 					return
 				case e := <-events:
+					// ignore control events
+					if e.Control {
+						continue
+					}
 					match := len(filters) == 0 // if there are no filters match all events
 					for _, f := range filters {
-						if f.t != nil && *f.t != e.Type {
+						if f.eventType == simulations.EventTypeConn && e.Conn != nil {
+							if *f.connUp != e.Conn.Up {
 								continue
 							}
-						if f.protocol != nil && *f.protocol != e.Protocol {
+							// all connection filter parameters matched, break the loop
-							continue
-						}
-						if f.msgCode != nil && e.MsgCode != nil && *f.msgCode != *e.MsgCode {
-							continue
-						}
-						// all filter parameters matched, break the loop
 							match = true
 							break
 						}
+						if f.eventType == simulations.EventTypeMsg && e.Msg != nil {
+							if f.msgReceive != nil && *f.msgReceive != e.Msg.Received {
+								continue
+							}
+							if f.protocol != nil && *f.protocol != e.Msg.Protocol {
+								continue
+							}
+							if f.msgCode != nil && *f.msgCode != e.Msg.Code {
+								continue
+							}
+							// all message filter parameters matched, break the loop
+							match = true
+							break
+						}
+					}
+					var peerID enode.ID
+					switch e.Type {
+					case simulations.EventTypeConn:
+						peerID = e.Conn.One
+						if peerID == id {
+							peerID = e.Conn.Other
+						}
+					case simulations.EventTypeMsg:
+						peerID = e.Msg.One
+						if peerID == id {
+							peerID = e.Msg.Other
+						}
+					}
 					if match {
 						select {
-						case eventC <- PeerEvent{NodeID: id, Event: e}:
+						case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Event: e}:
 						case <-ctx.Done():
 							if err := ctx.Err(); err != nil {
 								select {
-								case eventC <- PeerEvent{NodeID: id, Error: err}:
+								case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Error: err}:
 								case <-s.Done():
 								}
 							}

swarm/network/simulation/example_test.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/network/simulation"
@@ -34,6 +33,10 @@ import (
 // BucketKeyKademlia key. This allows to use WaitTillHealthy to block until
 // all nodes have the their Kadmlias healthy.
 func ExampleSimulation_WaitTillHealthy() {
+
+	log.Error("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
+	return
+
 	sim := simulation.New(map[string]simulation.ServiceFunc{
 		"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
 			addr := network.NewAddr(ctx.Config.Node())
@@ -87,7 +90,7 @@ func ExampleSimulation_PeerEvents() {
 				log.Error("peer event", "err", e.Error)
 				continue
 			}
-			log.Info("peer event", "node", e.NodeID, "peer", e.Event.Peer, "msgcode", e.Event.MsgCode)
+			log.Info("peer event", "node", e.NodeID, "peer", e.PeerID, "type", e.Event.Type)
 		}
 	}()
 }
@@ -100,7 +103,7 @@ func ExampleSimulation_PeerEvents_disconnections() {
 	disconnections := sim.PeerEvents(
 		context.Background(),
 		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+		simulation.NewPeerEventsFilter().Drop(),
 	)
 
 	go func() {
@@ -109,7 +112,7 @@ func ExampleSimulation_PeerEvents_disconnections() {
 				log.Error("peer drop", "err", d.Error)
 				continue
 			}
-			log.Warn("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+			log.Warn("peer drop", "node", d.NodeID, "peer", d.PeerID)
 		}
 	}()
 }
@@ -124,8 +127,8 @@ func ExampleSimulation_PeerEvents_multipleFilters() {
 		context.Background(),
 		sim.NodeIDs(),
 		// Watch when bzz messages 1 and 4 are received.
-		simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(1),
+		simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(1),
-		simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(4),
+		simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(4),
 	)
 
 	go func() {
@@ -134,7 +137,7 @@ func ExampleSimulation_PeerEvents_multipleFilters() {
 				log.Error("bzz message", "err", m.Error)
 				continue
 			}
-			log.Info("bzz message", "node", m.NodeID, "peer", m.Event.Peer)
+			log.Info("bzz message", "node", m.NodeID, "peer", m.PeerID)
 		}
 	}()
 }

swarm/network/simulation/kademlia.go

@@ -33,6 +33,7 @@ var BucketKeyKademlia BucketKey = "kademlia"
 
 // WaitTillHealthy is blocking until the health of all kademlias is true.
 // If error is not nil, a map of kademlia that was found not healthy is returned.
+// TODO: Check correctness since change in kademlia depth calculation logic
 func (s *Simulation) WaitTillHealthy(ctx context.Context, kadMinProxSize int) (ill map[enode.ID]*network.Kademlia, err error) {
 	// Prepare PeerPot map for checking Kademlia health
 	var ppmap map[string]*network.PeerPot

swarm/network/simulation/kademlia_test.go

@@ -28,6 +28,7 @@ import (
 )
 
 func TestWaitTillHealthy(t *testing.T) {
+
 	sim := New(map[string]ServiceFunc{
 		"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
 			addr := network.NewAddr(ctx.Config.Node())

swarm/network/simulation/node_test.go

@@ -160,6 +160,41 @@ func TestAddNodeWithService(t *testing.T) {
 	}
 }
 
+func TestAddNodeMultipleServices(t *testing.T) {
+	sim := New(map[string]ServiceFunc{
+		"noop1": noopServiceFunc,
+		"noop2": noopService2Func,
+	})
+	defer sim.Close()
+
+	id, err := sim.AddNode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n := sim.Net.GetNode(id).Node.(*adapters.SimNode)
+	if n.Service("noop1") == nil {
+		t.Error("service noop1 not found on node")
+	}
+	if n.Service("noop2") == nil {
+		t.Error("service noop2 not found on node")
+	}
+}
+
+func TestAddNodeDuplicateServiceError(t *testing.T) {
+	sim := New(map[string]ServiceFunc{
+		"noop1": noopServiceFunc,
+		"noop2": noopServiceFunc,
+	})
+	defer sim.Close()
+
+	wantErr := "duplicate service: *simulation.noopService"
+	_, err := sim.AddNode()
+	if err.Error() != wantErr {
+		t.Errorf("got error %q, want %q", err, wantErr)
+	}
+}
+
 func TestAddNodes(t *testing.T) {
 	sim := New(noopServiceFuncMap)
 	defer sim.Close()

swarm/network/simulation/simulation.go

@@ -68,6 +68,10 @@ type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Se
 
 // New creates a new Simulation instance with new
 // simulations.Network initialized with provided services.
+// Services map must have unique keys as service names and
+// every ServiceFunc must return a node.Service of the unique type.
+// This restriction is required by node.Node.Start() function
+// which is used to start node.Service returned by ServiceFunc.
 func New(services map[string]ServiceFunc) (s *Simulation) {
 	s = &Simulation{
 		buckets: make(map[enode.ID]*sync.Map),
@@ -76,6 +80,9 @@ func New(services map[string]ServiceFunc) (s *Simulation) {
 
 	adapterServices := make(map[string]adapters.ServiceFunc, len(services))
 	for name, serviceFunc := range services {
+		// Scope this variables correctly
+		// as they will be in the adapterServices[name] function accessed later.
+		name, serviceFunc := name, serviceFunc
 		s.serviceNames = append(s.serviceNames, name)
 		adapterServices[name] = func(ctx *adapters.ServiceContext) (node.Service, error) {
 			b := new(sync.Map)

swarm/network/simulation/simulation_test.go

@@ -205,3 +205,16 @@ func (t *noopService) Start(server *p2p.Server) error {
 func (t *noopService) Stop() error {
 	return nil
 }
+
+// a helper function for most basic noop service
+// of a different type then noopService to test
+// multiple services on one node.
+func noopService2Func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
+	return new(noopService2), nil, nil
+}
+
+// noopService2 is the service that does not do anything
+// but implements node.Service interface.
+type noopService2 struct {
+	noopService
+}

swarm/network/simulations/overlay.go

@@ -64,12 +64,12 @@ func init() {
 
 type Simulation struct {
 	mtx    sync.Mutex
-	stores map[enode.ID]*state.InmemoryStore
+	stores map[enode.ID]state.Store
 }
 
 func NewSimulation() *Simulation {
 	return &Simulation{
-		stores: make(map[enode.ID]*state.InmemoryStore),
+		stores: make(map[enode.ID]state.Store),
 	}
 }
 

swarm/network/stream/common_test.go

@@ -18,7 +18,6 @@ package stream
 
 import (
 	"context"
-	crand "crypto/rand"
 	"errors"
 	"flag"
 	"fmt"
@@ -39,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/pot"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-	mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 	colorable "github.com/mattn/go-colorable"
 )
 
@@ -69,21 +68,6 @@ func init() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 }
 
-func createGlobalStore() (string, *mockdb.GlobalStore, error) {
-	var globalStore *mockdb.GlobalStore
-	globalStoreDir, err := ioutil.TempDir("", "global.store")
-	if err != nil {
-		log.Error("Error initiating global store temp directory!", "err", err)
-		return "", nil, err
-	}
-	globalStore, err = mockdb.NewGlobalStore(globalStoreDir)
-	if err != nil {
-		log.Error("Error initiating global store!", "err", err)
-		return "", nil, err
-	}
-	return globalStoreDir, globalStore, nil
-}
-
 func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
 	// setup
 	addr := network.RandomAddr() // tested peers peer address
@@ -114,7 +98,7 @@ func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest
 
 	delivery := NewDelivery(to, netStore)
 	netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
-	streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions)
+	streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil)
 	teardown := func() {
 		streamer.Close()
 		removeDataDir()
@@ -230,12 +214,7 @@ func generateRandomFile() (string, error) {
 	//generate a random file size between minFileSize and maxFileSize
 	fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
 	log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
-	b := make([]byte, fileSize*1024)
+	b := testutil.RandomBytes(1, fileSize*1024)
-	_, err := crand.Read(b)
-	if err != nil {
-		log.Error("Error generating random file.", "err", err)
-		return "", err
-	}
 	return string(b), nil
 }
 

swarm/network/stream/delivery.go

@@ -169,7 +169,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req *
 	go func() {
 		chunk, err := d.chunkStore.Get(ctx, req.Addr)
 		if err != nil {
-			log.Warn("ChunkStore.Get can not retrieve chunk", "err", err)
+			log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err)
 			return
 		}
 		if req.SkipCheck {
@@ -255,7 +255,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) (
 			}
 			sp = d.getPeer(id)
 			if sp == nil {
-				log.Warn("Delivery.RequestFromPeers: peer not found", "id", id)
+				//log.Warn("Delivery.RequestFromPeers: peer not found", "id", id)
 				return true
 			}
 			spID = &id

swarm/network/stream/delivery_test.go

@@ -19,9 +19,7 @@ package stream
 import (
 	"bytes"
 	"context"
-	crand "crypto/rand"
 	"fmt"
-	"io"
 	"os"
 	"sync"
 	"testing"
@@ -39,6 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network/simulation"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 //Tests initializing a retrieve request
@@ -291,7 +290,7 @@ func TestRequestFromPeers(t *testing.T) {
 		Peer:      protocolsPeer,
 	}, to)
 	to.On(peer)
-	r := NewRegistry(addr.ID(), delivery, nil, nil, nil)
+	r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil)
 
 	// an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished
 	sp := &Peer{
@@ -332,7 +331,7 @@ func TestRequestFromPeersWithLightNode(t *testing.T) {
 		Peer:      protocolsPeer,
 	}, to)
 	to.On(peer)
-	r := NewRegistry(addr.ID(), delivery, nil, nil, nil)
+	r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil)
 	// an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished
 	sp := &Peer{
 		Peer:     protocolsPeer,
@@ -454,6 +453,8 @@ func TestDeliveryFromNodes(t *testing.T) {
 }
 
 func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	sim := simulation.New(map[string]simulation.ServiceFunc{
 		"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
 			node := ctx.Config.Node()
@@ -481,7 +482,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 				SkipCheck: skipCheck,
 				Syncing:   SyncingDisabled,
 				Retrieval: RetrievalEnabled,
-			})
+			}, nil)
 			bucket.Store(bucketKeyRegistry, r)
 
 			fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
@@ -530,7 +531,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 		//now we can actually upload a (random) file to the round-robin store
 		size := chunkCount * chunkSize
 		log.Debug("Storing data to file store")
-		fileHash, wait, err := roundRobinFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
 		// wait until all chunks stored
 		if err != nil {
 			return err
@@ -566,13 +567,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 		disconnections := sim.PeerEvents(
 			context.Background(),
 			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+			simulation.NewPeerEventsFilter().Drop(),
 		)
 
 		go func() {
 			for d := range disconnections {
 				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
 					t.Fatal(d.Error)
 				}
 			}
@@ -656,7 +657,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
 				Syncing:         SyncingDisabled,
 				Retrieval:       RetrievalDisabled,
 				SyncUpdateDelay: 0,
-			})
+			}, nil)
 
 			fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
 			bucket.Store(bucketKeyFileStore, fileStore)
@@ -698,13 +699,13 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
 		disconnections := sim.PeerEvents(
 			context.Background(),
 			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+			simulation.NewPeerEventsFilter().Drop(),
 		)
 
 		go func() {
 			for d := range disconnections {
 				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
 					b.Fatal(d.Error)
 				}
 			}
@@ -719,7 +720,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
 			for i := 0; i < chunkCount; i++ {
 				// create actual size real chunks
 				ctx := context.TODO()
-				hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false)
+				hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false)
 				if err != nil {
 					b.Fatalf("expected no error. got %v", err)
 				}

swarm/network/stream/intervals_test.go

@@ -18,10 +18,8 @@ package stream
 
 import (
 	"context"
-	crand "crypto/rand"
 	"encoding/binary"
 	"fmt"
-	"io"
 	"os"
 	"sync"
 	"testing"
@@ -29,13 +27,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/network/simulation"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestIntervalsLive(t *testing.T) {
@@ -54,6 +52,8 @@ func TestIntervalsLiveAndHistory(t *testing.T) {
 }
 
 func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	nodes := 2
 	chunkCount := dataChunkCount
 	externalStreamName := "externalStream"
@@ -86,7 +86,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 				Retrieval: RetrievalDisabled,
 				Syncing:   SyncingRegisterOnly,
 				SkipCheck: skipCheck,
-			})
+			}, nil)
 			bucket.Store(bucketKeyRegistry, r)
 
 			r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) {
@@ -130,7 +130,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 		fileStore := item.(*storage.FileStore)
 
 		size := chunkCount * chunkSize
-		_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+
+		_, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
 		if err != nil {
 			log.Error("Store error: %v", "err", err)
 			t.Fatal(err)
@@ -154,7 +155,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 		disconnections := sim.PeerEvents(
 			context.Background(),
 			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+			simulation.NewPeerEventsFilter().Drop(),
 		)
 
 		err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
@@ -165,7 +166,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 		go func() {
 			for d := range disconnections {
 				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
 					t.Fatal(d.Error)
 				}
 			}

swarm/network/stream/snapshot_retrieval_test.go

@@ -130,7 +130,7 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no
 		Retrieval:       RetrievalEnabled,
 		Syncing:         SyncingAutoSubscribe,
 		SyncUpdateDelay: 3 * time.Second,
-	})
+	}, nil)
 
 	fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
 	bucket.Store(bucketKeyFileStore, fileStore)
@@ -246,6 +246,7 @@ simulation's `action` function.
 The snapshot should have 'streamer' in its service list.
 */
 func runRetrievalTest(chunkCount int, nodeCount int) error {
+
 	sim := simulation.New(retrievalSimServiceMap)
 	defer sim.Close()
 

swarm/network/stream/snapshot_sync_test.go

@@ -17,9 +17,7 @@ package stream
 
 import (
 	"context"
-	crand "crypto/rand"
 	"fmt"
-	"io"
 	"os"
 	"runtime"
 	"sync"
@@ -29,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
@@ -38,7 +35,9 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/pot"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-	mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 const MaxTimeout = 600
@@ -168,7 +167,7 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic
 		Retrieval:       RetrievalDisabled,
 		Syncing:         SyncingAutoSubscribe,
 		SyncUpdateDelay: 3 * time.Second,
-	})
+	}, nil)
 
 	bucket.Store(bucketKeyRegistry, r)
 
@@ -183,6 +182,8 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic
 }
 
 func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	sim := simulation.New(simServiceMap)
 	defer sim.Close()
 
@@ -211,12 +212,12 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
 	disconnections := sim.PeerEvents(
 		context.Background(),
 		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+		simulation.NewPeerEventsFilter().Drop(),
 	)
 
 	go func() {
 		for d := range disconnections {
-			log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+			log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
 			t.Fatal("unexpected disconnect")
 			cancelSimRun()
 		}
@@ -270,20 +271,9 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio
 
 		// File retrieval check is repeated until all uploaded files are retrieved from all nodes
 		// or until the timeout is reached.
-		var gDir string
+		var globalStore mock.GlobalStorer
-		var globalStore *mockdb.GlobalStore
 		if *useMockStore {
-			gDir, globalStore, err = createGlobalStore()
+			globalStore = mockmem.NewGlobalStore()
-			if err != nil {
-				return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
-			}
-			defer func() {
-				os.RemoveAll(gDir)
-				err := globalStore.Close()
-				if err != nil {
-					log.Error("Error closing global store! %v", "err", err)
-				}
-			}()
 		}
 	REPEAT:
 		for {
@@ -341,6 +331,8 @@ assuming that the snapshot file identifies a healthy
 kademlia network. The snapshot should have 'streamer' in its service list.
 */
 func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) error {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	sim := simulation.New(map[string]simulation.ServiceFunc{
 		"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
 			n := ctx.Config.Node()
@@ -362,7 +354,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
 			r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
 				Retrieval: RetrievalDisabled,
 				Syncing:   SyncingRegisterOnly,
-			})
+			}, nil)
 			bucket.Store(bucketKeyRegistry, r)
 
 			fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
@@ -403,12 +395,12 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
 	disconnections := sim.PeerEvents(
 		context.Background(),
 		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+		simulation.NewPeerEventsFilter().Drop(),
 	)
 
 	go func() {
 		for d := range disconnections {
-			log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+			log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
 			t.Fatal("unexpected disconnect")
 			cancelSimRun()
 		}
@@ -429,7 +421,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
 
 		var subscriptionCount int
 
-		filter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(4)
+		filter := simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("stream").MsgCode(4)
 		eventC := sim.PeerEvents(ctx, nodeIDs, filter)
 
 		for j, node := range nodeIDs {
@@ -478,14 +470,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
 			return err
 		}
 
-		var gDir string
+		var globalStore mock.GlobalStorer
-		var globalStore *mockdb.GlobalStore
 		if *useMockStore {
-			gDir, globalStore, err = createGlobalStore()
+			globalStore = mockmem.NewGlobalStore()
-			if err != nil {
-				return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
-			}
-			defer os.RemoveAll(gDir)
 		}
 		// File retrieval check is repeated until all uploaded files are retrieved from all nodes
 		// or until the timeout is reached.
@@ -603,7 +590,7 @@ func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, lstore *storage.Lo
 	size := chunkSize
 	var rootAddrs []storage.Address
 	for i := 0; i < chunkCount; i++ {
-		rk, wait, err := fileStore.Store(context.TODO(), io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		rk, wait, err := fileStore.Store(context.TODO(), testutil.RandomReader(i, size), int64(size), false)
 		if err != nil {
 			return nil, err
 		}

swarm/network/stream/stream.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"reflect"
 	"sync"
 	"time"
 
@@ -87,6 +88,9 @@ type Registry struct {
 	intervalsStore state.Store
 	autoRetrieval  bool //automatically subscribe to retrieve request stream
 	maxPeerServers int
+	spec           *protocols.Spec   //this protocol's spec
+	balance        protocols.Balance //implements protocols.Balance, for accounting
+	prices         protocols.Prices  //implements protocols.Prices, provides prices to accounting
 }
 
 // RegistryOptions holds optional values for NewRegistry constructor.
@@ -99,7 +103,7 @@ type RegistryOptions struct {
 }
 
 // NewRegistry is Streamer constructor
-func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry {
+func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions, balance protocols.Balance) *Registry {
 	if options == nil {
 		options = &RegistryOptions{}
 	}
@@ -119,7 +123,10 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
 		intervalsStore: intervalsStore,
 		autoRetrieval:  retrieval,
 		maxPeerServers: options.MaxPeerServers,
+		balance:        balance,
 	}
+	streamer.setupSpec()
+
 	streamer.api = NewAPI(streamer)
 	delivery.getPeer = streamer.getPeer
 
@@ -228,6 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
 	return streamer
 }
 
+//we need to construct a spec instance per node instance
+func (r *Registry) setupSpec() {
+	//first create the "bare" spec
+	r.createSpec()
+	//if balance is nil, this node has been started without swap support (swapEnabled flag is false)
+	if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() {
+		//swap is enabled, so setup the hook
+		r.spec.Hook = protocols.NewAccounting(r.balance, r.prices)
+	}
+}
+
 // RegisterClient registers an incoming streamer constructor
 func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) {
 	r.clientMu.Lock()
@@ -492,7 +510,7 @@ func (r *Registry) updateSyncing() {
 }
 
 func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error {
-	peer := protocols.NewPeer(p, rw, Spec)
+	peer := protocols.NewPeer(p, rw, r.spec)
 	bp := network.NewBzzPeer(peer)
 	np := network.NewPeer(bp, r.delivery.kad)
 	r.delivery.kad.On(np)
@@ -716,8 +734,16 @@ func (c *clientParams) clientCreated() {
 	close(c.clientCreatedC)
 }
 
-// Spec is the spec of the streamer protocol
+//GetSpec returns the streamer spec to callers
-var Spec = &protocols.Spec{
+//This used to be a global variable but for simulations with
+//multiple nodes its fields (notably the Hook) would be overwritten
+func (r *Registry) GetSpec() *protocols.Spec {
+	return r.spec
+}
+
+func (r *Registry) createSpec() {
+	// Spec is the spec of the streamer protocol
+	var spec = &protocols.Spec{
 		Name:       "stream",
 		Version:    8,
 		MaxMsgSize: 10 * 1024 * 1024,
@@ -734,17 +760,17 @@ var Spec = &protocols.Spec{
 			QuitMsg{},
 			ChunkDeliveryMsgSyncing{},
 		},
+	}
+	r.spec = spec
 }
 
 func (r *Registry) Protocols() []p2p.Protocol {
 	return []p2p.Protocol{
 		{
-			Name:    Spec.Name,
+			Name:    r.spec.Name,
-			Version: Spec.Version,
+			Version: r.spec.Version,
-			Length:  Spec.Length(),
+			Length:  r.spec.Length(),
 			Run:     r.runProtocol,
-			// NodeInfo: ,
-			// PeerInfo: ,
 		},
 	}
 }

swarm/network/stream/syncer_test.go

@@ -18,9 +18,7 @@ package stream
 
 import (
 	"context"
-	crand "crypto/rand"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"math"
 	"os"
@@ -30,7 +28,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/swarm/log"
@@ -38,7 +35,9 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network/simulation"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-	mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 const dataChunkCount = 200
@@ -50,7 +49,7 @@ func TestSyncerSimulation(t *testing.T) {
 	testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1)
 }
 
-func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
+func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
 	address := common.BytesToAddress(id.Bytes())
 	mockStore := globalStore.NewNodeStore(address)
 	params := storage.NewDefaultLocalStoreParams()
@@ -69,11 +68,12 @@ func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network
 }
 
 func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	sim := simulation.New(map[string]simulation.ServiceFunc{
 		"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
 			var store storage.ChunkStore
-			var globalStore *mockdb.GlobalStore
+			var datadir string
-			var gDir, datadir string
 
 			node := ctx.Config.Node()
 			addr := network.NewAddr(node)
@@ -81,11 +81,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 			addr.OAddr[0] = byte(0)
 
 			if *useMockStore {
-				gDir, globalStore, err = createGlobalStore()
+				store, datadir, err = createMockStore(mockmem.NewGlobalStore(), node.ID(), addr)
-				if err != nil {
-					return nil, nil, fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
-				}
-				store, datadir, err = createMockStore(globalStore, node.ID(), addr)
 			} else {
 				store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
 			}
@@ -96,13 +92,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 			cleanup = func() {
 				store.Close()
 				os.RemoveAll(datadir)
-				if *useMockStore {
-					err := globalStore.Close()
-					if err != nil {
-						log.Error("Error closing global store! %v", "err", err)
-					}
-					os.RemoveAll(gDir)
-				}
 			}
 			localStore := store.(*storage.LocalStore)
 			netStore, err := storage.NewNetStore(localStore, nil)
@@ -120,7 +109,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 				Retrieval: RetrievalDisabled,
 				Syncing:   SyncingAutoSubscribe,
 				SkipCheck: skipCheck,
-			})
+			}, nil)
 
 			fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
 			bucket.Store(bucketKeyFileStore, fileStore)
@@ -152,13 +141,13 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 		disconnections := sim.PeerEvents(
 			context.Background(),
 			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+			simulation.NewPeerEventsFilter().Drop(),
 		)
 
 		go func() {
 			for d := range disconnections {
 				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
 					t.Fatal(d.Error)
 				}
 			}
@@ -183,7 +172,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 				}
 				fileStore := item.(*storage.FileStore)
 				size := chunkCount * chunkSize
-				_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+				_, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false)
 				if err != nil {
 					t.Fatal(err.Error())
 				}
@@ -245,3 +234,170 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 		t.Fatal(result.Error)
 	}
 }
+
+//TestSameVersionID just checks that if the version is not changed,
+//then streamer peers see each other
+func TestSameVersionID(t *testing.T) {
+	//test version ID
+	v := uint(1)
+	sim := simulation.New(map[string]simulation.ServiceFunc{
+		"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
+			var store storage.ChunkStore
+			var datadir string
+
+			node := ctx.Config.Node()
+			addr := network.NewAddr(node)
+
+			store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
+			if err != nil {
+				return nil, nil, err
+			}
+			bucket.Store(bucketKeyStore, store)
+			cleanup = func() {
+				store.Close()
+				os.RemoveAll(datadir)
+			}
+			localStore := store.(*storage.LocalStore)
+			netStore, err := storage.NewNetStore(localStore, nil)
+			if err != nil {
+				return nil, nil, err
+			}
+			bucket.Store(bucketKeyDB, netStore)
+			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+			delivery := NewDelivery(kad, netStore)
+			netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
+
+			bucket.Store(bucketKeyDelivery, delivery)
+
+			r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+				Retrieval: RetrievalDisabled,
+				Syncing:   SyncingAutoSubscribe,
+			}, nil)
+			//assign to each node the same version ID
+			r.spec.Version = v
+
+			bucket.Store(bucketKeyRegistry, r)
+
+			return r, cleanup, nil
+
+		},
+	})
+	defer sim.Close()
+
+	//connect just two nodes
+	log.Info("Adding nodes to simulation")
+	_, err := sim.AddNodesAndConnectChain(2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Info("Starting simulation")
+	ctx := context.Background()
+	//make sure they have time to connect
+	time.Sleep(200 * time.Millisecond)
+	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+		//get the pivot node's filestore
+		nodes := sim.UpNodeIDs()
+
+		item, ok := sim.NodeItem(nodes[0], bucketKeyRegistry)
+		if !ok {
+			return fmt.Errorf("No filestore")
+		}
+		registry := item.(*Registry)
+
+		//the peers should connect, thus getting the peer should not return nil
+		if registry.getPeer(nodes[1]) == nil {
+			t.Fatal("Expected the peer to not be nil, but it is")
+		}
+		return nil
+	})
+	if result.Error != nil {
+		t.Fatal(result.Error)
+	}
+	log.Info("Simulation ended")
+}
+
+//TestDifferentVersionID proves that if the streamer protocol version doesn't match,
+//then the peers are not connected at streamer level
+func TestDifferentVersionID(t *testing.T) {
+	//create a variable to hold the version ID
+	v := uint(0)
+	sim := simulation.New(map[string]simulation.ServiceFunc{
+		"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
+			var store storage.ChunkStore
+			var datadir string
+
+			node := ctx.Config.Node()
+			addr := network.NewAddr(node)
+
+			store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
+			if err != nil {
+				return nil, nil, err
+			}
+			bucket.Store(bucketKeyStore, store)
+			cleanup = func() {
+				store.Close()
+				os.RemoveAll(datadir)
+			}
+			localStore := store.(*storage.LocalStore)
+			netStore, err := storage.NewNetStore(localStore, nil)
+			if err != nil {
+				return nil, nil, err
+			}
+			bucket.Store(bucketKeyDB, netStore)
+			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+			delivery := NewDelivery(kad, netStore)
+			netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
+
+			bucket.Store(bucketKeyDelivery, delivery)
+
+			r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+				Retrieval: RetrievalDisabled,
+				Syncing:   SyncingAutoSubscribe,
+			}, nil)
+
+			//increase the version ID for each node
+			v++
+			r.spec.Version = v
+
+			bucket.Store(bucketKeyRegistry, r)
+
+			return r, cleanup, nil
+
+		},
+	})
+	defer sim.Close()
+
+	//connect the nodes
+	log.Info("Adding nodes to simulation")
+	_, err := sim.AddNodesAndConnectChain(2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Info("Starting simulation")
+	ctx := context.Background()
+	//make sure they have time to connect
+	time.Sleep(200 * time.Millisecond)
+	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+		//get the pivot node's filestore
+		nodes := sim.UpNodeIDs()
+
+		item, ok := sim.NodeItem(nodes[0], bucketKeyRegistry)
+		if !ok {
+			return fmt.Errorf("No filestore")
+		}
+		registry := item.(*Registry)
+
+		//getting the other peer should fail due to the different version numbers
+		if registry.getPeer(nodes[1]) != nil {
+			t.Fatal("Expected the peer to be nil, but it is not")
+		}
+		return nil
+	})
+	if result.Error != nil {
+		t.Fatal(result.Error)
+	}
+	log.Info("Simulation ended")
+
+}

swarm/network/stream/visualized_snapshot_sync_sim_test.go

@@ -84,6 +84,8 @@ func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc)
 
 //This test requests bogus hashes into the network
 func TestNonExistingHashesWithServer(t *testing.T) {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	nodeCount, _, sim := setupSim(retrievalSimServiceMap)
 	defer sim.Close()
 
@@ -143,6 +145,7 @@ func sendSimTerminatedEvent(sim *simulation.Simulation) {
 //can visualize messages like SendOfferedMsg, WantedHashesMsg, DeliveryMsg
 func TestSnapshotSyncWithServer(t *testing.T) {
 
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	nodeCount, chunkCount, sim := setupSim(simServiceMap)
 	defer sim.Close()
 

swarm/network_test.go

@@ -259,6 +259,8 @@ type testSwarmNetworkOptions struct {
 //  - May wait for Kademlia on every node to be healthy.
 //  - Checking if a file is retrievable from all nodes.
 func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwarmNetworkStep) {
+
+	t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
 	if o == nil {
 		o = new(testSwarmNetworkOptions)
 	}
@@ -335,7 +337,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa
 
 		result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
 			nodeIDs := sim.UpNodeIDs()
-			shuffle(len(nodeIDs), func(i, j int) {
+			rand.Shuffle(len(nodeIDs), func(i, j int) {
 				nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i]
 			})
 			for _, id := range nodeIDs {
@@ -404,7 +406,7 @@ func retrieve(
 	nodeStatusM *sync.Map,
 	totalFoundCount *uint64,
 ) (missing uint64) {
-	shuffle(len(files), func(i, j int) {
+	rand.Shuffle(len(files), func(i, j int) {
 		files[i], files[j] = files[j], files[i]
 	})
 
@@ -499,32 +501,3 @@ func retrieve(
 
 	return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount)
 }
-
-// Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333
-//
-// Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped.
-//
-// shuffle pseudo-randomizes the order of elements.
-// n is the number of elements. Shuffle panics if n < 0.
-// swap swaps the elements with indexes i and j.
-func shuffle(n int, swap func(i, j int)) {
-	if n < 0 {
-		panic("invalid argument to Shuffle")
-	}
-
-	// Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
-	// Shuffle really ought not be called with n that doesn't fit in 32 bits.
-	// Not only will it take a very long time, but with 2³¹! possible permutations,
-	// there's no way that any PRNG can have a big enough internal state to
-	// generate even a minuscule percentage of the possible permutations.
-	// Nevertheless, the right API signature accepts an int n, so handle it as best we can.
-	i := n - 1
-	for ; i > 1<<31-1-1; i-- {
-		j := int(rand.Int63n(int64(i + 1)))
-		swap(i, j)
-	}
-	for ; i > 0; i-- {
-		j := int(rand.Int31n(int32(i + 1)))
-		swap(i, j)
-	}
-}