Merge branch 'master' into graphql

.github/CODEOWNERS

@@ -9,23 +9,26 @@ les/                            @zsfelfoldi
 light/                          @zsfelfoldi
 mobile/                         @karalabe
 p2p/                            @fjl @zsfelfoldi
+p2p/simulations                 @lmars
+p2p/protocols                   @zelig
+swarm/api/http                  @justelad
 swarm/bmt                       @zelig
 swarm/dev                       @lmars
 swarm/fuse                      @jmozah @holisticode
 swarm/grafana_dashboards        @nonsense
 swarm/metrics                   @nonsense @holisticode
 swarm/multihash                 @nolash
-swarm/network/bitvector         @zelig @janos @gbalint
-swarm/network/priorityqueue     @zelig @janos @gbalint
-swarm/network/simulations       @zelig
-swarm/network/stream            @janos @zelig @gbalint @holisticode @justelad
+swarm/network/bitvector         @zelig @janos 
+swarm/network/priorityqueue     @zelig @janos 
+swarm/network/simulations       @zelig @janos
+swarm/network/stream            @janos @zelig @holisticode @justelad
 swarm/network/stream/intervals  @janos
 swarm/network/stream/testing    @zelig
 swarm/pot                       @zelig
 swarm/pss                       @nolash @zelig @nonsense
 swarm/services                  @zelig
 swarm/state                     @justelad
-swarm/storage/encryption        @gbalint @zelig @nagydani
+swarm/storage/encryption        @zelig @nagydani
 swarm/storage/mock              @janos
 swarm/storage/feed              @nolash @jpeletier
 swarm/testutil                  @lmars

.travis.yml

@@ -148,7 +148,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go

README.md

@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 

accounts/hd.go

@@ -30,8 +30,8 @@ import (
 var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
-// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
-// at m/44'/60'/0'/1, etc.
+// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
+// at m/44'/60'/0'/0/1, etc.
 var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
 
 // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

accounts/usbwallet/ledger.go

@@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

accounts/usbwallet/trezor.go

@@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
+  - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

cmd/bootnode/main.go

@@ -38,7 +38,7 @@ func main() {
 	var (
 		listenAddr  = flag.String("addr", ":30301", "listen address")
 		genKey      = flag.String("genkey", "", "generate a node key")
-		writeAddr   = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit")
+		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -86,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey))
+		fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
 		os.Exit(0)
 	}
 
@@ -119,16 +119,17 @@ func main() {
 	}
 
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
+		db, _ := enode.OpenDB("")
+		ln := enode.NewLocalNode(db, nodeKey)
 		cfg := discover.Config{
 			PrivateKey:  nodeKey,
-			AnnounceAddr: realaddr,
 			NetRestrict: restrictList,
 		}
-		if _, err := discover.ListenUDP(conn, cfg); err != nil {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/clef/README.md

@@ -91,7 +91,7 @@ invoking methods with the following info:
   * [x] Version info about the signer
   * [x] Address of API (http/ipc)
   * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
 * [ ] `account_bulkSignTransactions([] transactions)` should
    * only exist if enabled via config/flag
@@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
 expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
 Some of these call can require user interaction. Clients must be aware that responses
-may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
 The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
 that requests have any authority.
@@ -862,7 +862,7 @@ A UI should conform to the following rules.
 * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
 * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
   * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
 along with the UI.
 
 

cmd/clef/main.go

@@ -157,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
 
-	addCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(addCredential),
-		Name:      "addpw",
+	setCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(setCredential),
+		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
-		ArgsUsage: "<address> <password>",
+		ArgsUsage: "<address>",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			configdirFlag,
 			signerSecretFlag,
 		},
 		Description: `
-The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+		The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
 `,
 	}
@@ -200,7 +200,7 @@ func init() {
 		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
 
 }
 func main() {
@@ -293,14 +293,17 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
-func addCredential(ctx *cli.Context) error {
+func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires at leaste one argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument.")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
 
+	address := ctx.Args().First()
+	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+
 	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
@@ -311,13 +314,8 @@ func addCredential(ctx *cli.Context) error {
 
 	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	key := ctx.Args().First()
-	value := ""
-	if len(ctx.Args()) > 1 {
-		value = ctx.Args().Get(1)
-	}
-	pwStorage.Put(key, value)
-	log.Info("Credential store updated", "key", key)
+	pwStorage.Put(address, password)
+	log.Info("Credential store updated", "key", address)
 	return nil
 }
 

cmd/evm/json_logger.go

@@ -52,6 +52,7 @@ func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cos
 		MemorySize:    memory.Len(),
 		Storage:       nil,
 		Depth:         depth,
+		RefundCounter: env.StateDB.GetRefund(),
 		Err:           err,
 	}
 	if !l.cfg.DisableMemory {

cmd/geth/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -90,6 +89,7 @@ var (
 		utils.LightKDFFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
+		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
 		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,
@@ -212,8 +212,6 @@ func init() {
 	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-
 		logdir := ""
 		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
 			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")

cmd/geth/usage.go

@@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
 			utils.TrieCacheGenFlag,
 		},

cmd/swarm/access.go

@@ -29,7 +29,65 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-var salt = make([]byte, 32)
+var (
+	salt          = make([]byte, 32)
+	accessCommand = cli.Command{
+		CustomHelpTemplate: helpTemplate,
+		Name:               "access",
+		Usage:              "encrypts a reference and embeds it into a root manifest",
+		ArgsUsage:          "<ref>",
+		Description:        "encrypts a reference and embeds it into a root manifest",
+		Subcommands: []cli.Command{
+			{
+				CustomHelpTemplate: helpTemplate,
+				Name:               "new",
+				Usage:              "encrypts a reference and embeds it into a root manifest",
+				ArgsUsage:          "<ref>",
+				Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+				Subcommands: []cli.Command{
+					{
+						Action:             accessNewPass,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+						},
+						Name:        "pass",
+						Usage:       "encrypts a reference with a password and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewPK,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+							SwarmAccessGrantKeyFlag,
+						},
+						Name:        "pk",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewACT,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							SwarmAccessGrantKeysFlag,
+							SwarmDryRunFlag,
+							utils.PasswordFileFlag,
+						},
+						Name:        "act",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+				},
+			},
+		},
+	}
+)
 
 func init() {
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
@@ -56,6 +114,9 @@ func accessNewPass(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {
@@ -89,6 +150,9 @@ func accessNewPK(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {

cmd/swarm/access_test.go

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !windows
-
 package main
 
 import (
@@ -28,6 +26,7 @@ import (
 	gorand "math/rand"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -37,7 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -48,22 +47,41 @@ const (
 
 var DefaultCurve = crypto.S256()
 
-// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
+func TestACT(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"Password", testPassword},
+		{"PK", testPK},
+		{"ACTWithoutBogus", testACTWithoutBogus},
+		{"ACTWithBogus", testACTWithBogus},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testPassword tests for the correct creation of an ACT manifest protected by a password.
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
 // is then fetched through 2nd node. since the tested code is not key-aware - we can just
 // fetch from the 2nd node using HTTP BasicAuth
-func TestAccessPassword(t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
-	defer srv.Close()
-
+func testPassword(t *testing.T) {
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
 		"--bzzapi",
-		srv.URL, //it doesn't matter through which node we upload content
+		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
 		dataFilename)
@@ -137,16 +155,17 @@ func TestAccessPassword(t *testing.T) {
 	if a.Publisher != "" {
 		t.Fatal("should be empty")
 	}
-	client := swarm.NewClient(srv.URL)
+
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	httpClient := &http.Client{}
+	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
 
-	url := srv.URL + "/" + "bzz:/" + hash
+	httpClient := &http.Client{}
 	response, err := httpClient.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -188,7 +207,7 @@ func TestAccessPassword(t *testing.T) {
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
 		"--bzzapi",
-		srv.URL,
+		cluster.Nodes[0].URL,
 		"down",
 		"bzz:/"+hash,
 		tmp,
@@ -202,16 +221,12 @@ func TestAccessPassword(t *testing.T) {
 	up.ExpectExit()
 }
 
-// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
+// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
 // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
 // the test will fail if the proxy's given private key is not granted on the ACT.
-func TestAccessPK(t *testing.T) {
-	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 2)
-	defer cluster.Shutdown()
-
+func testPK(t *testing.T) {
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
@@ -317,7 +332,7 @@ func TestAccessPK(t *testing.T) {
 	if a.Publisher != pkComp {
 		t.Fatal("publisher key did not match")
 	}
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
@@ -343,29 +358,24 @@ func TestAccessPK(t *testing.T) {
 	}
 }
 
-// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
-func TestAccessACT(t *testing.T) {
-	testAccessACT(t, 0)
+// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
+func testACTWithoutBogus(t *testing.T) {
+	testACT(t, 0)
 }
 
-// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
-func TestAccessACTScale(t *testing.T) {
-	testAccessACT(t, 1000)
+// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
+func testACTWithBogus(t *testing.T) {
+	testACT(t, 100)
 }
 
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
 // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
 // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
 // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
 // the publisher uploads through one of the nodes then disappears.
-func testAccessACT(t *testing.T, bogusEntries int) {
-	// Setup Swarm and upload a test file to it
-	const clusterSize = 3
-	cluster := newTestCluster(t, clusterSize)
-	defer cluster.Shutdown()
-
+func testACT(t *testing.T, bogusEntries int) {
 	var uploadThroughNode = cluster.Nodes[0]
-	client := swarm.NewClient(uploadThroughNode.URL)
+	client := swarmapi.NewClient(uploadThroughNode.URL)
 
 	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
 	nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)

cmd/swarm/config.go

@@ -80,6 +80,7 @@ const (
 	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
 	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 

cmd/swarm/db.go

@@ -29,6 +29,48 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var dbCommand = cli.Command{
+	Name:               "db",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "manage the local chunk database",
+	ArgsUsage:          "db COMMAND",
+	Description:        "Manage the local chunk database",
+	Subcommands: []cli.Command{
+		{
+			Action:             dbExport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "export",
+			Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `
+Export a local chunk database as a tar archive (use - to send to stdout).
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The export may be quite large, consider piping the output through the Unix
+pv(1) tool to get a progress bar:
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
+`,
+		},
+		{
+			Action:             dbImport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "import",
+			Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
+
+    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The import may be quite large, consider piping the input through the Unix
+pv(1) tool to get a progress bar:
+
+    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
+		},
+	},
+}
+
 func dbExport(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) != 3 {

cmd/swarm/download.go

@@ -28,6 +28,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var downloadCommand = cli.Command{
+	Action:      download,
+	Name:        "down",
+	Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
+	Usage:       "downloads a swarm manifest or a file inside a manifest",
+	ArgsUsage:   " <uri> [<dir>]",
+	Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
+}
+
 func download(ctx *cli.Context) {
 	log.Debug("downloading content using swarm down")
 	args := ctx.Args()

cmd/swarm/export_test.go

@@ -19,9 +19,7 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"runtime"
@@ -29,6 +27,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // TestCLISwarmExportImport perform the following test:
@@ -44,12 +43,13 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 	cluster := newTestCluster(t, 1)
 
-	// generate random 10mb file
-	f, cleanup := generateRandomFile(t, 10000000)
-	defer cleanup()
+	// generate random 1mb file
+	content := testutil.RandomBytes(1, 1000000)
+	fileName := testutil.TempFileWithContent(t, string(content))
+	defer os.Remove(fileName)
 
 	// upload the file with 'swarm up' and expect a hash
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
 	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
 	up.ExpectExit()
 	hash := matches[0]
@@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 
 	// compare downloaded file with the generated random file
-	mustEqualFiles(t, f, res.Body)
+	mustEqualFiles(t, bytes.NewReader(content), res.Body)
 }
 
 func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
 		t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
 	}
 }
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	// write 10mb random data to file
-	buf := make([]byte, 10000000)
-	_, err = rand.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
-}

cmd/swarm/feeds.go

@@ -31,6 +31,68 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var feedCommand = cli.Command{
+	CustomHelpTemplate: helpTemplate,
+	Name:               "feed",
+	Usage:              "(Advanced) Create and update Swarm Feeds",
+	ArgsUsage:          "<create|update|info>",
+	Description:        "Works with Swarm Feeds",
+	Subcommands: []cli.Command{
+		{
+			Action:             feedCreateManifest,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "create",
+			Usage:              "creates and publishes a new feed manifest",
+			Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+			Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+		{
+			Action:             feedUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "updates the content of an existing Swarm Feed",
+			ArgsUsage:          "<0x Hex data>",
+			Description: `publishes a new update on the specified topic
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the feed,
+					instead of using --topic / --name
+					`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
+		},
+		{
+			Action:             feedInfo,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "info",
+			Usage:              "obtains information about an existing Swarm feed",
+			Description: `obtains information about an existing Swarm feed
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the feed`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+	},
+}
+
 func NewGenericSigner(ctx *cli.Context) feed.Signer {
 	return feed.NewGenericSigner(getPrivKey(ctx))
 }

cmd/swarm/feeds_test.go

@@ -20,49 +20,35 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestCLIFeedUpdate(t *testing.T) {
-
-	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+	srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
 		return swarmhttp.NewServer(api, "")
 	}, nil)
 	log.Info("starting a test swarm server")
 	defer srv.Close()
 
 	// create a private key file for signing
-	pkfile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer pkfile.Close()
-	defer os.Remove(pkfile.Name())
-
 	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
 	privKey, _ := crypto.HexToECDSA(privkeyHex)
 	address := crypto.PubkeyToAddress(privKey.PublicKey)
 
-	// save the private key to a file
-	_, err = io.WriteString(pkfile, privkeyHex)
-	if err != nil {
-		t.Fatal(err)
-	}
+	pkFileName := testutil.TempFileWithContent(t, privkeyHex)
+	defer os.Remove(pkFileName)
 
 	// compose a topic. We'll be doing quotes about Miguel de Cervantes
 	var topic feed.Topic
@@ -76,7 +62,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	flags := []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "update",
 		"--topic", topic.Hex(),
 		"--name", name,
@@ -89,13 +75,10 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	// now try to get the update using the client
 	client := swarm.NewClient(srv.URL)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// build the same topic as before, this time
 	// we use NewTopic to create a topic automatically.
-	topic, err = feed.NewTopic(name, subject)
+	topic, err := feed.NewTopic(name, subject)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +136,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 	// test publishing a manifest
 	flags = []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "create",
 		"--topic", topic.Hex(),
 	}

cmd/swarm/flags.go

@@ -0,0 +1,179 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command feed allows the user to create and update signed Swarm feeds
+package main
+
+import cli "gopkg.in/urfave/cli.v1"
+
+var (
+	ChequebookAddrFlag = cli.StringFlag{
+		Name:   "chequebook",
+		Usage:  "chequebook contract address",
+		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
+	}
+	SwarmAccountFlag = cli.StringFlag{
+		Name:   "bzzaccount",
+		Usage:  "Swarm account key file",
+		EnvVar: SWARM_ENV_ACCOUNT,
+	}
+	SwarmListenAddrFlag = cli.StringFlag{
+		Name:   "httpaddr",
+		Usage:  "Swarm HTTP API listening interface",
+		EnvVar: SWARM_ENV_LISTEN_ADDR,
+	}
+	SwarmPortFlag = cli.StringFlag{
+		Name:   "bzzport",
+		Usage:  "Swarm local http api port",
+		EnvVar: SWARM_ENV_PORT,
+	}
+	SwarmNetworkIdFlag = cli.IntFlag{
+		Name:   "bzznetworkid",
+		Usage:  "Network identifier (integer, default 3=swarm testnet)",
+		EnvVar: SWARM_ENV_NETWORK_ID,
+	}
+	SwarmSwapEnabledFlag = cli.BoolFlag{
+		Name:   "swap",
+		Usage:  "Swarm SWAP enabled (default false)",
+		EnvVar: SWARM_ENV_SWAP_ENABLE,
+	}
+	SwarmSwapAPIFlag = cli.StringFlag{
+		Name:   "swap-api",
+		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
+		EnvVar: SWARM_ENV_SWAP_API,
+	}
+	SwarmSyncDisabledFlag = cli.BoolTFlag{
+		Name:   "nosync",
+		Usage:  "Disable swarm syncing",
+		EnvVar: SWARM_ENV_SYNC_DISABLE,
+	}
+	SwarmSyncUpdateDelay = cli.DurationFlag{
+		Name:   "sync-update-delay",
+		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
+		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
+	}
+	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
+		Name:   "max-stream-peer-servers",
+		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
+		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
+		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
+	}
+	SwarmLightNodeEnabled = cli.BoolFlag{
+		Name:   "lightnode",
+		Usage:  "Enable Swarm LightNode (default false)",
+		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
+	}
+	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
+		Name:   "delivery-skip-check",
+		Usage:  "Skip chunk delivery check (default false)",
+		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
+	}
+	EnsAPIFlag = cli.StringSliceFlag{
+		Name:   "ens-api",
+		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
+		EnvVar: SWARM_ENV_ENS_API,
+	}
+	SwarmApiFlag = cli.StringFlag{
+		Name:  "bzzapi",
+		Usage: "Specifies the Swarm HTTP endpoint to connect to",
+		Value: "http://127.0.0.1:8500",
+	}
+	SwarmRecursiveFlag = cli.BoolFlag{
+		Name:  "recursive",
+		Usage: "Upload directories recursively",
+	}
+	SwarmWantManifestFlag = cli.BoolTFlag{
+		Name:  "manifest",
+		Usage: "Automatic manifest upload (default true)",
+	}
+	SwarmUploadDefaultPath = cli.StringFlag{
+		Name:  "defaultpath",
+		Usage: "path to file served for empty url path (none)",
+	}
+	SwarmAccessGrantKeyFlag = cli.StringFlag{
+		Name:  "grant-key",
+		Usage: "grants a given public key access to an ACT",
+	}
+	SwarmAccessGrantKeysFlag = cli.StringFlag{
+		Name:  "grant-keys",
+		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
+	}
+	SwarmUpFromStdinFlag = cli.BoolFlag{
+		Name:  "stdin",
+		Usage: "reads data to be uploaded from stdin",
+	}
+	SwarmUploadMimeType = cli.StringFlag{
+		Name:  "mime",
+		Usage: "Manually specify MIME type",
+	}
+	SwarmEncryptedFlag = cli.BoolFlag{
+		Name:  "encrypt",
+		Usage: "use encrypted upload",
+	}
+	SwarmAccessPasswordFlag = cli.StringFlag{
+		Name:   "password",
+		Usage:  "Password",
+		EnvVar: SWARM_ACCESS_PASSWORD,
+	}
+	SwarmDryRunFlag = cli.BoolFlag{
+		Name:  "dry-run",
+		Usage: "dry-run",
+	}
+	CorsStringFlag = cli.StringFlag{
+		Name:   "corsdomain",
+		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
+		EnvVar: SWARM_ENV_CORS,
+	}
+	SwarmStorePath = cli.StringFlag{
+		Name:   "store.path",
+		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
+		EnvVar: SWARM_ENV_STORE_PATH,
+	}
+	SwarmStoreCapacity = cli.Uint64Flag{
+		Name:   "store.size",
+		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
+		EnvVar: SWARM_ENV_STORE_CAPACITY,
+	}
+	SwarmStoreCacheCapacity = cli.UintFlag{
+		Name:   "store.cache.size",
+		Usage:  "Number of recent chunks cached in memory (default 5000)",
+		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
+	}
+	SwarmCompressedFlag = cli.BoolFlag{
+		Name:  "compressed",
+		Usage: "Prints encryption keys in compressed form",
+	}
+	SwarmFeedNameFlag = cli.StringFlag{
+		Name:  "name",
+		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
+	}
+	SwarmFeedTopicFlag = cli.StringFlag{
+		Name:  "topic",
+		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
+	}
+	SwarmFeedDataOnCreateFlag = cli.StringFlag{
+		Name:  "data",
+		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
+	}
+	SwarmFeedManifestFlag = cli.StringFlag{
+		Name:  "manifest",
+		Usage: "Refers to the feed through a manifest",
+	}
+	SwarmFeedUserFlag = cli.StringFlag{
+		Name:  "user",
+		Usage: "Indicates the user who updates the feed",
+	}
+)

cmd/swarm/fs.go

@@ -30,6 +30,43 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var fsCommand = cli.Command{
+	Name:               "fs",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "perform FUSE operations",
+	ArgsUsage:          "fs COMMAND",
+	Description:        "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
+	Subcommands: []cli.Command{
+		{
+			Action:             mount,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "mount",
+			Flags:              []cli.Flag{utils.IPCPathFlag},
+			Usage:              "mount a swarm hash to a mount point",
+			ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
+			Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+		{
+			Action:             unmount,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "unmount",
+			Flags:              []cli.Flag{utils.IPCPathFlag},
+			Usage:              "unmount a swarmfs mount",
+			ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
+			Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+		{
+			Action:             listMounts,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "list",
+			Flags:              []cli.Flag{utils.IPCPathFlag},
+			Usage:              "list swarmfs mounts",
+			ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
+			Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+	},
+}
+
 func mount(cliContext *cli.Context) {
 	args := cliContext.Args()
 	if len(args) < 2 {

cmd/swarm/fs_test.go

@@ -29,14 +29,8 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 )
 
-func init() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
-}
-
 type testFile struct {
 	filePath string
 	content  string
@@ -80,6 +74,9 @@ func TestCLISwarmFs(t *testing.T) {
 		t.Fatal(err)
 	}
 	dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	dummyContent := "somerandomtestcontentthatshouldbeasserted"
 	dirs := []string{

cmd/swarm/hash.go

@@ -27,6 +27,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var hashCommand = cli.Command{
+	Action:             hash,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "hash",
+	Usage:              "print the swarm hash of a file or directory",
+	ArgsUsage:          "<file>",
+	Description:        "Prints the swarm hash of file or directory",
+}
+
 func hash(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 1 {

cmd/swarm/list.go

@@ -27,6 +27,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var listCommand = cli.Command{
+	Action:             list,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "ls",
+	Usage:              "list files and directories contained in a manifest",
+	ArgsUsage:          "<manifest> [<prefix>]",
+	Description:        "Lists files and directories contained in a manifest",
+}
+
 func list(ctx *cli.Context) {
 	args := ctx.Args()
 

cmd/swarm/main.go

@@ -70,165 +70,6 @@ var (
 	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
 )
 
-var (
-	ChequebookAddrFlag = cli.StringFlag{
-		Name:   "chequebook",
-		Usage:  "chequebook contract address",
-		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
-	}
-	SwarmAccountFlag = cli.StringFlag{
-		Name:   "bzzaccount",
-		Usage:  "Swarm account key file",
-		EnvVar: SWARM_ENV_ACCOUNT,
-	}
-	SwarmListenAddrFlag = cli.StringFlag{
-		Name:   "httpaddr",
-		Usage:  "Swarm HTTP API listening interface",
-		EnvVar: SWARM_ENV_LISTEN_ADDR,
-	}
-	SwarmPortFlag = cli.StringFlag{
-		Name:   "bzzport",
-		Usage:  "Swarm local http api port",
-		EnvVar: SWARM_ENV_PORT,
-	}
-	SwarmNetworkIdFlag = cli.IntFlag{
-		Name:   "bzznetworkid",
-		Usage:  "Network identifier (integer, default 3=swarm testnet)",
-		EnvVar: SWARM_ENV_NETWORK_ID,
-	}
-	SwarmSwapEnabledFlag = cli.BoolFlag{
-		Name:   "swap",
-		Usage:  "Swarm SWAP enabled (default false)",
-		EnvVar: SWARM_ENV_SWAP_ENABLE,
-	}
-	SwarmSwapAPIFlag = cli.StringFlag{
-		Name:   "swap-api",
-		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
-		EnvVar: SWARM_ENV_SWAP_API,
-	}
-	SwarmSyncDisabledFlag = cli.BoolTFlag{
-		Name:   "nosync",
-		Usage:  "Disable swarm syncing",
-		EnvVar: SWARM_ENV_SYNC_DISABLE,
-	}
-	SwarmSyncUpdateDelay = cli.DurationFlag{
-		Name:   "sync-update-delay",
-		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
-		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
-	}
-	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
-		Name:   "max-stream-peer-servers",
-		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
-		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
-		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
-	}
-	SwarmLightNodeEnabled = cli.BoolFlag{
-		Name:   "lightnode",
-		Usage:  "Enable Swarm LightNode (default false)",
-		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
-	}
-	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
-		Name:   "delivery-skip-check",
-		Usage:  "Skip chunk delivery check (default false)",
-		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
-	}
-	EnsAPIFlag = cli.StringSliceFlag{
-		Name:   "ens-api",
-		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
-		EnvVar: SWARM_ENV_ENS_API,
-	}
-	SwarmApiFlag = cli.StringFlag{
-		Name:  "bzzapi",
-		Usage: "Swarm HTTP endpoint",
-		Value: "http://127.0.0.1:8500",
-	}
-	SwarmRecursiveFlag = cli.BoolFlag{
-		Name:  "recursive",
-		Usage: "Upload directories recursively",
-	}
-	SwarmWantManifestFlag = cli.BoolTFlag{
-		Name:  "manifest",
-		Usage: "Automatic manifest upload (default true)",
-	}
-	SwarmUploadDefaultPath = cli.StringFlag{
-		Name:  "defaultpath",
-		Usage: "path to file served for empty url path (none)",
-	}
-	SwarmAccessGrantKeyFlag = cli.StringFlag{
-		Name:  "grant-key",
-		Usage: "grants a given public key access to an ACT",
-	}
-	SwarmAccessGrantKeysFlag = cli.StringFlag{
-		Name:  "grant-keys",
-		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
-	}
-	SwarmUpFromStdinFlag = cli.BoolFlag{
-		Name:  "stdin",
-		Usage: "reads data to be uploaded from stdin",
-	}
-	SwarmUploadMimeType = cli.StringFlag{
-		Name:  "mime",
-		Usage: "Manually specify MIME type",
-	}
-	SwarmEncryptedFlag = cli.BoolFlag{
-		Name:  "encrypt",
-		Usage: "use encrypted upload",
-	}
-	SwarmAccessPasswordFlag = cli.StringFlag{
-		Name:   "password",
-		Usage:  "Password",
-		EnvVar: SWARM_ACCESS_PASSWORD,
-	}
-	SwarmDryRunFlag = cli.BoolFlag{
-		Name:  "dry-run",
-		Usage: "dry-run",
-	}
-	CorsStringFlag = cli.StringFlag{
-		Name:   "corsdomain",
-		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
-		EnvVar: SWARM_ENV_CORS,
-	}
-	SwarmStorePath = cli.StringFlag{
-		Name:   "store.path",
-		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
-		EnvVar: SWARM_ENV_STORE_PATH,
-	}
-	SwarmStoreCapacity = cli.Uint64Flag{
-		Name:   "store.size",
-		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
-		EnvVar: SWARM_ENV_STORE_CAPACITY,
-	}
-	SwarmStoreCacheCapacity = cli.UintFlag{
-		Name:   "store.cache.size",
-		Usage:  "Number of recent chunks cached in memory (default 5000)",
-		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
-	}
-	SwarmCompressedFlag = cli.BoolFlag{
-		Name:  "compressed",
-		Usage: "Prints encryption keys in compressed form",
-	}
-	SwarmFeedNameFlag = cli.StringFlag{
-		Name:  "name",
-		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
-	}
-	SwarmFeedTopicFlag = cli.StringFlag{
-		Name:  "topic",
-		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
-	}
-	SwarmFeedDataOnCreateFlag = cli.StringFlag{
-		Name:  "data",
-		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
-	}
-	SwarmFeedManifestFlag = cli.StringFlag{
-		Name:  "manifest",
-		Usage: "Refers to the feed through a manifest",
-	}
-	SwarmFeedUserFlag = cli.StringFlag{
-		Name:  "user",
-		Usage: "Indicates the user who updates the feed",
-	}
-)
-
 //declare a few constant error messages, useful for later error check comparisons in test
 var (
 	SWARM_ERR_NO_BZZACCOUNT   = "bzzaccount option is required but not set; check your config file, command line or environment variables"
@@ -279,267 +120,24 @@ func init() {
 			Usage:              "Print public key information",
 			Description:        "The output of this command is supposed to be machine-readable",
 		},
-		{
-			Action:             upload,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "up",
-			Usage:              "uploads a file or directory to swarm using the HTTP API",
-			ArgsUsage:          "<file>",
-			Flags:              []cli.Flag{SwarmEncryptedFlag},
-			Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
-		},
-		{
-			CustomHelpTemplate: helpTemplate,
-			Name:               "access",
-			Usage:              "encrypts a reference and embeds it into a root manifest",
-			ArgsUsage:          "<ref>",
-			Description:        "encrypts a reference and embeds it into a root manifest",
-			Subcommands: []cli.Command{
-				{
-					CustomHelpTemplate: helpTemplate,
-					Name:               "new",
-					Usage:              "encrypts a reference and embeds it into a root manifest",
-					ArgsUsage:          "<ref>",
-					Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-					Subcommands: []cli.Command{
-						{
-							Action:             accessNewPass,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								utils.PasswordFileFlag,
-								SwarmDryRunFlag,
-							},
-							Name:        "pass",
-							Usage:       "encrypts a reference with a password and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-						{
-							Action:             accessNewPK,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								utils.PasswordFileFlag,
-								SwarmDryRunFlag,
-								SwarmAccessGrantKeyFlag,
-							},
-							Name:        "pk",
-							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-						{
-							Action:             accessNewACT,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								SwarmAccessGrantKeysFlag,
-								SwarmDryRunFlag,
-								utils.PasswordFileFlag,
-							},
-							Name:        "act",
-							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-					},
-				},
-			},
-		},
-		{
-			CustomHelpTemplate: helpTemplate,
-			Name:               "feed",
-			Usage:              "(Advanced) Create and update Swarm Feeds",
-			ArgsUsage:          "<create|update|info>",
-			Description:        "Works with Swarm Feeds",
-			Subcommands: []cli.Command{
-				{
-					Action:             feedCreateManifest,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "create",
-					Usage:              "creates and publishes a new feed manifest",
-					Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
-					The feed topic can be built in the following ways:
-					* use --topic to set the topic to an arbitrary binary hex string.
-					* use --name to set the topic to a human-readable name.
-					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
-					* use both --topic and --name to create named subtopics. 
-						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
-						this feed tracks a discussion about that contract.
-					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
-					it will then default to your local account (--bzzaccount)`,
-					Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
-				},
-				{
-					Action:             feedUpdate,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "update",
-					Usage:              "updates the content of an existing Swarm Feed",
-					ArgsUsage:          "<0x Hex data>",
-					Description: `publishes a new update on the specified topic
-					The feed topic can be built in the following ways:
-					* use --topic to set the topic to an arbitrary binary hex string.
-					* use --name to set the topic to a human-readable name.
-					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
-					* use both --topic and --name to create named subtopics. 
-						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
-						this feed tracks a discussion about that contract.
-					
-					If you have a manifest, you can specify it with --manifest to refer to the feed,
-					instead of using --topic / --name
-					`,
-					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
-				},
-				{
-					Action:             feedInfo,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "info",
-					Usage:              "obtains information about an existing Swarm feed",
-					Description: `obtains information about an existing Swarm feed
-					The topic can be specified directly with the --topic flag as an hex string
-					If no topic is specified, the default topic (zero) will be used
-					The --name flag can be used to specify subtopics with a specific name.
-					The --user flag allows to refer to a user other than yourself. If not specified,
-					it will then default to your local account (--bzzaccount)
-					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
-					to refer to the feed`,
-					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
-				},
-			},
-		},
-		{
-			Action:             list,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "ls",
-			Usage:              "list files and directories contained in a manifest",
-			ArgsUsage:          "<manifest> [<prefix>]",
-			Description:        "Lists files and directories contained in a manifest",
-		},
-		{
-			Action:             hash,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "hash",
-			Usage:              "print the swarm hash of a file or directory",
-			ArgsUsage:          "<file>",
-			Description:        "Prints the swarm hash of file or directory",
-		},
-		{
-			Action:      download,
-			Name:        "down",
-			Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
-			Usage:       "downloads a swarm manifest or a file inside a manifest",
-			ArgsUsage:   " <uri> [<dir>]",
-			Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
-		},
-		{
-			Name:               "manifest",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "perform operations on swarm manifests",
-			ArgsUsage:          "COMMAND",
-			Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
-			Subcommands: []cli.Command{
-				{
-					Action:             manifestAdd,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "add",
-					Usage:              "add a new path to the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <hash>",
-					Description:        "Adds a new path to the manifest",
-				},
-				{
-					Action:             manifestUpdate,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "update",
-					Usage:              "update the hash for an already existing path in the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <newhash>",
-					Description:        "Update the hash for an already existing path in the manifest",
-				},
-				{
-					Action:             manifestRemove,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "remove",
-					Usage:              "removes a path from the manifest",
-					ArgsUsage:          "<MANIFEST> <path>",
-					Description:        "Removes a path from the manifest",
-				},
-			},
-		},
-		{
-			Name:               "fs",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "perform FUSE operations",
-			ArgsUsage:          "fs COMMAND",
-			Description:        "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
-			Subcommands: []cli.Command{
-				{
-					Action:             mount,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "mount",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "mount a swarm hash to a mount point",
-					ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
-					Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-				{
-					Action:             unmount,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "unmount",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "unmount a swarmfs mount",
-					ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
-					Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-				{
-					Action:             listMounts,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "list",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "list swarmfs mounts",
-					ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
-					Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-			},
-		},
-		{
-			Name:               "db",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "manage the local chunk database",
-			ArgsUsage:          "db COMMAND",
-			Description:        "Manage the local chunk database",
-			Subcommands: []cli.Command{
-				{
-					Action:             dbExport,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "export",
-					Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
-					ArgsUsage:          "<chunkdb> <file>",
-					Description: `
-Export a local chunk database as a tar archive (use - to send to stdout).
-
-    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
-
-The export may be quite large, consider piping the output through the Unix
-pv(1) tool to get a progress bar:
-
-    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
-`,
-				},
-				{
-					Action:             dbImport,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "import",
-					Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
-					ArgsUsage:          "<chunkdb> <file>",
-					Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
-
-    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
-
-The import may be quite large, consider piping the input through the Unix
-pv(1) tool to get a progress bar:
-
-    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
-				},
-			},
-		},
-
+		// See upload.go
+		upCommand,
+		// See access.go
+		accessCommand,
+		// See feeds.go
+		feedCommand,
+		// See list.go
+		listCommand,
+		// See hash.go
+		hashCommand,
+		// See download.go
+		downloadCommand,
+		// See manifest.go
+		manifestCommand,
+		// See fs.go
+		fsCommand,
+		// See db.go
+		dbCommand,
 		// See config.go
 		DumpConfigCommand,
 	}

cmd/swarm/manifest.go

@@ -28,6 +28,40 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var manifestCommand = cli.Command{
+	Name:               "manifest",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "perform operations on swarm manifests",
+	ArgsUsage:          "COMMAND",
+	Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
+	Subcommands: []cli.Command{
+		{
+			Action:             manifestAdd,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "add",
+			Usage:              "add a new path to the manifest",
+			ArgsUsage:          "<MANIFEST> <path> <hash>",
+			Description:        "Adds a new path to the manifest",
+		},
+		{
+			Action:             manifestUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "update the hash for an already existing path in the manifest",
+			ArgsUsage:          "<MANIFEST> <path> <newhash>",
+			Description:        "Update the hash for an already existing path in the manifest",
+		},
+		{
+			Action:             manifestRemove,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "remove",
+			Usage:              "removes a path from the manifest",
+			ArgsUsage:          "<MANIFEST> <path>",
+			Description:        "Removes a path from the manifest",
+		},
+	},
+}
+
 // manifestAdd adds a new entry to the manifest at the given path.
 // New entry hash, the last argument, must be the hash of a manifest
 // with only one entry, which meta-data will be added to the original manifest.

cmd/swarm/manifest_test.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 // TestManifestChange tests manifest add, update and remove
@@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) {
 // Argument encrypt controls whether to use encryption or not.
 func testManifestChange(t *testing.T, encrypt bool) {
 	t.Parallel()
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
@@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
 
 func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 	t.Parallel()
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")

cmd/swarm/run_test.go

@@ -42,7 +42,6 @@ import (
 	"github.com/ethereum/go-ethereum/swarm"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@@ -58,7 +57,18 @@ func init() {
 	})
 }
 
-func serverFunc(api *api.API) testutil.TestServer {
+const clusterSize = 3
+
+var clusteronce sync.Once
+var cluster *testCluster
+
+func initCluster(t *testing.T) {
+	clusteronce.Do(func() {
+		cluster = newTestCluster(t, clusterSize)
+	})
+}
+
+func serverFunc(api *api.API) swarmhttp.TestServer {
 	return swarmhttp.NewServer(api, "")
 }
 func TestMain(m *testing.M) {

cmd/swarm/swarm-smoke/feed_upload_and_sync.go

@@ -0,0 +1,320 @@
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/multihash"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	colorable "github.com/mattn/go-colorable"
+	"github.com/pborman/uuid"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+const (
+	feedRandomDataLength = 8
+)
+
+// TODO: retrieve with manifest + extract repeating code
+func cliFeedUploadAndSync(c *cli.Context) error {
+
+	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))))
+
+	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
+
+	generateEndpoints(scheme, cluster, from, to)
+
+	log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing")
+
+	// create a random private key to sign updates with and derive the address
+	pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
+	if err != nil {
+		return err
+	}
+	defer pkFile.Close()
+	defer os.Remove(pkFile.Name())
+
+	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976"
+	privKey, err := crypto.HexToECDSA(privkeyHex)
+	if err != nil {
+		return err
+	}
+	user := crypto.PubkeyToAddress(privKey.PublicKey)
+	userHex := hexutil.Encode(user.Bytes())
+
+	// save the private key to a file
+	_, err = io.WriteString(pkFile, privkeyHex)
+	if err != nil {
+		return err
+	}
+
+	// keep hex strings for topic and subtopic
+	var topicHex string
+	var subTopicHex string
+
+	// and create combination hex topics for bzz-feed retrieval
+	// xor'ed with topic (zero-value topic if no topic)
+	var subTopicOnlyHex string
+	var mergedSubTopicHex string
+
+	// generate random topic and subtopic and put a hex on them
+	topicBytes, err := generateRandomData(feed.TopicLength)
+	topicHex = hexutil.Encode(topicBytes)
+	subTopicBytes, err := generateRandomData(8)
+	subTopicHex = hexutil.Encode(subTopicBytes)
+	if err != nil {
+		return err
+	}
+	mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes)
+	if err != nil {
+		return err
+	}
+	mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:])
+	subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil)
+	if err != nil {
+		return err
+	}
+	subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:])
+
+	// create feed manifest, topic only
+	var out bytes.Buffer
+	cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic)
+	}
+	log.Debug("create topic feed", "manifest", manifestWithTopic)
+	out.Reset()
+
+	// create feed manifest, subtopic only
+	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithSubTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic)
+	}
+	log.Debug("create subtopic feed", "manifest", manifestWithTopic)
+	out.Reset()
+
+	// create feed manifest, merged topic
+	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		log.Error(err.Error())
+		return err
+	}
+	manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithMergedTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic)
+	}
+	log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic)
+	out.Reset()
+
+	// create test data
+	data, err := generateRandomData(feedRandomDataLength)
+	if err != nil {
+		return err
+	}
+	h := md5.New()
+	h.Write(data)
+	dataHash := h.Sum(nil)
+	dataHex := hexutil.Encode(data)
+
+	// update with topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update topic", "out", out)
+	out.Reset()
+
+	// update with subtopic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update subtopic", "out", out)
+	out.Reset()
+
+	// update with merged topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update mergedtopic", "out", out)
+	out.Reset()
+
+	time.Sleep(3 * time.Second)
+
+	// retrieve the data
+	wg := sync.WaitGroup{}
+	for _, endpoint := range endpoints {
+		// raw retrieve, topic only
+		for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
+			wg.Add(1)
+			ruid := uuid.New()[:8]
+			go func(hex string, endpoint string, ruid string) {
+				for {
+					err := fetchFeed(hex, userHex, endpoint, dataHash, ruid)
+					if err != nil {
+						continue
+					}
+
+					wg.Done()
+					return
+				}
+			}(hex, endpoint, ruid)
+
+		}
+	}
+	wg.Wait()
+	log.Info("all endpoints synced random data successfully")
+
+	// upload test file
+	log.Info("uploading to " + endpoints[0] + " and syncing")
+
+	f, cleanup := generateRandomFile(filesize * 1000)
+	defer cleanup()
+
+	hash, err := upload(f, endpoints[0])
+	if err != nil {
+		return err
+	}
+	hashBytes, err := hexutil.Decode("0x" + hash)
+	if err != nil {
+		return err
+	}
+	multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes))
+
+	fileHash, err := digest(f)
+	if err != nil {
+		return err
+	}
+
+	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
+
+	// update file with topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update topic", "out", out)
+	out.Reset()
+
+	// update file with subtopic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update subtopic", "out", out)
+	out.Reset()
+
+	// update file with merged topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update mergedtopic", "out", out)
+	out.Reset()
+
+	time.Sleep(3 * time.Second)
+
+	for _, endpoint := range endpoints {
+
+		// manifest retrieve, topic only
+		for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
+			wg.Add(1)
+			ruid := uuid.New()[:8]
+			go func(url string, endpoint string, ruid string) {
+				for {
+					err := fetch(url, endpoint, fileHash, ruid)
+					if err != nil {
+						continue
+					}
+
+					wg.Done()
+					return
+				}
+			}(url, endpoint, ruid)
+		}
+
+	}
+	wg.Wait()
+	log.Info("all endpoints synced random file successfully")
+
+	return nil
+}
+
+func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
+	log.Trace("sleeping", "ruid", ruid)
+	time.Sleep(3 * time.Second)
+
+	log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
+	res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user)
+	if err != nil {
+		return err
+	}
+	log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
+
+	if res.StatusCode != 200 {
+		return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
+	}
+
+	defer res.Body.Close()
+
+	rdigest, err := digest(res.Body)
+	if err != nil {
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	if !bytes.Equal(rdigest, original) {
+		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
+
+	return nil
+}

cmd/swarm/swarm-smoke/main.go

@@ -21,7 +21,6 @@ import (
 	"sort"
 
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 
 	cli "gopkg.in/urfave/cli.v1"
 )
@@ -34,11 +33,10 @@ var (
 	filesize         int
 	from             int
 	to               int
+	verbosity        int
 )
 
 func main() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 
 	app := cli.NewApp()
 	app.Name = "smoke-test"
@@ -47,8 +45,8 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "cluster-endpoint",
-			Value:       "testing",
-			Usage:       "cluster to point to (local, open or testing)",
+			Value:       "prod",
+			Usage:       "cluster to point to (prod or a given namespace)",
 			Destination: &cluster,
 		},
 		cli.IntFlag{
@@ -80,6 +78,12 @@ func main() {
 			Usage:       "file size for generated random file in KB",
 			Destination: &filesize,
 		},
+		cli.IntFlag{
+			Name:        "verbosity",
+			Value:       1,
+			Usage:       "verbosity",
+			Destination: &verbosity,
+		},
 	}
 
 	app.Commands = []cli.Command{
@@ -89,6 +93,12 @@ func main() {
 			Usage:   "upload and sync",
 			Action:  cliUploadAndSync,
 		},
+		{
+			Name:    "feed_sync",
+			Aliases: []string{"f"},
+			Usage:   "feed update generate, upload and sync",
+			Action:  cliFeedUploadAndSync,
+		},
 	}
 
 	sort.Sort(cli.FlagsByName(app.Flags))

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -19,7 +19,9 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
+	crand "crypto/rand"
+	"crypto/tls"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -31,6 +33,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
+	colorable "github.com/mattn/go-colorable"
 	"github.com/pborman/uuid"
 
 	cli "gopkg.in/urfave/cli.v1"
@@ -38,18 +41,13 @@ import (
 
 func generateEndpoints(scheme string, cluster string, from int, to int) {
 	if cluster == "prod" {
-		cluster = ""
-	} else if cluster == "local" {
 		for port := from; port <= to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
+			endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
 		}
-		return
 	} else {
-		cluster = cluster + "."
-	}
-
 		for port := from; port <= to; port++ {
-		endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
+			endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster))
+		}
 	}
 
 	if includeLocalhost {
@@ -58,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
 }
 
 func cliUploadAndSync(c *cli.Context) error {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+
 	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
 
 	generateEndpoints(scheme, cluster, from, to)
@@ -85,7 +86,6 @@ func cliUploadAndSync(c *cli.Context) error {
 
 	wg := sync.WaitGroup{}
 	for _, endpoint := range endpoints {
-		endpoint := endpoint
 		ruid := uuid.New()[:8]
 		wg.Add(1)
 		go func(endpoint string, ruid string) {
@@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
 	time.Sleep(3 * time.Second)
 
 	log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
-	res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
+	client := &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}}
+	res, err := client.Get(endpoint + "/bzz:/" + hash + "/")
 	if err != nil {
 		log.Warn(err.Error(), "ruid", ruid)
 		return err
@@ -166,6 +169,18 @@ func digest(r io.Reader) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
+// generates random data in heap buffer
+func generateRandomData(datasize int) ([]byte, error) {
+	b := make([]byte, datasize)
+	c, err := crand.Read(b)
+	if err != nil {
+		return nil, err
+	} else if c != datasize {
+		return nil, errors.New("short read")
+	}
+	return b, nil
+}
+
 // generateRandomFile is creating a temporary file with the requested byte size
 func generateRandomFile(size int) (f *os.File, teardown func()) {
 	// create a tmp file
@@ -181,7 +196,7 @@ func generateRandomFile(size int) (f *os.File, teardown func()) {
 	}
 
 	buf := make([]byte, size)
-	_, err = rand.Read(buf)
+	_, err = crand.Read(buf)
 	if err != nil {
 		panic(err)
 	}

cmd/swarm/upload.go

@@ -26,16 +26,27 @@ import (
 	"os/user"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/log"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"gopkg.in/urfave/cli.v1"
 )
 
-func upload(ctx *cli.Context) {
+var upCommand = cli.Command{
+	Action:             upload,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "up",
+	Usage:              "uploads a file or directory to swarm using the HTTP API",
+	ArgsUsage:          "<file>",
+	Flags:              []cli.Flag{SwarmEncryptedFlag},
+	Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
+}
 
+func upload(ctx *cli.Context) {
 	args := ctx.Args()
 	var (
 		bzzapi          = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
@@ -46,9 +57,16 @@ func upload(ctx *cli.Context) {
 		mimeType        = ctx.GlobalString(SwarmUploadMimeType.Name)
 		client          = swarm.NewClient(bzzapi)
 		toEncrypt       = ctx.Bool(SwarmEncryptedFlag.Name)
+		autoDefaultPath = false
 		file            string
 	)
-
+	if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" {
+		b, err := strconv.ParseBool(autoDefaultPathString)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err)
+		}
+		autoDefaultPath = b
+	}
 	if len(args) != 1 {
 		if fromStdin {
 			tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -97,6 +115,15 @@ func upload(ctx *cli.Context) {
 			if !recursive {
 				return "", errors.New("Argument is a directory and recursive upload is disabled")
 			}
+			if autoDefaultPath && defaultPath == "" {
+				defaultEntryCandidate := path.Join(file, "index.html")
+				log.Debug("trying to find default path", "path", defaultEntryCandidate)
+				defaultEntryStat, err := os.Stat(defaultEntryCandidate)
+				if err == nil && !defaultEntryStat.IsDir() {
+					log.Debug("setting auto detected default path", "path", defaultEntryCandidate)
+					defaultPath = defaultEntryCandidate
+				}
+			}
 			if defaultPath != "" {
 				// construct absolute default path
 				absDefaultPath, _ := filepath.Abs(defaultPath)

cmd/swarm/upload_test.go

@@ -31,7 +31,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 	"github.com/mattn/go-colorable"
 )
@@ -41,69 +41,66 @@ func init() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 }
 
-// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
+func TestSwarmUp(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"NoEncryption", testNoEncryption},
+		{"Encrypted", testEncrypted},
+		{"RecursiveNoEncryption", testRecursiveNoEncryption},
+		{"RecursiveEncrypted", testRecursiveEncrypted},
+		{"DefaultPathAll", testDefaultPathAll},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testNoEncryption tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUp(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
+func testNoEncryption(t *testing.T) {
+	testDefault(false, t)
 }
 
-	testCLISwarmUp(false, t)
-}
-func TestCLISwarmUpRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(false, t)
-}
-
-// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
+// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUpEncrypted(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUp(true, t)
-}
-func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(true, t)
+func testEncrypted(t *testing.T) {
+	testDefault(true, t)
 }
 
-func testCLISwarmUp(toEncrypt bool, t *testing.T) {
-	log.Info("starting 3 node cluster")
-	cluster := newTestCluster(t, 3)
-	defer cluster.Shutdown()
-
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
+func testRecursiveNoEncryption(t *testing.T) {
+	testRecursive(false, t)
 }
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
+
+func testRecursiveEncrypted(t *testing.T) {
+	testRecursive(true, t)
+}
+
+func testDefault(toEncrypt bool, t *testing.T) {
+	tmpFileName := testutil.TempFileWithContent(t, data)
+	defer os.Remove(tmpFileName)
 
 	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	hashRegexp := `[a-f\d]{64}`
 	flags := []string{
 		"--bzzapi", cluster.Nodes[0].URL,
 		"up",
-		tmp.Name()}
+		tmpFileName}
 	if toEncrypt {
 		hashRegexp = `[a-f\d]{128}`
 		flags = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"up",
 			"--encrypt",
-			tmp.Name()}
+			tmpFileName}
 	}
 	// upload the file with 'swarm up' and expect a hash
 	log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@@ -192,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
 	}
 }
 
-func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
-	fmt.Println("starting 3 node cluster")
-	cluster := newTestCluster(t, 3)
-	defer cluster.Shutdown()
-
+func testRecursive(toEncrypt bool, t *testing.T) {
 	tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpUploadDir)
 	// create tmp files
-	data := "notsorandomdata"
 	for _, path := range []string{"tmp1", "tmp2"} {
 		if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
 			t.Fatal(err)
@@ -243,8 +235,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 		}
 		defer os.RemoveAll(tmpDownload)
 		bzzLocator := "bzz:/" + hash
-		flagss := []string{}
-		flagss = []string{
+		flagss := []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"down",
 			"--recursive",
@@ -265,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 
 			switch mode := fi.Mode(); {
 			case mode.IsRegular():
-				if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
+				if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
 					t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
 				} else {
 					ff := make([]byte, len(data))
@@ -286,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 	}
 }
 
-// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// testDefaultPathAll tests swarm recursive upload with relative and absolute
 // default paths and with encryption.
-func TestCLISwarmUpDefaultPath(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpDefaultPath(false, false, t)
-	testCLISwarmUpDefaultPath(false, true, t)
-	testCLISwarmUpDefaultPath(true, false, t)
-	testCLISwarmUpDefaultPath(true, true, t)
+func testDefaultPathAll(t *testing.T) {
+	testDefaultPath(false, false, t)
+	testDefaultPath(false, true, t)
+	testDefaultPath(true, false, t)
+	testDefaultPath(true, true, t)
 }
 
-func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
-	srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
-	defer srv.Close()
-
+func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
 	tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
 	if err != nil {
 		t.Fatal(err)
@@ -324,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 
 	args := []string{
 		"--bzzapi",
-		srv.URL,
+		cluster.Nodes[0].URL,
 		"--recursive",
 		"--defaultpath",
 		defaultPath,
@@ -341,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 	up.ExpectExit()
 	hash := matches[0]
 
-	client := swarm.NewClient(srv.URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	m, isEncrypted, err := client.DownloadManifest(hash)
 	if err != nil {

cmd/utils/cmd.go

@@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
 		// Accumulate the preimages and flush when enough ws gathered
 		preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
 		if len(preimages) > 1024 {
-			rawdb.WritePreimages(db, 0, preimages)
+			rawdb.WritePreimages(db, preimages)
 			preimages = make(map[common.Hash][]byte)
 		}
 	}
 	// Flush the last batch preimage data
 	if len(preimages) > 0 {
-		rawdb.WritePreimages(db, 0, preimages)
+		rawdb.WritePreimages(db, preimages)
 	}
 	return nil
 }

cmd/utils/flags.go

@@ -295,7 +295,12 @@ var (
 	CacheDatabaseFlag = cli.IntFlag{
 		Name:  "cache.database",
 		Usage: "Percentage of cache memory allowance to use for database io",
-		Value: 75,
+		Value: 50,
+	}
+	CacheTrieFlag = cli.IntFlag{
+		Name:  "cache.trie",
+		Usage: "Percentage of cache memory allowance to use for trie caching",
+		Value: 25,
 	}
 	CacheGCFlag = cli.IntFlag{
 		Name:  "cache.gc",
@@ -1185,8 +1190,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	}
 	cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
 
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
 		cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
@@ -1422,11 +1430,15 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	}
 	cache := &core.CacheConfig{
 		Disabled:       ctx.GlobalString(GCModeFlag.Name) == "archive",
-		TrieNodeLimit: eth.DefaultConfig.TrieCache,
+		TrieCleanLimit: eth.DefaultConfig.TrieCleanCache,
+		TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
 		TrieTimeLimit:  eth.DefaultConfig.TrieTimeout,
 	}
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
 	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)

common/bytes.go

@@ -31,6 +31,15 @@ func ToHex(b []byte) string {
 	return "0x" + hex
 }
 
+// ToHexArray creates a array of hex-string based on []byte
+func ToHexArray(b [][]byte) []string {
+	r := make([]string, len(b))
+	for i := range b {
+		r[i] = ToHex(b[i])
+	}
+	return r
+}
+
 // FromHex returns the bytes represented by the hexadecimal string s.
 // s may be prefixed with "0x".
 func FromHex(s string) []byte {

common/compiler/solidity.go

@@ -31,14 +31,15 @@ import (
 
 var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
-// Contract contains information about a compiled contract, alongside its code.
+// Contract contains information about a compiled contract, alongside its code and runtime code.
 type Contract struct {
 	Code        string       `json:"code"`
+	RuntimeCode string       `json:"runtime-code"`
 	Info        ContractInfo `json:"info"`
 }
 
 // ContractInfo contains information about a compiled contract, including access
-// to the ABI definition, user and developer docs, and metadata.
+// to the ABI definition, source mapping, user and developer docs, and metadata.
 //
 // Depending on the source, language version, compiler version, and compiler
 // options will provide information about how the contract was compiled.
@@ -48,6 +49,8 @@ type ContractInfo struct {
 	LanguageVersion string      `json:"languageVersion"`
 	CompilerVersion string      `json:"compilerVersion"`
 	CompilerOptions string      `json:"compilerOptions"`
+	SrcMap          string      `json:"srcMap"`
+	SrcMapRuntime   string      `json:"srcMapRuntime"`
 	AbiDefinition   interface{} `json:"abiDefinition"`
 	UserDoc         interface{} `json:"userDoc"`
 	DeveloperDoc    interface{} `json:"developerDoc"`
@@ -63,14 +66,16 @@ type Solidity struct {
 // --combined-output format
 type solcOutput struct {
 	Contracts map[string]struct {
-		Bin, Abi, Devdoc, Userdoc, Metadata string
+		BinRuntime                                  string `json:"bin-runtime"`
+		SrcMapRuntime                               string `json:"srcmap-runtime"`
+		Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string
 	}
 	Version string
 }
 
 func (s *Solidity) makeArgs() []string {
 	p := []string{
-		"--combined-json", "bin,abi,userdoc,devdoc",
+		"--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
 		"--optimize", // code optimizer switched on
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
@@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro
 // provided source, language and compiler version, and compiler options are all
 // passed through into the Contract structs.
 //
-// The solc output is expected to contain ABI, user docs, and dev docs.
+// The solc output is expected to contain ABI, source mapping, user docs, and dev docs.
 //
 // Returns an error if the JSON is malformed or missing data, or if the JSON
 // embedded within the JSON is malformed.
@@ -185,12 +190,15 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
 		}
 		contracts[name] = &Contract{
 			Code:        "0x" + info.Bin,
+			RuntimeCode: "0x" + info.BinRuntime,
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",
 				LanguageVersion: languageVersion,
 				CompilerVersion: compilerVersion,
 				CompilerOptions: compilerOptions,
+				SrcMap:          info.SrcMap,
+				SrcMapRuntime:   info.SrcMapRuntime,
 				AbiDefinition:   abi,
 				UserDoc:         userdoc,
 				DeveloperDoc:    devdoc,

consensus/clique/clique.go

@@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash {
 	return sigHash(header)
 }
 
-// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
+// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
 func (c *Clique) Close() error {
 	return nil
 }

consensus/ethash/api.go

@@ -37,27 +37,28 @@ type API struct {
 //   result[0] - 32 bytes hex encoded current block header pow-hash
 //   result[1] - 32 bytes hex encoded seed hash used for DAG
 //   result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
-func (api *API) GetWork() ([3]string, error) {
+//   result[3] - hex encoded block number
+func (api *API) GetWork() ([4]string, error) {
 	if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest {
-		return [3]string{}, errors.New("not supported")
+		return [4]string{}, errors.New("not supported")
 	}
 
 	var (
-		workCh = make(chan [3]string, 1)
+		workCh = make(chan [4]string, 1)
 		errc   = make(chan error, 1)
 	)
 
 	select {
 	case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}:
 	case <-api.ethash.exitCh:
-		return [3]string{}, errEthashStopped
+		return [4]string{}, errEthashStopped
 	}
 
 	select {
 	case work := <-workCh:
 		return work, nil
 	case err := <-errc:
-		return [3]string{}, err
+		return [4]string{}, err
 	}
 }
 

consensus/ethash/ethash.go

@@ -432,7 +432,7 @@ type hashrate struct {
 // sealWork wraps a seal work package for remote sealer.
 type sealWork struct {
 	errc chan error
-	res  chan [3]string
+	res  chan [4]string
 }
 
 // Ethash is a consensus engine based on proof-of-work implementing the ethash

consensus/ethash/ethash_test.go

@@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) {
 	ethash.Seal(nil, block, results, nil)
 
 	var (
-		work [3]string
+		work [4]string
 		err  error
 	)
 	if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {

consensus/ethash/sealer.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 
 		results      chan<- *types.Block
 		currentBlock *types.Block
-		currentWork  [3]string
+		currentWork  [4]string
 
 		notifyTransport = &http.Transport{}
 		notifyClient    = &http.Client{
@@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 	//   result[0], 32 bytes hex encoded current block header pow-hash
 	//   result[1], 32 bytes hex encoded seed hash used for DAG
 	//   result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
+	//   result[3], hex encoded block number
 	makeWork := func(block *types.Block) {
 		hash := ethash.SealHash(block.Header())
 
 		currentWork[0] = hash.Hex()
 		currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
 		currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex()
+		currentWork[3] = hexutil.EncodeBig(block.Number())
 
 		// Trace the seal work fetched by remote sealer.
 		currentBlock = block

core/block_validator.go

@@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
 		return ErrKnownBlock
 	}
-	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
-		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
-			return consensus.ErrUnknownAncestor
-		}
-		return consensus.ErrPrunedAncestor
-	}
 	// Header validity is known at this point, check the uncles and transactions
 	header := block.Header()
 	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
@@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
 		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
 	}
+	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
+			return consensus.ErrUnknownAncestor
+		}
+		return consensus.ErrPrunedAncestor
+	}
 	return nil
 }
 

core/blockchain.go

@@ -69,7 +69,8 @@ const (
 // that's resident in a blockchain.
 type CacheConfig struct {
 	Disabled       bool          // Whether to disable trie write caching (archive node)
-	TrieNodeLimit int           // Memory limit (MB) at which to flush the current in-memory trie to disk
+	TrieCleanLimit int           // Memory allowance (MB) to use for caching trie nodes in memory
+	TrieDirtyLimit int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
 	TrieTimeLimit  time.Duration // Time limit after which to flush the current in-memory trie to disk
 }
 
@@ -140,7 +141,8 @@ type BlockChain struct {
 func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = &CacheConfig{
-			TrieNodeLimit: 256 * 1024 * 1024,
+			TrieCleanLimit: 256,
+			TrieDirtyLimit: 256,
 			TrieTimeLimit:  5 * time.Minute,
 		}
 	}
@@ -156,7 +158,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		cacheConfig:    cacheConfig,
 		db:             db,
 		triegc:         prque.New(nil),
-		stateCache:     state.NewDatabase(db),
+		stateCache:     state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
 		quit:           make(chan struct{}),
 		shouldPreserve: shouldPreserve,
 		bodyCache:      bodyCache,
@@ -393,6 +395,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
 	return state.New(root, bc.stateCache)
 }
 
+// StateCache returns the caching database underpinning the blockchain instance.
+func (bc *BlockChain) StateCache() state.Database {
+	return bc.stateCache
+}
+
 // Reset purges the entire blockchain, restoring it to its genesis state.
 func (bc *BlockChain) Reset() error {
 	return bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -554,6 +561,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
 	return rawdb.HasBody(bc.db, hash, number)
 }
 
+// HasFastBlock checks if a fast block is fully present in the database or not.
+func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
+	if !bc.HasBlock(hash, number) {
+		return false
+	}
+	if bc.receiptsCache.Contains(hash) {
+		return true
+	}
+	return rawdb.HasReceipts(bc.db, hash, number)
+}
+
 // HasState checks if state trie is fully present in the database or not.
 func (bc *BlockChain) HasState(hash common.Hash) bool {
 	_, err := bc.stateCache.OpenTrie(hash)
@@ -611,12 +629,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
 		return receipts.(types.Receipts)
 	}
-
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
-
 	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
 	bc.receiptsCache.Add(hash, receipts)
 	return receipts
@@ -938,7 +954,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 			// If we exceeded our memory allowance, flush matured singleton nodes to disk
 			var (
 				nodes, imgs = triedb.Size()
-				limit       = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
+				limit       = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
 			)
 			if nodes > limit || imgs > 4*1024*1024 {
 				triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1002,7 +1018,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 		}
 		// Write the positional metadata for transaction/receipt lookups and preimages
 		rawdb.WriteTxLookupEntries(batch, block)
-		rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages())
+		rawdb.WritePreimages(batch, state.Preimages())
 
 		status = CanonStatTy
 	} else {

core/chain_makers.go

@@ -36,7 +36,6 @@ type BlockGen struct {
 	i       int
 	parent  *types.Block
 	chain   []*types.Block
-	chainReader consensus.ChainReader
 	header  *types.Header
 	statedb *state.StateDB
 
@@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) {
 // For index -1, PrevBlock returns the parent block given to GenerateChain.
 func (b *BlockGen) PrevBlock(index int) *types.Block {
 	if index >= b.i {
-		panic("block index out of range")
+		panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
 	}
 	if index == -1 {
 		return b.parent
@@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}
-	b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header())
+	chainreader := &fakeChainReader{config: b.config}
+	b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		config = params.TestChainConfig
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	chainreader := &fakeChainReader{config: config}
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
-		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
-		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
-		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
-		defer blockchain.Stop()
-
-		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
-		b.header = makeHeader(b.chainReader, parent, statedb, b.engine)
+		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
+		b.header = makeHeader(chainreader, parent, statedb, b.engine)
 
 		// Mutate the state and block according to any hard-fork specs
 		if daoBlock := config.DAOForkBlock; daoBlock != nil {
@@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		}
 		if b.engine != nil {
 			// Finalize and seal the block
-			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+			block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
 
 			// Write state changes to db
 			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
@@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd
 	})
 	return blocks
 }
+
+type fakeChainReader struct {
+	config  *params.ChainConfig
+	genesis *types.Block
+}
+
+// Config returns the chain configuration.
+func (cr *fakeChainReader) Config() *params.ChainConfig {
+	return cr.config
+}
+
+func (cr *fakeChainReader) CurrentHeader() *types.Header                            { return nil }
+func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header           { return nil }
+func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header          { return nil }
+func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block   { return nil }

core/rawdb/accessors_chain.go

@@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
 	}
 }
 
+// HasReceipts verifies the existence of all the transaction receipts belonging
+// to a block.
+func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
+	if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
+		return false
+	}
+	return true
+}
+
 // ReadReceipts retrieves all the transaction receipts belonging to a block.
 func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
 	// Retrieve the flattened receipt slice
@@ -278,7 +287,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece
 	if len(data) == 0 {
 		return nil
 	}
-	// Convert the revceipts from their storage form to their internal representation
+	// Convert the receipts from their storage form to their internal representation
 	storageReceipts := []*types.ReceiptForStorage{}
 	if err := rlp.DecodeBytes(data, &storageReceipts); err != nil {
 		log.Error("Invalid receipt array RLP", "hash", hash, "err", err)

core/rawdb/accessors_metadata.go

@@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte {
 	return data
 }
 
-// WritePreimages writes the provided set of preimages to the database. `number` is the
-// current block number, and is used for debug messages only.
-func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) {
+// WritePreimages writes the provided set of preimages to the database.
+func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) {
 	for hash, preimage := range preimages {
 		if err := db.Put(preimageKey(hash), preimage); err != nil {
 			log.Crit("Failed to store trie preimage", "err", err)

core/rawdb/schema.go

@@ -35,7 +35,7 @@ var (
 	// headBlockKey tracks the latest know full block's hash.
 	headBlockKey = []byte("LastBlock")
 
-	// headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync.
+	// headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
 	headFastBlockKey = []byte("LastFast")
 
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.

core/state/database.go

@@ -72,13 +72,19 @@ type Trie interface {
 }
 
 // NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains cached trie nodes in memory. The pool is an optional
-// intermediate trie-node memory pool between the low level storage layer and the
-// high level trie abstraction.
+// concurrent use and retains a few recent expanded trie nodes in memory. To keep
+// more historical state in memory, use the NewDatabaseWithCache constructor.
 func NewDatabase(db ethdb.Database) Database {
+	return NewDatabaseWithCache(db, 0)
+}
+
+// NewDatabase creates a backing store for state. The returned database is safe for
+// concurrent use and retains both a few recent expanded trie nodes in memory, as
+// well as a lot of collapsed RLP trie nodes in a large memory cache.
+func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	return &cachingDB{
-		db:            trie.NewDatabase(db),
+		db:            trie.NewDatabaseWithCache(db, cache),
 		codeSizeCache: csc,
 	}
 }

core/state/statedb.go

@@ -18,10 +18,10 @@
 package state
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"sort"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -44,6 +44,13 @@ var (
 	emptyCode = crypto.Keccak256Hash(nil)
 )
 
+type proofList [][]byte
+
+func (n *proofList) Put(key []byte, value []byte) error {
+	*n = append(*n, value)
+	return nil
+}
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -79,8 +86,6 @@ type StateDB struct {
 	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
-
-	lock sync.Mutex
 }
 
 // Create a new state from a given trie.
@@ -256,6 +261,24 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash
 	return common.Hash{}
 }
 
+// GetProof returns the MerkleProof for a given Account
+func (self *StateDB) GetProof(a common.Address) ([][]byte, error) {
+	var proof proofList
+	err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof)
+	return [][]byte(proof), err
+}
+
+// GetProof returns the StorageProof for given key
+func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) {
+	var proof proofList
+	trie := self.StorageTrie(a)
+	if trie == nil {
+		return proof, errors.New("storage trie for requested address does not exist")
+	}
+	err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
+	return [][]byte(proof), err
+}
+
 // GetCommittedState retrieves a value from the given account's committed storage trie.
 func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash {
 	stateObject := self.getStateObject(addr)
@@ -470,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common
 // Copy creates a deep, independent copy of the state.
 // Snapshots of the copied state cannot be applied to the copy.
 func (self *StateDB) Copy() *StateDB {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-
 	// Copy all the basic fields, initialize the memory ones
 	state := &StateDB{
 		db:                self.db,

core/tx_pool.go

@@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
 // addTxsLocked attempts to queue a batch of transactions if they are valid,
 // whilst assuming the transaction pool lock is already held.
 func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
-	// Add the batch of transaction, tracking the accepted ones
+	// Add the batch of transactions, tracking the accepted ones
 	dirty := make(map[common.Address]struct{})
 	errs := make([]error, len(txs))
 

core/types/transaction_signing.go

@@ -136,7 +136,7 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) {
 	return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true)
 }
 
-// WithSignature returns a new transaction with the given signature. This signature
+// SignatureValues returns signature values. This signature
 // needs to be in the [R || S || V] format where V is 0 or 1.
 func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) {
 	R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig)

core/vm/gen_structlog.go

@@ -13,6 +13,7 @@ import (
 
 var _ = (*structLogMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (s StructLog) MarshalJSON() ([]byte, error) {
 	type StructLog struct {
 		Pc            uint64                      `json:"pc"`
@@ -24,6 +25,7 @@ func (s StructLog) MarshalJSON() ([]byte, error) {
 		Stack         []*math.HexOrDecimal256     `json:"stack"`
 		Storage       map[common.Hash]common.Hash `json:"-"`
 		Depth         int                         `json:"depth"`
+		RefundCounter uint64                      `json:"refund"`
 		Err           error                       `json:"-"`
 		OpName        string                      `json:"opName"`
 		ErrorString   string                      `json:"error"`
@@ -43,12 +45,14 @@ func (s StructLog) MarshalJSON() ([]byte, error) {
 	}
 	enc.Storage = s.Storage
 	enc.Depth = s.Depth
+	enc.RefundCounter = s.RefundCounter
 	enc.Err = s.Err
 	enc.OpName = s.OpName()
 	enc.ErrorString = s.ErrorString()
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (s *StructLog) UnmarshalJSON(input []byte) error {
 	type StructLog struct {
 		Pc            *uint64                     `json:"pc"`
@@ -60,6 +64,7 @@ func (s *StructLog) UnmarshalJSON(input []byte) error {
 		Stack         []*math.HexOrDecimal256     `json:"stack"`
 		Storage       map[common.Hash]common.Hash `json:"-"`
 		Depth         *int                        `json:"depth"`
+		RefundCounter *uint64                     `json:"refund"`
 		Err           error                       `json:"-"`
 	}
 	var dec StructLog
@@ -96,6 +101,9 @@ func (s *StructLog) UnmarshalJSON(input []byte) error {
 	if dec.Depth != nil {
 		s.Depth = *dec.Depth
 	}
+	if dec.RefundCounter != nil {
+		s.RefundCounter = *dec.RefundCounter
+	}
 	if dec.Err != nil {
 		s.Err = dec.Err
 	}

core/vm/instructions.go

@@ -124,10 +124,22 @@ func opSmod(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory
 
 func opExp(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	base, exponent := stack.pop(), stack.pop()
+	// some shortcuts
+	cmpToOne := exponent.Cmp(big1)
+	if cmpToOne < 0 { // Exponent is zero
+		// x ^ 0 == 1
+		stack.push(base.SetUint64(1))
+	} else if base.Sign() == 0 {
+		// 0 ^ y, if y != 0, == 0
+		stack.push(base.SetUint64(0))
+	} else if cmpToOne == 0 { // Exponent is one
+		// x ^ 1 == x
+		stack.push(base)
+	} else {
 		stack.push(math.Exp(base, exponent))
-
-	interpreter.intPool.put(base, exponent)
-
+		interpreter.intPool.put(base)
+	}
+	interpreter.intPool.put(exponent)
 	return nil, nil
 }
 
@@ -532,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract,
 // this account should be regarded as a non-existent account and zero should be returned.
 func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	slot := stack.peek()
-	slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes())
+	address := common.BigToAddress(slot)
+	if interpreter.evm.StateDB.Empty(address) {
+		slot.SetUint64(0)
+	} else {
+		slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes())
+	}
 	return nil, nil
 }
 

core/vm/logger.go

@@ -65,6 +65,7 @@ type StructLog struct {
 	Stack         []*big.Int                  `json:"stack"`
 	Storage       map[common.Hash]common.Hash `json:"-"`
 	Depth         int                         `json:"depth"`
+	RefundCounter uint64                      `json:"refund"`
 	Err           error                       `json:"-"`
 }
 
@@ -177,7 +178,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui
 		storage = l.changedValues[contract.Address()].Copy()
 	}
 	// create a new snaptshot of the EVM.
-	log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, err}
+	log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, env.StateDB.GetRefund(), err}
 
 	l.logs = append(l.logs, log)
 	return nil

core/vm/logger_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -41,9 +42,15 @@ func (d *dummyContractRef) SetBalance(*big.Int)        {}
 func (d *dummyContractRef) SetNonce(uint64)            {}
 func (d *dummyContractRef) Balance() *big.Int          { return new(big.Int) }
 
+type dummyStatedb struct {
+	state.StateDB
+}
+
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
+
 func TestStoreCapture(t *testing.T) {
 	var (
-		env      = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
+		env      = NewEVM(Context{}, &dummyStatedb{}, params.TestChainConfig, Config{})
 		logger   = NewStructLogger(nil)
 		mem      = NewMemory()
 		stack    = newstack()
@@ -51,9 +58,7 @@ func TestStoreCapture(t *testing.T) {
 	)
 	stack.push(big.NewInt(1))
 	stack.push(big.NewInt(0))
-
 	var index common.Hash
-
 	logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, contract, 0, nil)
 	if len(logger.changedValues[contract.Address()]) == 0 {
 		t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()]))

eth/api.go

@@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
 	if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
 		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
 	}
+	triedb := api.eth.BlockChain().StateCache().TrieDB()
 
-	oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+	oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0)
 	if err != nil {
 		return nil, err
 	}
-	newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+	newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0)
 	if err != nil {
 		return nil, err
 	}
-
 	diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}))
 	iter := trie.NewIterator(diff)
 

eth/api_tracer.go

@@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 
 	// Ensure we have a valid starting state before doing any work
 	origin := start.NumberU64()
-	database := state.NewDatabase(api.eth.ChainDb())
+	database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis
 
 	if number := start.NumberU64(); number > 0 {
 		start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
@@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 	}
 	// Otherwise try to reexec blocks until we find a state or reach our limit
 	origin := block.NumberU64()
-	database := state.NewDatabase(api.eth.ChainDb())
+	database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16)
 
 	for i := uint64(0); i < reexec; i++ {
 		block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)

eth/backend.go

@@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 			EWASMInterpreter:        config.EWASMInterpreter,
 			EVMInterpreter:          config.EVMInterpreter,
 		}
-		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
+		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout}
 	)
 	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
 	if err != nil {

eth/config.go

@@ -45,8 +45,9 @@ var DefaultConfig = Config{
 	},
 	NetworkId:      1,
 	LightPeers:     100,
-	DatabaseCache: 768,
-	TrieCache:     256,
+	DatabaseCache:  512,
+	TrieCleanCache: 256,
+	TrieDirtyCache: 256,
 	TrieTimeout:    60 * time.Minute,
 	MinerGasFloor:  8000000,
 	MinerGasCeil:   8000000,
@@ -94,7 +95,8 @@ type Config struct {
 	SkipBcVersionCheck bool `toml:"-"`
 	DatabaseHandles    int  `toml:"-"`
 	DatabaseCache      int
-	TrieCache          int
+	TrieCleanCache     int
+	TrieDirtyCache     int
 	TrieTimeout        time.Duration
 
 	// Mining-related options
@@ -122,7 +124,7 @@ type Config struct {
 	// Miscellaneous options
 	DocRoot string `toml:"-"`
 
-	// Type of the EWASM interpreter ("" for detault)
+	// Type of the EWASM interpreter ("" for default)
 	EWASMInterpreter string
 	// Type of the EVM interpreter ("" for default)
 	EVMInterpreter string

eth/downloader/downloader.go

@@ -181,6 +181,9 @@ type BlockChain interface {
 	// HasBlock verifies a block's presence in the local chain.
 	HasBlock(common.Hash, uint64) bool
 
+	// HasFastBlock verifies a fast block's presence in the local chain.
+	HasFastBlock(common.Hash, uint64) bool
+
 	// GetBlockByHash retrieves a block from the local chain.
 	GetBlockByHash(common.Hash) *types.Block
 
@@ -430,7 +433,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
 	}
 	height := latest.Number.Uint64()
 
-	origin, err := d.findAncestor(p, height)
+	origin, err := d.findAncestor(p, latest)
 	if err != nil {
 		return err
 	}
@@ -587,41 +590,86 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) {
 	}
 }
 
+// calculateRequestSpan calculates what headers to request from a peer when trying to determine the
+// common ancestor.
+// It returns parameters to be used for peer.RequestHeadersByNumber:
+//  from - starting block number
+//  count - number of headers to request
+//  skip - number of headers to skip
+// and also returns 'max', the last block which is expected to be returned by the remote peers,
+// given the (from,count,skip)
+func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) {
+	var (
+		from     int
+		count    int
+		MaxCount = MaxHeaderFetch / 16
+	)
+	// requestHead is the highest block that we will ask for. If requestHead is not offset,
+	// the highest block that we will get is 16 blocks back from head, which means we
+	// will fetch 14 or 15 blocks unnecessarily in the case the height difference
+	// between us and the peer is 1-2 blocks, which is most common
+	requestHead := int(remoteHeight) - 1
+	if requestHead < 0 {
+		requestHead = 0
+	}
+	// requestBottom is the lowest block we want included in the query
+	// Ideally, we want to include just below own head
+	requestBottom := int(localHeight - 1)
+	if requestBottom < 0 {
+		requestBottom = 0
+	}
+	totalSpan := requestHead - requestBottom
+	span := 1 + totalSpan/MaxCount
+	if span < 2 {
+		span = 2
+	}
+	if span > 16 {
+		span = 16
+	}
+
+	count = 1 + totalSpan/span
+	if count > MaxCount {
+		count = MaxCount
+	}
+	if count < 2 {
+		count = 2
+	}
+	from = requestHead - (count-1)*span
+	if from < 0 {
+		from = 0
+	}
+	max := from + (count-1)*span
+	return int64(from), count, span - 1, uint64(max)
+}
+
 // findAncestor tries to locate the common ancestor link of the local chain and
 // a remote peers blockchain. In the general case when our node was in sync and
 // on the correct chain, checking the top N links should already get us a match.
 // In the rare scenario when we ended up on a long reorganisation (i.e. none of
 // the head links match), we do a binary search to find the common ancestor.
-func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) {
+func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) {
 	// Figure out the valid ancestor range to prevent rewrite attacks
-	floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
+	var (
+		floor        = int64(-1)
+		localHeight  uint64
+		remoteHeight = remoteHeader.Number.Uint64()
+	)
+	switch d.mode {
+	case FullSync:
+		localHeight = d.blockchain.CurrentBlock().NumberU64()
+	case FastSync:
+		localHeight = d.blockchain.CurrentFastBlock().NumberU64()
+	default:
+		localHeight = d.lightchain.CurrentHeader().Number.Uint64()
+	}
+	p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight)
+	if localHeight >= MaxForkAncestry {
+		floor = int64(localHeight - MaxForkAncestry)
+	}
+	from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight)
 
-	if d.mode == FullSync {
-		ceil = d.blockchain.CurrentBlock().NumberU64()
-	} else if d.mode == FastSync {
-		ceil = d.blockchain.CurrentFastBlock().NumberU64()
-	}
-	if ceil >= MaxForkAncestry {
-		floor = int64(ceil - MaxForkAncestry)
-	}
-	p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
-
-	// Request the topmost blocks to short circuit binary ancestor lookup
-	head := ceil
-	if head > height {
-		head = height
-	}
-	from := int64(head) - int64(MaxHeaderFetch)
-	if from < 0 {
-		from = 0
-	}
-	// Span out with 15 block gaps into the future to catch bad head reports
-	limit := 2 * MaxHeaderFetch / 16
-	count := 1 + int((int64(ceil)-from)/16)
-	if count > limit {
-		count = limit
-	}
-	go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false)
+	p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip)
+	go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false)
 
 	// Wait for the remote response to the head fetch
 	number, hash := uint64(0), common.Hash{}
@@ -647,9 +695,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				return 0, errEmptyHeaderSet
 			}
 			// Make sure the peer's reply conforms to the request
-			for i := 0; i < len(headers); i++ {
-				if number := headers[i].Number.Int64(); number != from+int64(i)*16 {
-					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number)
+			for i, header := range headers {
+				expectNumber := from + int64(i)*int64((skip+1))
+				if number := header.Number.Int64(); number != expectNumber {
+					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
 					return 0, errInvalidChain
 				}
 			}
@@ -657,20 +706,24 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 			finished = true
 			for i := len(headers) - 1; i >= 0; i-- {
 				// Skip any headers that underflow/overflow our requested set
-				if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil {
+				if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max {
 					continue
 				}
 				// Otherwise check if we already know the header or not
 				h := headers[i].Hash()
 				n := headers[i].Number.Uint64()
-				if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) {
-					number, hash = n, h
 
-					// If every header is known, even future ones, the peer straight out lied about its head
-					if number > height && i == limit-1 {
-						p.log.Warn("Lied about chain head", "reported", height, "found", number)
-						return 0, errStallingPeer
+				var known bool
+				switch d.mode {
+				case FullSync:
+					known = d.blockchain.HasBlock(h, n)
+				case FastSync:
+					known = d.blockchain.HasFastBlock(h, n)
+				default:
+					known = d.lightchain.HasHeader(h, n)
 				}
+				if known {
+					number, hash = n, h
 					break
 				}
 			}
@@ -694,10 +747,12 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 		return number, nil
 	}
 	// Ancestor not found, we need to binary search over our chain
-	start, end := uint64(0), head
+	start, end := uint64(0), remoteHeight
 	if floor > 0 {
 		start = uint64(floor)
 	}
+	p.log.Trace("Binary searching for common ancestor", "start", start, "end", end)
+
 	for start+1 < end {
 		// Split our chain interval in two, and request the hash to cross check
 		check := (start + end) / 2
@@ -730,7 +785,17 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				// Modify the search interval based on the response
 				h := headers[0].Hash()
 				n := headers[0].Number.Uint64()
-				if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) {
+
+				var known bool
+				switch d.mode {
+				case FullSync:
+					known = d.blockchain.HasBlock(h, n)
+				case FastSync:
+					known = d.blockchain.HasFastBlock(h, n)
+				default:
+					known = d.lightchain.HasHeader(h, n)
+				}
+				if !known {
 					end = check
 					break
 				}
@@ -740,6 +805,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 					return 0, errBadPeer
 				}
 				start = check
+				hash = h
 
 			case <-timeout:
 				p.log.Debug("Waiting for search header timed out", "elapsed", ttl)
@@ -1246,7 +1312,7 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er
 				}
 				// If no headers were retrieved at all, the peer violated its TD promise that it had a
 				// better chain compared to ours. The only exception is if its promised blocks were
-				// already imported by other means (e.g. fecher):
+				// already imported by other means (e.g. fetcher):
 				//
 				// R <remote peer>, L <local node>: Both at block 10
 				// R: Mine block 11, and propagate it to L
@@ -1415,7 +1481,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 	defer stateSync.Cancel()
 	go func() {
 		if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
-			d.queue.Close() // wake up WaitResults
+			d.queue.Close() // wake up Results
 		}
 	}()
 	// Figure out the ideal pivot block. Note, that this goalpost may move if the
@@ -1473,7 +1539,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 				defer stateSync.Cancel()
 				go func() {
 					if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
-						d.queue.Close() // wake up WaitResults
+						d.queue.Close() // wake up Results
 					}
 				}()
 				oldPivot = P

eth/downloader/peer.go

@@ -229,13 +229,6 @@ func (p *peerConnection) SetHeadersIdle(delivered int) {
 	p.setIdle(p.headerStarted, delivered, &p.headerThroughput, &p.headerIdle)
 }
 
-// SetBlocksIdle sets the peer to idle, allowing it to execute new block retrieval
-// requests. Its estimated block retrieval throughput is updated with that measured
-// just now.
-func (p *peerConnection) SetBlocksIdle(delivered int) {
-	p.setIdle(p.blockStarted, delivered, &p.blockThroughput, &p.blockIdle)
-}
-
 // SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval
 // requests. Its estimated body retrieval throughput is updated with that measured
 // just now.

eth/downloader/queue.go

@@ -143,7 +143,7 @@ func (q *queue) Reset() {
 	q.resultOffset = 0
 }
 
-// Close marks the end of the sync, unblocking WaitResults.
+// Close marks the end of the sync, unblocking Results.
 // It may be called even if the queue is already closed.
 func (q *queue) Close() {
 	q.lock.Lock()
@@ -545,7 +545,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
 		taskQueue.Push(header, -int64(header.Number.Uint64()))
 	}
 	if progress {
-		// Wake WaitResults, resultCache was modified
+		// Wake Results, resultCache was modified
 		q.active.Signal()
 	}
 	// Assemble and return the block download request
@@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest,
 			}
 			// Add the peer to the expiry report along the number of failed requests
 			expiries[id] = len(request.Headers)
-		}
-	}
-	// Remove the expired requests from the pending pool
-	for id := range expiries {
+
+			// Remove the expired requests from the pending pool directly
 			delete(pendPool, id)
 		}
+	}
 	return expiries
 }
 
@@ -857,7 +856,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
 			taskQueue.Push(header, -int64(header.Number.Uint64()))
 		}
 	}
-	// Wake up WaitResults
+	// Wake up Results
 	if accepted > 0 {
 		q.active.Signal()
 	}

eth/downloader/statesync.go

@@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) {
 				s.d.dropPeer(req.peer.id)
 			}
 			// Process all the received blobs and check for stale delivery
-			if err = s.process(req); err != nil {
+			delivered, err := s.process(req)
+			if err != nil {
 				log.Warn("Node data write error", "err", err)
 				return err
 			}
-			req.peer.SetNodeDataIdle(len(req.response))
+			req.peer.SetNodeDataIdle(delivered)
 		}
 	}
 	return nil
@@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) {
 // process iterates over a batch of delivered state data, injecting each item
 // into a running state sync, re-queuing any items that were requested but not
 // delivered.
-func (s *stateSync) process(req *stateReq) error {
+// Returns whether the peer actually managed to deliver anything of value,
+// and any error that occurred
+func (s *stateSync) process(req *stateReq) (int, error) {
 	// Collect processing stats and update progress if valid data was received
-	duplicate, unexpected := 0, 0
+	duplicate, unexpected, successful := 0, 0, 0
 
 	defer func(start time.Time) {
 		if duplicate > 0 || unexpected > 0 {
@@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error {
 
 	// Iterate over all the delivered data and inject one-by-one into the trie
 	progress := false
-
 	for _, blob := range req.response {
 		prog, hash, err := s.processNodeData(blob)
 		switch err {
@@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error {
 			s.numUncommitted++
 			s.bytesUncommitted += len(blob)
 			progress = progress || prog
+			successful++
 		case trie.ErrNotRequested:
 			unexpected++
 		case trie.ErrAlreadyProcessed:
 			duplicate++
 		default:
-			return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
+			return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
 		}
 		if _, ok := req.tasks[hash]; ok {
 			delete(req.tasks, hash)
@@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error {
 		// If we've requested the node too many times already, it may be a malicious
 		// sync where nobody has the right data. Abort.
 		if len(task.attempts) >= npeers {
-			return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
+			return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
 		}
 		// Missing item, place into the retry queue.
 		s.tasks[hash] = task
 	}
-	return nil
+	return successful, nil
 }
 
 // processNodeData tries to inject a trie node data blob delivered from a remote

eth/downloader/testchain_test.go

@@ -0,0 +1,221 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package downloader
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// Test chain parameters.
+var (
+	testKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
+	testDB      = ethdb.NewMemDatabase()
+	testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000))
+)
+
+// The common prefix of all test chains:
+var testChainBase = newTestChain(blockCacheItems+200, testGenesis)
+
+// Different forks on top of the base chain:
+var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain
+
+func init() {
+	var forkLen = int(MaxForkAncestry + 50)
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }()
+	go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }()
+	go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }()
+	wg.Wait()
+}
+
+type testChain struct {
+	genesis  *types.Block
+	chain    []common.Hash
+	headerm  map[common.Hash]*types.Header
+	blockm   map[common.Hash]*types.Block
+	receiptm map[common.Hash][]*types.Receipt
+	tdm      map[common.Hash]*big.Int
+}
+
+// newTestChain creates a blockchain of the given length.
+func newTestChain(length int, genesis *types.Block) *testChain {
+	tc := new(testChain).copy(length)
+	tc.genesis = genesis
+	tc.chain = append(tc.chain, genesis.Hash())
+	tc.headerm[tc.genesis.Hash()] = tc.genesis.Header()
+	tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty()
+	tc.blockm[tc.genesis.Hash()] = tc.genesis
+	tc.generate(length-1, 0, genesis, false)
+	return tc
+}
+
+// makeFork creates a fork on top of the test chain.
+func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain {
+	fork := tc.copy(tc.len() + length)
+	fork.generate(length, seed, tc.headBlock(), heavy)
+	return fork
+}
+
+// shorten creates a copy of the chain with the given length. It panics if the
+// length is longer than the number of available blocks.
+func (tc *testChain) shorten(length int) *testChain {
+	if length > tc.len() {
+		panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len()))
+	}
+	return tc.copy(length)
+}
+
+func (tc *testChain) copy(newlen int) *testChain {
+	cpy := &testChain{
+		genesis:  tc.genesis,
+		headerm:  make(map[common.Hash]*types.Header, newlen),
+		blockm:   make(map[common.Hash]*types.Block, newlen),
+		receiptm: make(map[common.Hash][]*types.Receipt, newlen),
+		tdm:      make(map[common.Hash]*big.Int, newlen),
+	}
+	for i := 0; i < len(tc.chain) && i < newlen; i++ {
+		hash := tc.chain[i]
+		cpy.chain = append(cpy.chain, tc.chain[i])
+		cpy.tdm[hash] = tc.tdm[hash]
+		cpy.blockm[hash] = tc.blockm[hash]
+		cpy.headerm[hash] = tc.headerm[hash]
+		cpy.receiptm[hash] = tc.receiptm[hash]
+	}
+	return cpy
+}
+
+// generate creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent. In addition, every 22th block
+// contains a transaction and every 5th an uncle to allow testing correct block
+// reassembly.
+func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) {
+	// start := time.Now()
+	// defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }()
+
+	blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) {
+		block.SetCoinbase(common.Address{seed})
+		// If a heavy chain is requested, delay blocks to raise difficulty
+		if heavy {
+			block.OffsetTime(-1)
+		}
+		// Include transactions to the miner to make blocks more interesting.
+		if parent == tc.genesis && i%22 == 0 {
+			signer := types.MakeSigner(params.TestChainConfig, block.Number())
+			tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey)
+			if err != nil {
+				panic(err)
+			}
+			block.AddTx(tx)
+		}
+		// if the block number is a multiple of 5, add a bonus uncle to the block
+		if i > 0 && i%5 == 0 {
+			block.AddUncle(&types.Header{
+				ParentHash: block.PrevBlock(i - 1).Hash(),
+				Number:     big.NewInt(block.Number().Int64() - 1),
+			})
+		}
+	})
+
+	// Convert the block-chain into a hash-chain and header/block maps
+	td := new(big.Int).Set(tc.td(parent.Hash()))
+	for i, b := range blocks {
+		td := td.Add(td, b.Difficulty())
+		hash := b.Hash()
+		tc.chain = append(tc.chain, hash)
+		tc.blockm[hash] = b
+		tc.headerm[hash] = b.Header()
+		tc.receiptm[hash] = receipts[i]
+		tc.tdm[hash] = new(big.Int).Set(td)
+	}
+}
+
+// len returns the total number of blocks in the chain.
+func (tc *testChain) len() int {
+	return len(tc.chain)
+}
+
+// headBlock returns the head of the chain.
+func (tc *testChain) headBlock() *types.Block {
+	return tc.blockm[tc.chain[len(tc.chain)-1]]
+}
+
+// td returns the total difficulty of the given block.
+func (tc *testChain) td(hash common.Hash) *big.Int {
+	return tc.tdm[hash]
+}
+
+// headersByHash returns headers in ascending order from the given hash.
+func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header {
+	num, _ := tc.hashToNumber(origin)
+	return tc.headersByNumber(num, amount, skip)
+}
+
+// headersByNumber returns headers in ascending order from the given number.
+func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header {
+	result := make([]*types.Header, 0, amount)
+	for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 {
+		if header, ok := tc.headerm[tc.chain[int(num)]]; ok {
+			result = append(result, header)
+		}
+	}
+	return result
+}
+
+// receipts returns the receipts of the given block hashes.
+func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt {
+	results := make([][]*types.Receipt, 0, len(hashes))
+	for _, hash := range hashes {
+		if receipt, ok := tc.receiptm[hash]; ok {
+			results = append(results, receipt)
+		}
+	}
+	return results
+}
+
+// bodies returns the block bodies of the given block hashes.
+func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) {
+	transactions := make([][]*types.Transaction, 0, len(hashes))
+	uncles := make([][]*types.Header, 0, len(hashes))
+	for _, hash := range hashes {
+		if block, ok := tc.blockm[hash]; ok {
+			transactions = append(transactions, block.Transactions())
+			uncles = append(uncles, block.Uncles())
+		}
+	}
+	return transactions, uncles
+}
+
+func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) {
+	for num, hash := range tc.chain {
+		if hash == target {
+			return uint64(num), true
+		}
+	}
+	return 0, false
+}

eth/gen_config.go

@@ -28,7 +28,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		SkipBcVersionCheck      bool `toml:"-"`
 		DatabaseHandles         int  `toml:"-"`
 		DatabaseCache           int
-		TrieCache               int
+		TrieCleanCache          int
+		TrieDirtyCache          int
 		TrieTimeout             time.Duration
 		Etherbase               common.Address `toml:",omitempty"`
 		MinerNotify             []string       `toml:",omitempty"`
@@ -43,6 +44,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		GPO                     gasprice.Config
 		EnablePreimageRecording bool
 		DocRoot                 string `toml:"-"`
+		EWASMInterpreter        string
+		EVMInterpreter          string
 	}
 	var enc Config
 	enc.Genesis = c.Genesis
@@ -54,7 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.SkipBcVersionCheck = c.SkipBcVersionCheck
 	enc.DatabaseHandles = c.DatabaseHandles
 	enc.DatabaseCache = c.DatabaseCache
-	enc.TrieCache = c.TrieCache
+	enc.TrieCleanCache = c.TrieCleanCache
+	enc.TrieDirtyCache = c.TrieDirtyCache
 	enc.TrieTimeout = c.TrieTimeout
 	enc.Etherbase = c.Etherbase
 	enc.MinerNotify = c.MinerNotify
@@ -69,6 +73,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.GPO = c.GPO
 	enc.EnablePreimageRecording = c.EnablePreimageRecording
 	enc.DocRoot = c.DocRoot
+	enc.EWASMInterpreter = c.EWASMInterpreter
+	enc.EVMInterpreter = c.EVMInterpreter
 	return &enc, nil
 }
 
@@ -84,7 +90,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		SkipBcVersionCheck      *bool `toml:"-"`
 		DatabaseHandles         *int  `toml:"-"`
 		DatabaseCache           *int
-		TrieCache               *int
+		TrieCleanCache          *int
+		TrieDirtyCache          *int
 		TrieTimeout             *time.Duration
 		Etherbase               *common.Address `toml:",omitempty"`
 		MinerNotify             []string        `toml:",omitempty"`
@@ -99,6 +106,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		GPO                     *gasprice.Config
 		EnablePreimageRecording *bool
 		DocRoot                 *string `toml:"-"`
+		EWASMInterpreter        *string
+		EVMInterpreter          *string
 	}
 	var dec Config
 	if err := unmarshal(&dec); err != nil {
@@ -131,8 +140,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.DatabaseCache != nil {
 		c.DatabaseCache = *dec.DatabaseCache
 	}
-	if dec.TrieCache != nil {
-		c.TrieCache = *dec.TrieCache
+	if dec.TrieCleanCache != nil {
+		c.TrieCleanCache = *dec.TrieCleanCache
+	}
+	if dec.TrieDirtyCache != nil {
+		c.TrieDirtyCache = *dec.TrieDirtyCache
 	}
 	if dec.TrieTimeout != nil {
 		c.TrieTimeout = *dec.TrieTimeout
@@ -176,5 +188,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.DocRoot != nil {
 		c.DocRoot = *dec.DocRoot
 	}
+	if dec.EWASMInterpreter != nil {
+		c.EWASMInterpreter = *dec.EWASMInterpreter
+	}
+	if dec.EVMInterpreter != nil {
+		c.EVMInterpreter = *dec.EVMInterpreter
+	}
 	return nil
 }

eth/handler.go

@@ -653,12 +653,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			trueHead = request.Block.ParentHash()
 			trueTD   = new(big.Int).Sub(request.TD, request.Block.Difficulty())
 		)
-		// Update the peers total difficulty if better than the previous
+		// Update the peer's total difficulty if better than the previous
 		if _, td := p.Head(); trueTD.Cmp(td) > 0 {
 			p.SetHead(trueHead, trueTD)
 
 			// Schedule a sync if above ours. Note, this will not fire a sync for a gap of
-			// a singe block (as the true TD is below the propagated block), however this
+			// a single block (as the true TD is below the propagated block), however this
 			// scenario should easily be covered by the fetcher.
 			currentBlock := pm.blockchain.CurrentBlock()
 			if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 {

eth/tracers/tracer.go

@@ -295,6 +295,7 @@ type Tracer struct {
 	costValue   *uint   // Swappable cost value wrapped by a log accessor
 	depthValue  *uint   // Swappable depth value wrapped by a log accessor
 	errorValue  *string // Swappable error value wrapped by a log accessor
+	refundValue *uint   // Swappable refund value wrapped by a log accessor
 
 	ctx map[string]interface{} // Transaction context gathered throughout execution
 	err error                  // Error, if one has occurred
@@ -323,6 +324,7 @@ func New(code string) (*Tracer, error) {
 		gasValue:        new(uint),
 		costValue:       new(uint),
 		depthValue:      new(uint),
+		refundValue:     new(uint),
 	}
 	// Set up builtins for this environment
 	tracer.vm.PushGlobalGoFunction("toHex", func(ctx *duktape.Context) int {
@@ -442,6 +444,9 @@ func New(code string) (*Tracer, error) {
 	tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.depthValue); return 1 })
 	tracer.vm.PutPropString(logObject, "getDepth")
 
+	tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.refundValue); return 1 })
+	tracer.vm.PutPropString(logObject, "getRefund")
+
 	tracer.vm.PushGoFunction(func(ctx *duktape.Context) int {
 		if tracer.errorValue != nil {
 			ctx.PushString(*tracer.errorValue)
@@ -527,6 +532,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost
 		*jst.gasValue = uint(gas)
 		*jst.costValue = uint(cost)
 		*jst.depthValue = uint(depth)
+		*jst.refundValue = uint(env.StateDB.GetRefund())
 
 		jst.errorValue = nil
 		if err != nil {

eth/tracers/tracer_test.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -43,8 +44,14 @@ func (account) ReturnGas(*big.Int)                                  {}
 func (account) SetCode(common.Hash, []byte)                         {}
 func (account) ForEachStorage(cb func(key, value common.Hash) bool) {}
 
+type dummyStatedb struct {
+	state.StateDB
+}
+
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
+
 func runTrace(tracer *Tracer) (json.RawMessage, error) {
-	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
+	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
 
 	contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000)
 	contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0}
@@ -126,7 +133,7 @@ func TestHaltBetweenSteps(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
+	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
 	contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0)
 
 	tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil)

ethclient/ethclient.go

@@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb
 // FilterLogs executes a filter query.
 func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
 	var result []types.Log
-	err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q))
+	arg, err := toFilterArg(q)
+	if err != nil {
+		return nil, err
+	}
+	err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg)
 	return result, err
 }
 
 // SubscribeFilterLogs subscribes to the results of a streaming filter query.
 func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
-	return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q))
+	arg, err := toFilterArg(q)
+	if err != nil {
+		return nil, err
+	}
+	return ec.c.EthSubscribe(ctx, ch, "logs", arg)
 }
 
-func toFilterArg(q ethereum.FilterQuery) interface{} {
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
 	arg := map[string]interface{}{
-		"fromBlock": toBlockNumArg(q.FromBlock),
-		"toBlock":   toBlockNumArg(q.ToBlock),
 		"address": q.Addresses,
 		"topics":  q.Topics,
 	}
+	if q.BlockHash != nil {
+		arg["blockHash"] = *q.BlockHash
+		if q.FromBlock != nil || q.ToBlock != nil {
+			return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+		}
+	} else {
 		if q.FromBlock == nil {
 			arg["fromBlock"] = "0x0"
+		} else {
+			arg["fromBlock"] = toBlockNumArg(q.FromBlock)
 		}
-	return arg
+		arg["toBlock"] = toBlockNumArg(q.ToBlock)
+	}
+	return arg, nil
 }
 
 // Pending State

ethclient/ethclient_test.go

@@ -16,7 +16,15 @@
 
 package ethclient
 
-import "github.com/ethereum/go-ethereum"
+import (
+	"fmt"
+	"math/big"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+)
 
 // Verify that Client implements the ethereum interfaces.
 var (
@@ -32,3 +40,113 @@ var (
 	// _ = ethereum.PendingStateEventer(&Client{})
 	_ = ethereum.PendingContractCaller(&Client{})
 )
+
+func TestToFilterArg(t *testing.T) {
+	blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+	addresses := []common.Address{
+		common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"),
+	}
+	blockHash := common.HexToHash(
+		"0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb",
+	)
+
+	for _, testCase := range []struct {
+		name   string
+		input  ethereum.FilterQuery
+		output interface{}
+		err    error
+	}{
+		{
+			"without BlockHash",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				FromBlock: big.NewInt(1),
+				ToBlock:   big.NewInt(2),
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"fromBlock": "0x1",
+				"toBlock":   "0x2",
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with nil fromBlock and nil toBlock",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"fromBlock": "0x0",
+				"toBlock":   "latest",
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with blockhash",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"blockHash": blockHash,
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with blockhash and from block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				FromBlock: big.NewInt(1),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+		{
+			"with blockhash and to block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				ToBlock:   big.NewInt(1),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+		{
+			"with blockhash and both from / to block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				FromBlock: big.NewInt(1),
+				ToBlock:   big.NewInt(2),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			output, err := toFilterArg(testCase.input)
+			if (testCase.err == nil) != (err == nil) {
+				t.Fatalf("expected error %v but got %v", testCase.err, err)
+			}
+			if testCase.err != nil {
+				if testCase.err.Error() != err.Error() {
+					t.Fatalf("expected error %v but got %v", testCase.err, err)
+				}
+			} else if !reflect.DeepEqual(testCase.output, output) {
+				t.Fatalf("expected filter arg %v but got %v", testCase.output, output)
+			}
+		})
+	}
+}

ethdb/database.go

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+// +build !js
+
 package ethdb
 
 import (
@@ -380,71 +382,3 @@ func (b *ldbBatch) Reset() {
 	b.b.Reset()
 	b.size = 0
 }
-
-type table struct {
-	db     Database
-	prefix string
-}
-
-// NewTable returns a Database object that prefixes all keys with a given
-// string.
-func NewTable(db Database, prefix string) Database {
-	return &table{
-		db:     db,
-		prefix: prefix,
-	}
-}
-
-func (dt *table) Put(key []byte, value []byte) error {
-	return dt.db.Put(append([]byte(dt.prefix), key...), value)
-}
-
-func (dt *table) Has(key []byte) (bool, error) {
-	return dt.db.Has(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Get(key []byte) ([]byte, error) {
-	return dt.db.Get(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Delete(key []byte) error {
-	return dt.db.Delete(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Close() {
-	// Do nothing; don't close the underlying DB.
-}
-
-type tableBatch struct {
-	batch  Batch
-	prefix string
-}
-
-// NewTableBatch returns a Batch object which prefixes all keys with a given string.
-func NewTableBatch(db Database, prefix string) Batch {
-	return &tableBatch{db.NewBatch(), prefix}
-}
-
-func (dt *table) NewBatch() Batch {
-	return &tableBatch{dt.db.NewBatch(), dt.prefix}
-}
-
-func (tb *tableBatch) Put(key, value []byte) error {
-	return tb.batch.Put(append([]byte(tb.prefix), key...), value)
-}
-
-func (tb *tableBatch) Delete(key []byte) error {
-	return tb.batch.Delete(append([]byte(tb.prefix), key...))
-}
-
-func (tb *tableBatch) Write() error {
-	return tb.batch.Write()
-}
-
-func (tb *tableBatch) ValueSize() int {
-	return tb.batch.ValueSize()
-}
-
-func (tb *tableBatch) Reset() {
-	tb.batch.Reset()
-}

ethdb/database_js.go

@@ -0,0 +1,68 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build js
+
+package ethdb
+
+import (
+	"errors"
+)
+
+var errNotSupported = errors.New("ethdb: not supported")
+
+type LDBDatabase struct {
+}
+
+// NewLDBDatabase returns a LevelDB wrapped object.
+func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
+	return nil, errNotSupported
+}
+
+// Path returns the path to the database directory.
+func (db *LDBDatabase) Path() string {
+	return ""
+}
+
+// Put puts the given key / value to the queue
+func (db *LDBDatabase) Put(key []byte, value []byte) error {
+	return errNotSupported
+}
+
+func (db *LDBDatabase) Has(key []byte) (bool, error) {
+	return false, errNotSupported
+}
+
+// Get returns the given key if it's present.
+func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
+	return nil, errNotSupported
+}
+
+// Delete deletes the key from the queue and database
+func (db *LDBDatabase) Delete(key []byte) error {
+	return errNotSupported
+}
+
+func (db *LDBDatabase) Close() {
+}
+
+// Meter configures the database metrics collectors and
+func (db *LDBDatabase) Meter(prefix string) {
+}
+
+func (db *LDBDatabase) NewBatch() Batch {
+	return nil
+}

ethdb/database_js_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The go-ethereum Authors
+// Copyright 2014 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,13 +14,12 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package state
+// +build js
 
-// Store defines methods required to get, set, delete values for different keys
-// and close the underlying resources.
-type Store interface {
-	Get(key string, i interface{}) (err error)
-	Put(key string, i interface{}) (err error)
-	Delete(key string) (err error)
-	Close() error
-}
+package ethdb_test
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+var _ ethdb.Database = &ethdb.LDBDatabase{}

ethdb/database_test.go

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+// +build !js
+
 package ethdb_test
 
 import (

ethdb/table.go

@@ -0,0 +1,51 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethdb
+
+type table struct {
+	db     Database
+	prefix string
+}
+
+// NewTable returns a Database object that prefixes all keys with a given
+// string.
+func NewTable(db Database, prefix string) Database {
+	return &table{
+		db:     db,
+		prefix: prefix,
+	}
+}
+
+func (dt *table) Put(key []byte, value []byte) error {
+	return dt.db.Put(append([]byte(dt.prefix), key...), value)
+}
+
+func (dt *table) Has(key []byte) (bool, error) {
+	return dt.db.Has(append([]byte(dt.prefix), key...))
+}
+
+func (dt *table) Get(key []byte) ([]byte, error) {
+	return dt.db.Get(append([]byte(dt.prefix), key...))
+}
+
+func (dt *table) Delete(key []byte) error {
+	return dt.db.Delete(append([]byte(dt.prefix), key...))
+}
+
+func (dt *table) Close() {
+	// Do nothing; don't close the underlying DB.
+}

ethdb/table_batch.go

@@ -0,0 +1,51 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethdb
+
+type tableBatch struct {
+	batch  Batch
+	prefix string
+}
+
+// NewTableBatch returns a Batch object which prefixes all keys with a given string.
+func NewTableBatch(db Database, prefix string) Batch {
+	return &tableBatch{db.NewBatch(), prefix}
+}
+
+func (dt *table) NewBatch() Batch {
+	return &tableBatch{dt.db.NewBatch(), dt.prefix}
+}
+
+func (tb *tableBatch) Put(key, value []byte) error {
+	return tb.batch.Put(append([]byte(tb.prefix), key...), value)
+}
+
+func (tb *tableBatch) Delete(key []byte) error {
+	return tb.batch.Delete(append([]byte(tb.prefix), key...))
+}
+
+func (tb *tableBatch) Write() error {
+	return tb.batch.Write()
+}
+
+func (tb *tableBatch) ValueSize() int {
+	return tb.batch.ValueSize()
+}
+
+func (tb *tableBatch) Reset() {
+	tb.batch.Reset()
+}

event/event_test.go

@@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) {
 	}
 }
 
-func emptySubscriber(mux *TypeMux, types ...interface{}) {
+func emptySubscriber(mux *TypeMux) {
 	s := mux.Subscribe(testEvent(0))
 	go func() {
 		for range s.Chan() {
@@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) {
 func BenchmarkPostConcurrent(b *testing.B) {
 	var mux = new(TypeMux)
 	defer mux.Stop()
-	emptySubscriber(mux, testEvent(0))
-	emptySubscriber(mux, testEvent(0))
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
+	emptySubscriber(mux)
+	emptySubscriber(mux)
 
 	var wg sync.WaitGroup
 	poster := func() {

event/filter/filter.go

@@ -1,95 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package filter implements event filters.
-package filter
-
-import "reflect"
-
-type Filter interface {
-	Compare(Filter) bool
-	Trigger(data interface{})
-}
-
-type FilterEvent struct {
-	filter Filter
-	data   interface{}
-}
-
-type Filters struct {
-	id       int
-	watchers map[int]Filter
-	ch       chan FilterEvent
-
-	quit chan struct{}
-}
-
-func New() *Filters {
-	return &Filters{
-		ch:       make(chan FilterEvent),
-		watchers: make(map[int]Filter),
-		quit:     make(chan struct{}),
-	}
-}
-
-func (f *Filters) Start() {
-	go f.loop()
-}
-
-func (f *Filters) Stop() {
-	close(f.quit)
-}
-
-func (f *Filters) Notify(filter Filter, data interface{}) {
-	f.ch <- FilterEvent{filter, data}
-}
-
-func (f *Filters) Install(watcher Filter) int {
-	f.watchers[f.id] = watcher
-	f.id++
-
-	return f.id - 1
-}
-
-func (f *Filters) Uninstall(id int) {
-	delete(f.watchers, id)
-}
-
-func (f *Filters) loop() {
-out:
-	for {
-		select {
-		case <-f.quit:
-			break out
-		case event := <-f.ch:
-			for _, watcher := range f.watchers {
-				if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) {
-					if watcher.Compare(event.filter) {
-						watcher.Trigger(event.data)
-					}
-				}
-			}
-		}
-	}
-}
-
-func (f *Filters) Match(a, b Filter) bool {
-	return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b)
-}
-
-func (f *Filters) Get(i int) Filter {
-	return f.watchers[i]
-}

event/filter/filter_test.go

@@ -1,60 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package filter
-
-import (
-	"testing"
-	"time"
-)
-
-// Simple test to check if baseline matching/mismatching filtering works.
-func TestFilters(t *testing.T) {
-	fm := New()
-	fm.Start()
-
-	// Register two filters to catch posted data
-	first := make(chan struct{})
-	fm.Install(Generic{
-		Str1: "hello",
-		Fn: func(data interface{}) {
-			first <- struct{}{}
-		},
-	})
-	second := make(chan struct{})
-	fm.Install(Generic{
-		Str1: "hello1",
-		Str2: "hello",
-		Fn: func(data interface{}) {
-			second <- struct{}{}
-		},
-	})
-	// Post an event that should only match the first filter
-	fm.Notify(Generic{Str1: "hello"}, true)
-	fm.Stop()
-
-	// Ensure only the mathcing filters fire
-	select {
-	case <-first:
-	case <-time.After(100 * time.Millisecond):
-		t.Error("matching filter timed out")
-	}
-	select {
-	case <-second:
-		t.Error("mismatching filter fired")
-	case <-time.After(100 * time.Millisecond):
-	}
-}

internal/cmdtest/test_cmd.go

@@ -27,6 +27,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"text/template"
 	"time"
@@ -50,6 +51,8 @@ type TestCmd struct {
 	stdout *bufio.Reader
 	stdin  io.WriteCloser
 	stderr *testlogger
+	// Err will contain the process exit error or interrupt signal error
+	Err error
 }
 
 // Run exec's the current binary using name as argv[0] which will trigger the
@@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() {
 }
 
 func (tt *TestCmd) WaitExit() {
-	tt.cmd.Wait()
+	tt.Err = tt.cmd.Wait()
 }
 
 func (tt *TestCmd) Interrupt() {
-	tt.cmd.Process.Signal(os.Interrupt)
+	tt.Err = tt.cmd.Process.Signal(os.Interrupt)
+}
+
+// ExitStatus exposes the process' OS exit code
+// It will only return a valid value after the process has finished.
+func (tt *TestCmd) ExitStatus() int {
+	if tt.Err != nil {
+		exitErr := tt.Err.(*exec.ExitError)
+		if exitErr != nil {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+				return status.ExitStatus()
+			}
+		}
+	}
+	return 0
 }
 
 // StderrText returns any stderr output written so far.

internal/ethapi/api.go

@@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
 	return fetchKeystore(s.am).Lock(addr) == nil
 }
 
-// signTransactions sets defaults and signs the given transaction
+// signTransaction sets defaults and signs the given transaction
 // NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
 // and release it after the transaction has been submitted to the tx pool
 func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) {
@@ -508,6 +508,72 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add
 	return (*hexutil.Big)(state.GetBalance(address)), state.Error()
 }
 
+// Result structs for GetProof
+type AccountResult struct {
+	Address      common.Address  `json:"address"`
+	AccountProof []string        `json:"accountProof"`
+	Balance      *hexutil.Big    `json:"balance"`
+	CodeHash     common.Hash     `json:"codeHash"`
+	Nonce        hexutil.Uint64  `json:"nonce"`
+	StorageHash  common.Hash     `json:"storageHash"`
+	StorageProof []StorageResult `json:"storageProof"`
+}
+type StorageResult struct {
+	Key   string       `json:"key"`
+	Value *hexutil.Big `json:"value"`
+	Proof []string     `json:"proof"`
+}
+
+// GetProof returns the Merkle-proof for a given account and optionally some storage keys.
+func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNr rpc.BlockNumber) (*AccountResult, error) {
+	state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr)
+	if state == nil || err != nil {
+		return nil, err
+	}
+
+	storageTrie := state.StorageTrie(address)
+	storageHash := types.EmptyRootHash
+	codeHash := state.GetCodeHash(address)
+	storageProof := make([]StorageResult, len(storageKeys))
+
+	// if we have a storageTrie, (which means the account exists), we can update the storagehash
+	if storageTrie != nil {
+		storageHash = storageTrie.Hash()
+	} else {
+		// no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray.
+		codeHash = crypto.Keccak256Hash(nil)
+	}
+
+	// create the proof for the storageKeys
+	for i, key := range storageKeys {
+		if storageTrie != nil {
+			proof, storageError := state.GetStorageProof(address, common.HexToHash(key))
+			if storageError != nil {
+				return nil, storageError
+			}
+			storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)}
+		} else {
+			storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}}
+		}
+	}
+
+	// create the accountProof
+	accountProof, proofErr := state.GetProof(address)
+	if proofErr != nil {
+		return nil, proofErr
+	}
+
+	return &AccountResult{
+		Address:      address,
+		AccountProof: common.ToHexArray(accountProof),
+		Balance:      (*hexutil.Big)(state.GetBalance(address)),
+		CodeHash:     codeHash,
+		Nonce:        hexutil.Uint64(state.GetNonce(address)),
+		StorageHash:  storageHash,
+		StorageProof: storageProof,
+	}, state.Error()
+}
+
 // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all
 // transactions in the block are returned in full detail, otherwise only the transaction hash is returned.
 func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) {

internal/web3ext/web3ext.go

@@ -481,6 +481,12 @@ web3._extend({
 			params: 2,
 			inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex]
 		}),
+		new web3._extend.Method({
+			name: 'getProof',
+			call: 'eth_getProof',
+			params: 3,
+			inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter]
+		}),
 	],
 	properties: [
 		new web3._extend.Property({

les/serverpool.go

@@ -683,7 +683,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
 }
 
 func encodePubkey64(pub *ecdsa.PublicKey) []byte {
-	return crypto.FromECDSAPub(pub)[:1]
+	return crypto.FromECDSAPub(pub)[1:]
 }
 
 func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {

light/postprocess.go

@@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co
 		diskdb:      db,
 		odr:         odr,
 		trieTable:   trieTable,
-		triedb:      trie.NewDatabase(trieTable),
+		triedb:      trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
 		sectionSize: size,
 	}
 	return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht")
@@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
 		diskdb:     db,
 		odr:        odr,
 		trieTable:  trieTable,
-		triedb:     trie.NewDatabase(trieTable),
+		triedb:     trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
 		parentSize: parentSize,
 		size:       size,
 	}

metrics/counter.go

@@ -1,6 +1,8 @@
 package metrics
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
 
 // Counters hold an int64 value that can be incremented and decremented.
 type Counter interface {
@@ -20,6 +22,17 @@ func GetOrRegisterCounter(name string, r Registry) Counter {
 	return r.GetOrRegister(name, NewCounter).(Counter)
 }
 
+// GetOrRegisterCounterForced returns an existing Counter or constructs and registers a
+// new Counter no matter the global switch is enabled or not.
+// Be sure to unregister the counter from the registry once it is of no use to
+// allow for garbage collection.
+func GetOrRegisterCounterForced(name string, r Registry) Counter {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewCounterForced).(Counter)
+}
+
 // NewCounter constructs a new StandardCounter.
 func NewCounter() Counter {
 	if !Enabled {
@@ -28,6 +41,12 @@ func NewCounter() Counter {
 	return &StandardCounter{0}
 }
 
+// NewCounterForced constructs a new StandardCounter and returns it no matter if
+// the global switch is enabled or not.
+func NewCounterForced() Counter {
+	return &StandardCounter{0}
+}
+
 // NewRegisteredCounter constructs and registers a new StandardCounter.
 func NewRegisteredCounter(name string, r Registry) Counter {
 	c := NewCounter()
@@ -38,6 +57,19 @@ func NewRegisteredCounter(name string, r Registry) Counter {
 	return c
 }
 
+// NewRegisteredCounterForced constructs and registers a new StandardCounter
+// and launches a goroutine no matter the global switch is enabled or not.
+// Be sure to unregister the counter from the registry once it is of no use to
+// allow for garbage collection.
+func NewRegisteredCounterForced(name string, r Registry) Counter {
+	c := NewCounterForced()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
 // CounterSnapshot is a read-only copy of another Counter.
 type CounterSnapshot int64
 

metrics/registry.go

@@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() {
 	r.underlying.UnregisterAll()
 }
 
-var DefaultRegistry Registry = NewRegistry()
+var (
+	DefaultRegistry   = NewRegistry()
+	EphemeralRegistry = NewRegistry()
+)
 
 // Call the given function for each registered metric.
 func Each(f func(string, interface{})) {

miner/stress_clique.go

@@ -22,7 +22,6 @@ package main
 import (
 	"bytes"
 	"crypto/ecdsa"
-	"fmt"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -40,7 +39,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -62,11 +61,11 @@ func main() {
 
 	var (
 		nodes  []*node.Node
-		enodes []string
+		enodes []*enode.Node
 	)
 	for _, sealer := range sealers {
 		// Start the node and wait until it's up
-		node, err := makeSealer(genesis, enodes)
+		node, err := makeSealer(genesis)
 		if err != nil {
 			panic(err)
 		}
@@ -76,18 +75,12 @@ func main() {
 			time.Sleep(250 * time.Millisecond)
 		}
 		// Connect the node to al the previous ones
-		for _, enode := range enodes {
-			enode, err := discover.ParseNode(enode)
-			if err != nil {
-				panic(err)
+		for _, n := range enodes {
+			node.Server().AddPeer(n)
 		}
-			node.Server().AddPeer(enode)
-		}
-		// Start tracking the node and it's enode url
+		// Start tracking the node and it's enode
 		nodes = append(nodes, node)
-
-		enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
-		enodes = append(enodes, enode)
+		enodes = append(enodes, node.Server().Self())
 
 		// Inject the signer key and start sealing with it
 		store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core
 	return genesis
 }
 
-func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeSealer(genesis *core.Genesis) (*node.Node, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := ioutil.TempDir("", "")
 

miner/stress_ethash.go

@@ -21,7 +21,6 @@ package main
 
 import (
 	"crypto/ecdsa"
-	"fmt"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -41,7 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -62,11 +61,11 @@ func main() {
 
 	var (
 		nodes  []*node.Node
-		enodes []string
+		enodes []*enode.Node
 	)
 	for i := 0; i < 4; i++ {
 		// Start the node and wait until it's up
-		node, err := makeMiner(genesis, enodes)
+		node, err := makeMiner(genesis)
 		if err != nil {
 			panic(err)
 		}
@@ -76,18 +75,12 @@ func main() {
 			time.Sleep(250 * time.Millisecond)
 		}
 		// Connect the node to al the previous ones
-		for _, enode := range enodes {
-			enode, err := discover.ParseNode(enode)
-			if err != nil {
-				panic(err)
+		for _, n := range enodes {
+			node.Server().AddPeer(n)
 		}
-			node.Server().AddPeer(enode)
-		}
-		// Start tracking the node and it's enode url
+		// Start tracking the node and it's enode
 		nodes = append(nodes, node)
-
-		enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
-		enodes = append(enodes, enode)
+		enodes = append(enodes, node.Server().Self())
 
 		// Inject the signer key and start sealing with it
 		store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis {
 	return genesis
 }
 
-func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeMiner(genesis *core.Genesis) (*node.Node, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := ioutil.TempDir("", "")
 

node/node.go

@@ -549,11 +549,23 @@ func (n *Node) IPCEndpoint() string {
 
 // HTTPEndpoint retrieves the current HTTP endpoint used by the protocol stack.
 func (n *Node) HTTPEndpoint() string {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	if n.httpListener != nil {
+		return n.httpListener.Addr().String()
+	}
 	return n.httpEndpoint
 }
 
 // WSEndpoint retrieves the current WS endpoint used by the protocol stack.
 func (n *Node) WSEndpoint() string {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	if n.wsListener != nil {
+		return n.wsListener.Addr().String()
+	}
 	return n.wsEndpoint
 }
 

node/node_test.go

@@ -454,9 +454,9 @@ func TestProtocolGather(t *testing.T) {
 		Count int
 		Maker InstrumentingWrapper
 	}{
-		"Zero Protocols":  {0, InstrumentedServiceMakerA},
-		"Single Protocol": {1, InstrumentedServiceMakerB},
-		"Many Protocols":  {25, InstrumentedServiceMakerC},
+		"zero": {0, InstrumentedServiceMakerA},
+		"one":  {1, InstrumentedServiceMakerB},
+		"many": {10, InstrumentedServiceMakerC},
 	}
 	for id, config := range services {
 		protocols := make([]p2p.Protocol, config.Count)
@@ -480,7 +480,7 @@ func TestProtocolGather(t *testing.T) {
 	defer stack.Stop()
 
 	protocols := stack.Server().Protocols
-	if len(protocols) != 26 {
+	if len(protocols) != 11 {
 		t.Fatalf("mismatching number of protocols launched: have %d, want %d", len(protocols), 26)
 	}
 	for id, config := range services {

p2p/dial.go

@@ -71,6 +71,7 @@ type dialstate struct {
 	maxDynDials int
 	ntab        discoverTable
 	netrestrict *netutil.Netlist
+	self        enode.ID
 
 	lookupRunning bool
 	dialing       map[enode.ID]connFlag
@@ -84,7 +85,6 @@ type dialstate struct {
 }
 
 type discoverTable interface {
-	Self() *enode.Node
 	Close()
 	Resolve(*enode.Node) *enode.Node
 	LookupRandom() []*enode.Node
@@ -126,10 +126,11 @@ type waitExpireTask struct {
 	time.Duration
 }
 
-func newDialState(static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate {
+func newDialState(self enode.ID, static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate {
 	s := &dialstate{
 		maxDynDials: maxdyn,
 		ntab:        ntab,
+		self:        self,
 		netrestrict: netrestrict,
 		static:      make(map[enode.ID]*dialTask),
 		dialing:     make(map[enode.ID]connFlag),
@@ -266,7 +267,7 @@ func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error {
 		return errAlreadyDialing
 	case peers[n.ID()] != nil:
 		return errAlreadyConnected
-	case s.ntab != nil && n.ID() == s.ntab.Self().ID():
+	case n.ID() == s.self:
 		return errSelf
 	case s.netrestrict != nil && !s.netrestrict.Contains(n.IP()):
 		return errNotWhitelisted
@@ -349,7 +350,7 @@ func (t *dialTask) dial(srv *Server, dest *enode.Node) error {
 	if err != nil {
 		return &dialError{err}
 	}
-	mfd := newMeteredConn(fd, false)
+	mfd := newMeteredConn(fd, false, dest.IP())
 	return srv.SetupConn(mfd, t.flags, dest)
 }
 

p2p/dial_test.go

@@ -89,7 +89,7 @@ func (t fakeTable) ReadRandomNodes(buf []*enode.Node) int { return copy(buf, t)
 // This test checks that dynamic dials are launched from discovery results.
 func TestDialStateDynDial(t *testing.T) {
 	runDialTest(t, dialtest{
-		init: newDialState(nil, nil, fakeTable{}, 5, nil),
+		init: newDialState(enode.ID{}, nil, nil, fakeTable{}, 5, nil),
 		rounds: []round{
 			// A discovery query is launched.
 			{
@@ -236,7 +236,7 @@ func TestDialStateDynDialBootnode(t *testing.T) {
 		newNode(uintID(8), nil),
 	}
 	runDialTest(t, dialtest{
-		init: newDialState(nil, bootnodes, table, 5, nil),
+		init: newDialState(enode.ID{}, nil, bootnodes, table, 5, nil),
 		rounds: []round{
 			// 2 dynamic dials attempted, bootnodes pending fallback interval
 			{
@@ -324,7 +324,7 @@ func TestDialStateDynDialFromTable(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(nil, nil, table, 10, nil),
+		init: newDialState(enode.ID{}, nil, nil, table, 10, nil),
 		rounds: []round{
 			// 5 out of 8 of the nodes returned by ReadRandomNodes are dialed.
 			{
@@ -430,7 +430,7 @@ func TestDialStateNetRestrict(t *testing.T) {
 	restrict.Add("127.0.2.0/24")
 
 	runDialTest(t, dialtest{
-		init: newDialState(nil, nil, table, 10, restrict),
+		init: newDialState(enode.ID{}, nil, nil, table, 10, restrict),
 		rounds: []round{
 			{
 				new: []task{
@@ -453,7 +453,7 @@ func TestDialStateStaticDial(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
 		rounds: []round{
 			// Static dials are launched for the nodes that
 			// aren't yet connected.
@@ -557,7 +557,7 @@ func TestDialStaticAfterReset(t *testing.T) {
 		},
 	}
 	dTest := dialtest{
-		init:   newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		init:   newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
 		rounds: rounds,
 	}
 	runDialTest(t, dTest)
@@ -578,7 +578,7 @@ func TestDialStateCache(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
 		rounds: []round{
 			// Static dials are launched for the nodes that
 			// aren't yet connected.
@@ -640,7 +640,7 @@ func TestDialStateCache(t *testing.T) {
 func TestDialResolve(t *testing.T) {
 	resolved := newNode(uintID(1), net.IP{127, 0, 55, 234})
 	table := &resolveMock{answer: resolved}
-	state := newDialState(nil, nil, table, 0, nil)
+	state := newDialState(enode.ID{}, nil, nil, table, 0, nil)
 
 	// Check that the task is generated with an incomplete ID.
 	dest := newNode(uintID(1), nil)

p2p/discover/table.go

@@ -72,21 +72,20 @@ type Table struct {
 	ips     netutil.DistinctNetSet
 
 	db         *enode.DB // database of known nodes
+	net        transport
 	refreshReq chan chan struct{}
 	initDone   chan struct{}
 	closeReq   chan struct{}
 	closed     chan struct{}
 
 	nodeAddedHook func(*node) // for testing
-
-	net  transport
-	self *node // metadata of the local node
 }
 
 // transport is implemented by the UDP transport.
 // it is an interface so we can test without opening lots of UDP
 // sockets and without generating a private key.
 type transport interface {
+	self() *enode.Node
 	ping(enode.ID, *net.UDPAddr) error
 	findnode(toid enode.ID, addr *net.UDPAddr, target encPubkey) ([]*node, error)
 	close()
@@ -100,11 +99,10 @@ type bucket struct {
 	ips          netutil.DistinctNetSet
 }
 
-func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.Node) (*Table, error) {
+func newTable(t transport, db *enode.DB, bootnodes []*enode.Node) (*Table, error) {
 	tab := &Table{
 		net:        t,
 		db:         db,
-		self:       wrapNode(self),
 		refreshReq: make(chan chan struct{}),
 		initDone:   make(chan struct{}),
 		closeReq:   make(chan struct{}),
@@ -127,6 +125,10 @@ func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.No
 	return tab, nil
 }
 
+func (tab *Table) self() *enode.Node {
+	return tab.net.self()
+}
+
 func (tab *Table) seedRand() {
 	var b [8]byte
 	crand.Read(b[:])
@@ -136,11 +138,6 @@ func (tab *Table) seedRand() {
 	tab.mutex.Unlock()
 }
 
-// Self returns the local node.
-func (tab *Table) Self() *enode.Node {
-	return unwrapNode(tab.self)
-}
-
 // ReadRandomNodes fills the given slice with random nodes from the table. The results
 // are guaranteed to be unique for a single invocation, no node will appear twice.
 func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) {
@@ -183,6 +180,10 @@ func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) {
 
 // Close terminates the network listener and flushes the node database.
 func (tab *Table) Close() {
+	if tab.net != nil {
+		tab.net.close()
+	}
+
 	select {
 	case <-tab.closed:
 		// already closed.
@@ -257,7 +258,7 @@ func (tab *Table) lookup(targetKey encPubkey, refreshIfEmpty bool) []*node {
 	)
 	// don't query further if we hit ourself.
 	// unlikely to happen often in practice.
-	asked[tab.self.ID()] = true
+	asked[tab.self().ID()] = true
 
 	for {
 		tab.mutex.Lock()
@@ -340,8 +341,8 @@ func (tab *Table) loop() {
 		revalidate     = time.NewTimer(tab.nextRevalidateTime())
 		refresh        = time.NewTicker(refreshInterval)
 		copyNodes      = time.NewTicker(copyNodesInterval)
-		revalidateDone = make(chan struct{})
 		refreshDone    = make(chan struct{})           // where doRefresh reports completion
+		revalidateDone chan struct{}                   // where doRevalidate reports completion
 		waiting        = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs
 	)
 	defer refresh.Stop()
@@ -372,9 +373,11 @@ loop:
 			}
 			waiting, refreshDone = nil, nil
 		case <-revalidate.C:
+			revalidateDone = make(chan struct{})
 			go tab.doRevalidate(revalidateDone)
 		case <-revalidateDone:
 			revalidate.Reset(tab.nextRevalidateTime())
+			revalidateDone = nil
 		case <-copyNodes.C:
 			go tab.copyLiveNodes()
 		case <-tab.closeReq:
@@ -382,15 +385,15 @@ loop:
 		}
 	}
 
-	if tab.net != nil {
-		tab.net.close()
-	}
 	if refreshDone != nil {
 		<-refreshDone
 	}
 	for _, ch := range waiting {
 		close(ch)
 	}
+	if revalidateDone != nil {
+		<-revalidateDone
+	}
 	close(tab.closed)
 }
 
@@ -408,7 +411,7 @@ func (tab *Table) doRefresh(done chan struct{}) {
 	// Run self lookup to discover new neighbor nodes.
 	// We can only do this if we have a secp256k1 identity.
 	var key ecdsa.PublicKey
-	if err := tab.self.Load((*enode.Secp256k1)(&key)); err == nil {
+	if err := tab.self().Load((*enode.Secp256k1)(&key)); err == nil {
 		tab.lookup(encodePubkey(&key), false)
 	}
 
@@ -530,7 +533,7 @@ func (tab *Table) len() (n int) {
 
 // bucket returns the bucket for the given node ID hash.
 func (tab *Table) bucket(id enode.ID) *bucket {
-	d := enode.LogDist(tab.self.ID(), id)
+	d := enode.LogDist(tab.self().ID(), id)
 	if d <= bucketMinDistance {
 		return tab.buckets[0]
 	}
@@ -543,7 +546,7 @@ func (tab *Table) bucket(id enode.ID) *bucket {
 //
 // The caller must not hold tab.mutex.
 func (tab *Table) add(n *node) {
-	if n.ID() == tab.self.ID() {
+	if n.ID() == tab.self().ID() {
 		return
 	}
 
@@ -576,7 +579,7 @@ func (tab *Table) stuff(nodes []*node) {
 	defer tab.mutex.Unlock()
 
 	for _, n := range nodes {
-		if n.ID() == tab.self.ID() {
+		if n.ID() == tab.self().ID() {
 			continue // don't add self
 		}
 		b := tab.bucket(n.ID())

p2p/discover/table_test.go

@@ -141,7 +141,7 @@ func TestTable_IPLimit(t *testing.T) {
 	defer db.Close()
 
 	for i := 0; i < tableIPLimit+1; i++ {
-		n := nodeAtDistance(tab.self.ID(), i, net.IP{172, 0, 1, byte(i)})
+		n := nodeAtDistance(tab.self().ID(), i, net.IP{172, 0, 1, byte(i)})
 		tab.add(n)
 	}
 	if tab.len() > tableIPLimit {
@@ -158,7 +158,7 @@ func TestTable_BucketIPLimit(t *testing.T) {
 
 	d := 3
 	for i := 0; i < bucketIPLimit+1; i++ {
-		n := nodeAtDistance(tab.self.ID(), d, net.IP{172, 0, 1, byte(i)})
+		n := nodeAtDistance(tab.self().ID(), d, net.IP{172, 0, 1, byte(i)})
 		tab.add(n)
 	}
 	if tab.len() > bucketIPLimit {
@@ -240,7 +240,7 @@ func TestTable_ReadRandomNodesGetAll(t *testing.T) {
 
 		for i := 0; i < len(buf); i++ {
 			ld := cfg.Rand.Intn(len(tab.buckets))
-			tab.stuff([]*node{nodeAtDistance(tab.self.ID(), ld, intIP(ld))})
+			tab.stuff([]*node{nodeAtDistance(tab.self().ID(), ld, intIP(ld))})
 		}
 		gotN := tab.ReadRandomNodes(buf)
 		if gotN != tab.len() {
@@ -510,6 +510,10 @@ type preminedTestnet struct {
 	dists     [hashBits + 1][]encPubkey
 }
 
+func (tn *preminedTestnet) self() *enode.Node {
+	return nullNode
+}
+
 func (tn *preminedTestnet) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) {
 	// current log distance is encoded in port number
 	// fmt.Println("findnode query at dist", toaddr.Port)

p2p/discover/table_util_test.go

@@ -28,12 +28,17 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enr"
 )
 
-func newTestTable(t transport) (*Table, *enode.DB) {
+var nullNode *enode.Node
+
+func init() {
 	var r enr.Record
 	r.Set(enr.IP{0, 0, 0, 0})
-	n := enode.SignNull(&r, enode.ID{})
+	nullNode = enode.SignNull(&r, enode.ID{})
+}
+
+func newTestTable(t transport) (*Table, *enode.DB) {
 	db, _ := enode.OpenDB("")
-	tab, _ := newTable(t, n, db, nil)
+	tab, _ := newTable(t, db, nil)
 	return tab, db
 }
 
@@ -70,10 +75,10 @@ func intIP(i int) net.IP {
 
 // fillBucket inserts nodes into the given bucket until it is full.
 func fillBucket(tab *Table, n *node) (last *node) {
-	ld := enode.LogDist(tab.self.ID(), n.ID())
+	ld := enode.LogDist(tab.self().ID(), n.ID())
 	b := tab.bucket(n.ID())
 	for len(b.entries) < bucketSize {
-		b.entries = append(b.entries, nodeAtDistance(tab.self.ID(), ld, intIP(ld)))
+		b.entries = append(b.entries, nodeAtDistance(tab.self().ID(), ld, intIP(ld)))
 	}
 	return b.entries[bucketSize-1]
 }
@@ -81,15 +86,25 @@ func fillBucket(tab *Table, n *node) (last *node) {
 type pingRecorder struct {
 	mu           sync.Mutex
 	dead, pinged map[enode.ID]bool
+	n            *enode.Node
 }
 
 func newPingRecorder() *pingRecorder {
+	var r enr.Record
+	r.Set(enr.IP{0, 0, 0, 0})
+	n := enode.SignNull(&r, enode.ID{})
+
 	return &pingRecorder{
 		dead:   make(map[enode.ID]bool),
 		pinged: make(map[enode.ID]bool),
+		n:      n,
 	}
 }
 
+func (t *pingRecorder) self() *enode.Node {
+	return nullNode
+}
+
 func (t *pingRecorder) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) {
 	return nil, nil
 }

p2p/discover/udp.go

@@ -23,12 +23,12 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/enode"
-	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -118,9 +118,11 @@ type (
 )
 
 func makeEndpoint(addr *net.UDPAddr, tcpPort uint16) rpcEndpoint {
-	ip := addr.IP.To4()
-	if ip == nil {
-		ip = addr.IP.To16()
+	ip := net.IP{}
+	if ip4 := addr.IP.To4(); ip4 != nil {
+		ip = ip4
+	} else if ip6 := addr.IP.To16(); ip6 != nil {
+		ip = ip6
 	}
 	return rpcEndpoint{IP: ip, UDP: uint16(addr.Port), TCP: tcpPort}
 }
@@ -165,20 +167,19 @@ type conn interface {
 	LocalAddr() net.Addr
 }
 
-// udp implements the RPC protocol.
+// udp implements the discovery v4 UDP wire protocol.
 type udp struct {
 	conn        conn
 	netrestrict *netutil.Netlist
 	priv        *ecdsa.PrivateKey
-	ourEndpoint rpcEndpoint
+	localNode   *enode.LocalNode
+	db          *enode.DB
+	tab         *Table
+	wg          sync.WaitGroup
 
 	addpending chan *pending
 	gotreply   chan reply
-
 	closing    chan struct{}
-	nat     nat.Interface
-
-	*Table
 }
 
 // pending represents a pending reply.
@@ -230,60 +231,57 @@ type Config struct {
 	PrivateKey *ecdsa.PrivateKey
 
 	// These settings are optional:
-	AnnounceAddr *net.UDPAddr      // local address announced in the DHT
-	NodeDBPath   string            // if set, the node database is stored at this filesystem location
 	NetRestrict *netutil.Netlist  // network whitelist
 	Bootnodes   []*enode.Node     // list of bootstrap nodes
 	Unhandled   chan<- ReadPacket // unhandled packets are sent on this channel
 }
 
 // ListenUDP returns a new table that listens for UDP packets on laddr.
-func ListenUDP(c conn, cfg Config) (*Table, error) {
-	tab, _, err := newUDP(c, cfg)
+func ListenUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, error) {
+	tab, _, err := newUDP(c, ln, cfg)
 	if err != nil {
 		return nil, err
 	}
-	log.Info("UDP listener up", "self", tab.self)
 	return tab, nil
 }
 
-func newUDP(c conn, cfg Config) (*Table, *udp, error) {
-	realaddr := c.LocalAddr().(*net.UDPAddr)
-	if cfg.AnnounceAddr != nil {
-		realaddr = cfg.AnnounceAddr
-	}
-	self := enode.NewV4(&cfg.PrivateKey.PublicKey, realaddr.IP, realaddr.Port, realaddr.Port)
-	db, err := enode.OpenDB(cfg.NodeDBPath)
-	if err != nil {
-		return nil, nil, err
-	}
-
+func newUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, *udp, error) {
 	udp := &udp{
 		conn:        c,
 		priv:        cfg.PrivateKey,
 		netrestrict: cfg.NetRestrict,
+		localNode:   ln,
+		db:          ln.Database(),
 		closing:     make(chan struct{}),
 		gotreply:    make(chan reply),
 		addpending:  make(chan *pending),
 	}
-	// TODO: separate TCP port
-	udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port))
-	tab, err := newTable(udp, self, db, cfg.Bootnodes)
+	tab, err := newTable(udp, ln.Database(), cfg.Bootnodes)
 	if err != nil {
 		return nil, nil, err
 	}
-	udp.Table = tab
+	udp.tab = tab
 
+	udp.wg.Add(2)
 	go udp.loop()
 	go udp.readLoop(cfg.Unhandled)
-	return udp.Table, udp, nil
+	return udp.tab, udp, nil
+}
+
+func (t *udp) self() *enode.Node {
+	return t.localNode.Node()
 }
 
 func (t *udp) close() {
 	close(t.closing)
 	t.conn.Close()
-	t.db.Close()
-	// TODO: wait for the loops to end.
+	t.wg.Wait()
+}
+
+func (t *udp) ourEndpoint() rpcEndpoint {
+	n := t.self()
+	a := &net.UDPAddr{IP: n.IP(), Port: n.UDP()}
+	return makeEndpoint(a, uint16(n.TCP()))
 }
 
 // ping sends a ping message to the given node and waits for a reply.
@@ -296,7 +294,7 @@ func (t *udp) ping(toid enode.ID, toaddr *net.UDPAddr) error {
 func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-chan error {
 	req := &ping{
 		Version:    4,
-		From:       t.ourEndpoint,
+		From:       t.ourEndpoint(),
 		To:         makeEndpoint(toaddr, 0), // TODO: maybe use known TCP port from DB
 		Expiration: uint64(time.Now().Add(expiration).Unix()),
 	}
@@ -313,6 +311,7 @@ func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-ch
 		}
 		return ok
 	})
+	t.localNode.UDPContact(toaddr)
 	t.write(toaddr, req.name(), packet)
 	return errc
 }
@@ -381,6 +380,8 @@ func (t *udp) handleReply(from enode.ID, ptype byte, req packet) bool {
 // loop runs in its own goroutine. it keeps track of
 // the refresh timer and the pending reply queue.
 func (t *udp) loop() {
+	defer t.wg.Done()
+
 	var (
 		plist        = list.New()
 		timeout      = time.NewTimer(0)
@@ -542,10 +543,11 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (packet,
 
 // readLoop runs in its own goroutine. it handles incoming UDP packets.
 func (t *udp) readLoop(unhandled chan<- ReadPacket) {
-	defer t.conn.Close()
+	defer t.wg.Done()
 	if unhandled != nil {
 		defer close(unhandled)
 	}
+
 	// Discovery packets are defined to be no larger than 1280 bytes.
 	// Packets larger than this size will be cut at the end and treated
 	// as invalid because their hash won't match.
@@ -629,10 +631,11 @@ func (req *ping) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte
 	n := wrapNode(enode.NewV4(key, from.IP, int(req.From.TCP), from.Port))
 	t.handleReply(n.ID(), pingPacket, req)
 	if time.Since(t.db.LastPongReceived(n.ID())) > bondExpiration {
-		t.sendPing(n.ID(), from, func() { t.addThroughPing(n) })
+		t.sendPing(n.ID(), from, func() { t.tab.addThroughPing(n) })
 	} else {
-		t.addThroughPing(n)
+		t.tab.addThroughPing(n)
 	}
+	t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)})
 	t.db.UpdateLastPingReceived(n.ID(), time.Now())
 	return nil
 }
@@ -647,6 +650,7 @@ func (req *pong) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte
 	if !t.handleReply(fromID, pongPacket, req) {
 		return errUnsolicitedReply
 	}
+	t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)})
 	t.db.UpdateLastPongReceived(fromID, time.Now())
 	return nil
 }
@@ -668,9 +672,9 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []
 		return errUnknownNode
 	}
 	target := enode.ID(crypto.Keccak256Hash(req.Target[:]))
-	t.mutex.Lock()
-	closest := t.closest(target, bucketSize).entries
-	t.mutex.Unlock()
+	t.tab.mutex.Lock()
+	closest := t.tab.closest(target, bucketSize).entries
+	t.tab.mutex.Unlock()
 
 	p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())}
 	var sent bool