Merge branch 'master' of https://github.com/ethereum/go-ethereum into whisper/drop_light_clients

README.md

@@ -34,7 +34,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 
 | Command    | Description |
 |:----------:|-------------|
-| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
+| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
 | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
 | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
@@ -69,7 +69,7 @@ This command will:
  * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
    as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
-   This too is optional and if you leave it out you can always attach to an already running Geth instance
+   This tool is optional and if you leave it out you can always attach to an already running Geth instance
    with `geth attach`.
 
 ### Full node on the Ethereum test network

accounts/abi/bind/backends/simulated.go

@@ -65,9 +65,9 @@ type SimulatedBackend struct {
 
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
-func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
+func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
 	database := ethdb.NewMemDatabase()
-	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
+	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
 	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
 

accounts/abi/bind/bind_test.go

@@ -229,7 +229,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy an interaction tester contract and call a transaction on it
 			_, _, interactor, err := DeployInteractor(auth, sim, "Deploy string")
@@ -270,7 +270,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a tuple tester contract and execute a structured call on it
 			_, _, getter, err := DeployGetter(auth, sim)
@@ -302,7 +302,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a tuple tester contract and execute a structured call on it
 			_, _, tupler, err := DeployTupler(auth, sim)
@@ -344,7 +344,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a slice tester contract and execute a n array call on it
 			_, _, slicer, err := DeploySlicer(auth, sim)
@@ -378,7 +378,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a default method invoker contract and execute its default method
 			_, _, defaulter, err := DeployDefaulter(auth, sim)
@@ -411,7 +411,7 @@ var bindTests = []struct {
 		`[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
 		`
 			// Create a simulator and wrap a non-deployed contract
-			sim := backends.NewSimulatedBackend(nil)
+			sim := backends.NewSimulatedBackend(nil, uint64(10000000000))
 
 			nonexistent, err := NewNonExistent(common.Address{}, sim)
 			if err != nil {
@@ -447,7 +447,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a funky gas pattern contract
 			_, _, limiter, err := DeployFunkyGasPattern(auth, sim)
@@ -482,7 +482,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a sender tester contract and execute a structured call on it
 			_, _, callfrom, err := DeployCallFrom(auth, sim)
@@ -542,7 +542,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy a underscorer tester contract and execute a structured call on it
 			_, _, underscorer, err := DeployUnderscorer(auth, sim)
@@ -612,7 +612,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			// Deploy an eventer contract
 			_, _, eventer, err := DeployEventer(auth, sim)
@@ -761,7 +761,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 
 			//deploy the test contract
 			_, _, testContract, err := DeployDeeplyNestedArray(auth, sim)
@@ -820,7 +820,7 @@ func TestBindings(t *testing.T) {
 		t.Skip("go sdk not found for testing")
 	}
 	// Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845)
-	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}")
+	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil,uint64(10000000000)))\n}")
 	linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil)
 	if err != nil {
 		t.Fatalf("failed check for goimports symlink bug: %v", err)

accounts/abi/bind/util_test.go

@@ -53,9 +53,11 @@ var waitDeployedTests = map[string]struct {
 
 func TestWaitDeployed(t *testing.T) {
 	for name, test := range waitDeployedTests {
-		backend := backends.NewSimulatedBackend(core.GenesisAlloc{
+		backend := backends.NewSimulatedBackend(
+			core.GenesisAlloc{
 				crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
-		})
+			}, 10000000,
+		)
 
 		// Create the transaction.
 		tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code))

cmd/geth/main.go

@@ -98,8 +98,9 @@ var (
 		utils.MaxPendingPeersFlag,
 		utils.EtherbaseFlag,
 		utils.GasPriceFlag,
-		utils.MinerThreadsFlag,
 		utils.MiningEnabledFlag,
+		utils.MinerThreadsFlag,
+		utils.MinerNotifyFlag,
 		utils.TargetGasLimitFlag,
 		utils.NATFlag,
 		utils.NoDiscoverFlag,

cmd/geth/usage.go

@@ -185,6 +185,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.MiningEnabledFlag,
 			utils.MinerThreadsFlag,
+			utils.MinerNotifyFlag,
 			utils.EtherbaseFlag,
 			utils.TargetGasLimitFlag,
 			utils.GasPriceFlag,

cmd/swarm/config.go

@@ -68,6 +68,7 @@ const (
 	SWARM_ENV_SWAP_API             = "SWARM_SWAP_API"
 	SWARM_ENV_SYNC_DISABLE         = "SWARM_SYNC_DISABLE"
 	SWARM_ENV_SYNC_UPDATE_DELAY    = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SWARM_ENV_LIGHT_NODE_ENABLE    = "SWARM_LIGHT_NODE_ENABLE"
 	SWARM_ENV_DELIVERY_SKIP_CHECK  = "SWARM_DELIVERY_SKIP_CHECK"
 	SWARM_ENV_ENS_API              = "SWARM_ENS_API"
 	SWARM_ENV_ENS_ADDR             = "SWARM_ENS_ADDR"
@@ -131,7 +132,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
 	log.Debug(printConfig(config))
 }
 
-//override the current config with whatever is in the config file, if a config file has been provided
+//configFileOverride overrides the current config with the config file, if a config file has been provided
 func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) {
 	var err error
 
@@ -141,7 +142,8 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config
 		if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" {
 			utils.Fatalf("Config file flag provided with invalid file path")
 		}
-		f, err := os.Open(filepath)
+		var f *os.File
+		f, err = os.Open(filepath)
 		if err != nil {
 			return nil, err
 		}
@@ -204,6 +206,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.SyncUpdateDelay = d
 	}
 
+	if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
+		currentConfig.LightNodeEnabled = true
+	}
+
 	if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) {
 		currentConfig.DeliverySkipCheck = true
 	}
@@ -301,6 +307,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		}
 	}
 
+	if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
+		if lightnode, err := strconv.ParseBool(lne); err != nil {
+			currentConfig.LightNodeEnabled = lightnode
+		}
+	}
+
 	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
 		currentConfig.SwapAPI = swapapi
 	}

cmd/swarm/config_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"testing"
@@ -559,3 +560,16 @@ func TestValidateConfig(t *testing.T) {
 		}
 	}
 }
+
+func assignTCPPort() (string, error) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return "", err
+	}
+	l.Close()
+	_, port, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		return "", err
+	}
+	return port, nil
+}

cmd/swarm/fs.go

@@ -92,7 +92,7 @@ func listMounts(cliContext *cli.Context) {
 	mf := []fuse.MountInfo{}
 	err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
 	if err != nil {
-		utils.Fatalf("encountered an error calling the RPC endpoint while unmounting: %v", err)
+		utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)
 	}
 	if len(mf) == 0 {
 		fmt.Print("Could not found any swarmfs mounts. Please make sure you've specified the correct RPC endpoint\n")

cmd/swarm/fs_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// +build linux darwin freebsd
+// +build linux freebsd
 
 package main
 
@@ -43,6 +43,11 @@ type testFile struct {
 }
 
 // TestCLISwarmFs is a high-level test of swarmfs
+//
+// This test fails on travis for macOS as this executable exits with code 1
+// and without any log messages in the log:
+// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse.
+// This is the reason for this file not being built on darwin architecture.
 func TestCLISwarmFs(t *testing.T) {
 	cluster := newTestCluster(t, 3)
 	defer cluster.Shutdown()

cmd/swarm/main.go

@@ -123,6 +123,11 @@ var (
 		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
 		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
 	}
+	SwarmLightNodeEnabled = cli.BoolFlag{
+		Name:   "lightnode",
+		Usage:  "Enable Swarm LightNode (default false)",
+		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
+	}
 	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
 		Name:   "delivery-skip-check",
 		Usage:  "Skip chunk delivery check (default false)",
@@ -317,23 +322,23 @@ Downloads a swarm bzz uri to the given dir. When no dir is provided, working dir
 			Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
 			Subcommands: []cli.Command{
 				{
-					Action:             add,
+					Action:             manifestAdd,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "add",
 					Usage:              "add a new path to the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <hash> [<content-type>]",
+					ArgsUsage:          "<MANIFEST> <path> <hash>",
 					Description:        "Adds a new path to the manifest",
 				},
 				{
-					Action:             update,
+					Action:             manifestUpdate,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "update",
 					Usage:              "update the hash for an already existing path in the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <newhash> [<newcontent-type>]",
+					ArgsUsage:          "<MANIFEST> <path> <newhash>",
 					Description:        "Update the hash for an already existing path in the manifest",
 				},
 				{
-					Action:             remove,
+					Action:             manifestRemove,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "remove",
 					Usage:              "removes a path from the manifest",
@@ -464,6 +469,7 @@ pv(1) tool to get a progress bar:
 		SwarmSwapAPIFlag,
 		SwarmSyncDisabledFlag,
 		SwarmSyncUpdateDelay,
+		SwarmLightNodeEnabled,
 		SwarmDeliverySkipCheckFlag,
 		SwarmListenAddrFlag,
 		SwarmPortFlag,

cmd/swarm/manifest.go

@@ -18,10 +18,8 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
-	"mime"
+	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -30,127 +28,118 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-const bzzManifestJSON = "application/bzz-manifest+json"
+// manifestAdd adds a new entry to the manifest at the given path.
-
+// New entry hash, the last argument, must be the hash of a manifest
-func add(ctx *cli.Context) {
+// with only one entry, which meta-data will be added to the original manifest.
+// On success, this function will print new (updated) manifest's hash.
+func manifestAdd(ctx *cli.Context) {
 	args := ctx.Args()
-	if len(args) < 3 {
+	if len(args) != 3 {
-		utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH> [<content-type>]")
+		utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
 	}
 
 	var (
 		mhash = args[0]
 		path  = args[1]
 		hash  = args[2]
-
-		ctype        string
-		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        api.Manifest
 	)
 
-	if len(args) > 3 {
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		ctype = args[3]
+	client := swarm.NewClient(bzzapi)
-	} else {
+
-		ctype = mime.TypeByExtension(filepath.Ext(path))
+	m, _, err := client.DownloadManifest(hash)
+	if err != nil {
+		utils.Fatalf("Error downloading manifest to add: %v", err)
+	}
+	l := len(m.Entries)
+	if l == 0 {
+		utils.Fatalf("No entries in manifest %s", hash)
+	} else if l > 1 {
+		utils.Fatalf("Too many entries in manifest %s", hash)
 	}
 
-	newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype)
+	newManifest := addEntryToManifest(client, mhash, path, m.Entries[0])
 	fmt.Println(newManifest)
-
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
 }
 
-func update(ctx *cli.Context) {
+// manifestUpdate replaces an existing entry of the manifest at the given path.
-
+// New entry hash, the last argument, must be the hash of a manifest
+// with only one entry, which meta-data will be added to the original manifest.
+// On success, this function will print hash of the updated manifest.
+func manifestUpdate(ctx *cli.Context) {
 	args := ctx.Args()
-	if len(args) < 3 {
+	if len(args) != 3 {
-		utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH>")
+		utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
 	}
 
 	var (
 		mhash = args[0]
 		path  = args[1]
 		hash  = args[2]
-
-		ctype        string
-		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        api.Manifest
 	)
-	if len(args) > 3 {
+
-		ctype = args[3]
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-	} else {
+	client := swarm.NewClient(bzzapi)
-		ctype = mime.TypeByExtension(filepath.Ext(path))
+
+	m, _, err := client.DownloadManifest(hash)
+	if err != nil {
+		utils.Fatalf("Error downloading manifest to update: %v", err)
+	}
+	l := len(m.Entries)
+	if l == 0 {
+		utils.Fatalf("No entries in manifest %s", hash)
+	} else if l > 1 {
+		utils.Fatalf("Too many entries in manifest %s", hash)
 	}
 
-	newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype)
+	newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true)
+	if defaultEntryUpdated {
+		// Print informational message to stderr
+		// allowing the user to get the new manifest hash from stdout
+		// without the need to parse the complete output.
+		fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too")
+	}
 	fmt.Println(newManifest)
-
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
 }
 
-func remove(ctx *cli.Context) {
+// manifestRemove removes an existing entry of the manifest at the given path.
+// On success, this function will print hash of the manifest which does not
+// contain the path.
+func manifestRemove(ctx *cli.Context) {
 	args := ctx.Args()
-	if len(args) < 2 {
+	if len(args) != 2 {
-		utils.Fatalf("Need at least two arguments <MHASH> <path>")
+		utils.Fatalf("Need exactly two arguments <MHASH> <path>")
 	}
 
 	var (
 		mhash = args[0]
 		path  = args[1]
-
-		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        api.Manifest
 	)
 
-	newManifest := removeEntryFromManifest(ctx, mhash, path)
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+
+	newManifest := removeEntryFromManifest(client, mhash, path)
 	fmt.Println(newManifest)
-
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
 }
 
-func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
+func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string {
-
+	var longestPathEntry = api.ManifestEntry{}
-	var (
-		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = swarm.NewClient(bzzapi)
-		longestPathEntry = api.ManifestEntry{}
-	)
 
 	mroot, isEncrypted, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
-	//TODO: check if the "hash" to add is valid and present in swarm
-	_, _, err = client.DownloadManifest(hash)
-	if err != nil {
-		utils.Fatalf("Hash to add is not present: %v", err)
-	}
-
 	// See if we path is in this Manifest or do we have to dig deeper
-	for _, entry := range mroot.Entries {
+	for _, e := range mroot.Entries {
-		if path == entry.Path {
+		if path == e.Path {
 			utils.Fatalf("Path %s already present, not adding anything", path)
 		} else {
-			if entry.ContentType == bzzManifestJSON {
+			if e.ContentType == api.ManifestType {
-				prfxlen := strings.HasPrefix(path, entry.Path)
+				prfxlen := strings.HasPrefix(path, e.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
-					longestPathEntry = entry
+					longestPathEntry = e
 				}
 			}
 		}
@@ -159,25 +148,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 	if longestPathEntry.Path != "" {
 		// Load the child Manifest add the entry there
 		newPath := path[len(longestPathEntry.Path):]
-		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
+		newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry)
 
 		// Replace the hash for parent Manifests
 		newMRoot := &api.Manifest{}
-		for _, entry := range mroot.Entries {
+		for _, e := range mroot.Entries {
-			if longestPathEntry.Path == entry.Path {
+			if longestPathEntry.Path == e.Path {
-				entry.Hash = newHash
+				e.Hash = newHash
 			}
-			newMRoot.Entries = append(newMRoot.Entries, entry)
+			newMRoot.Entries = append(newMRoot.Entries, e)
 		}
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := api.ManifestEntry{
+		entry.Path = path
-			Hash:        hash,
+		mroot.Entries = append(mroot.Entries, entry)
-			Path:        path,
-			ContentType: ctype,
-		}
-		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
 	newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
@@ -185,14 +170,16 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
 	return newManifestHash
-
 }
 
-func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
+// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash
-
+// finding the path recursively through all nested manifests. Argument isRoot is used for default
+// entry update detection. If the updated entry has the same hash as the default entry, then the
+// default entry in root manifest will be updated too.
+// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and
+// a a bool that is true if default entry is updated.
+func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) {
 	var (
-		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = swarm.NewClient(bzzapi)
 		newEntry         = api.ManifestEntry{}
 		longestPathEntry = api.ManifestEntry{}
 	)
@@ -202,17 +189,18 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
-	//TODO: check if the "hash" with which to update is valid and present in swarm
-
 	// See if we path is in this Manifest or do we have to dig deeper
-	for _, entry := range mroot.Entries {
+	for _, e := range mroot.Entries {
-		if path == entry.Path {
+		if path == e.Path {
-			newEntry = entry
+			newEntry = e
+			// keep the reference of the hash of the entry that should be replaced
+			// for default entry detection
+			oldHash = e.Hash
 		} else {
-			if entry.ContentType == bzzManifestJSON {
+			if e.ContentType == api.ManifestType {
-				prfxlen := strings.HasPrefix(path, entry.Path)
+				prfxlen := strings.HasPrefix(path, e.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
-					longestPathEntry = entry
+					longestPathEntry = e
 				}
 			}
 		}
@@ -225,50 +213,50 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 	if longestPathEntry.Path != "" {
 		// Load the child Manifest add the entry there
 		newPath := path[len(longestPathEntry.Path):]
-		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
+		var newHash string
+		newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false)
 
 		// Replace the hash for parent Manifests
 		newMRoot := &api.Manifest{}
-		for _, entry := range mroot.Entries {
+		for _, e := range mroot.Entries {
-			if longestPathEntry.Path == entry.Path {
+			if longestPathEntry.Path == e.Path {
-				entry.Hash = newHash
+				e.Hash = newHash
 			}
-			newMRoot.Entries = append(newMRoot.Entries, entry)
+			newMRoot.Entries = append(newMRoot.Entries, e)
 
 		}
 		mroot = newMRoot
 	}
 
-	if newEntry.Path != "" {
+	// update the manifest if the new entry is found and
+	// check if default entry should be updated
+	if newEntry.Path != "" || isRoot {
 		// Replace the hash for leaf Manifest
 		newMRoot := &api.Manifest{}
-		for _, entry := range mroot.Entries {
+		for _, e := range mroot.Entries {
-			if newEntry.Path == entry.Path {
+			if newEntry.Path == e.Path {
-				myEntry := api.ManifestEntry{
+				entry.Path = e.Path
-					Hash:        hash,
-					Path:        entry.Path,
-					ContentType: ctype,
-				}
-				newMRoot.Entries = append(newMRoot.Entries, myEntry)
-			} else {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
+			} else if isRoot && e.Path == "" && e.Hash == oldHash {
+				entry.Path = e.Path
+				newMRoot.Entries = append(newMRoot.Entries, entry)
+				defaultEntryUpdated = true
+			} else {
+				newMRoot.Entries = append(newMRoot.Entries, e)
 			}
 		}
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
+	newManifestHash, err = client.UploadManifest(mroot, isEncrypted)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
-	return newManifestHash
+	return newManifestHash, oldHash, defaultEntryUpdated
 }
 
-func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
+func removeEntryFromManifest(client *swarm.Client, mhash, path string) string {
-
 	var (
-		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = swarm.NewClient(bzzapi)
 		entryToRemove    = api.ManifestEntry{}
 		longestPathEntry = api.ManifestEntry{}
 	)
@@ -283,7 +271,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		if path == entry.Path {
 			entryToRemove = entry
 		} else {
-			if entry.ContentType == bzzManifestJSON {
+			if entry.ContentType == api.ManifestType {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
 					longestPathEntry = entry
@@ -299,7 +287,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 	if longestPathEntry.Path != "" {
 		// Load the child Manifest remove the entry there
 		newPath := path[len(longestPathEntry.Path):]
-		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
+		newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
 		newMRoot := &api.Manifest{}

cmd/swarm/manifest_test.go

@@ -0,0 +1,579 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+)
+
+// TestManifestChange tests manifest add, update and remove
+// cli commands without encryption.
+func TestManifestChange(t *testing.T) {
+	testManifestChange(t, false)
+}
+
+// TestManifestChange tests manifest add, update and remove
+// cli commands with encryption enabled.
+func TestManifestChangeEncrypted(t *testing.T) {
+	testManifestChange(t, true)
+}
+
+// testManifestChange performs cli commands:
+// - manifest add
+// - manifest update
+// - manifest remove
+// on a manifest, testing the functionality of this
+// comands on paths that are in root manifest or a nested one.
+// Argument encrypt controls whether to use encryption or not.
+func testManifestChange(t *testing.T, encrypt bool) {
+	t.Parallel()
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	origDir := filepath.Join(tmp, "orig")
+	if err := os.Mkdir(origDir, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	indexDataFilename := filepath.Join(origDir, "index.html")
+	err = ioutil.WriteFile(indexDataFilename, []byte("<h1>Test</h1>"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Files paths robots.txt and robots.html share the same prefix "robots."
+	// which will result a manifest with a nested manifest under path "robots.".
+	// This will allow testing manifest changes on both root and nested manifest.
+	err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("<strong>No Robots Allowed</strong>"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	args := []string{
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"--recursive",
+		"--defaultpath",
+		indexDataFilename,
+		"up",
+		origDir,
+	}
+	if encrypt {
+		args = append(args, "--encrypt")
+	}
+
+	origManifestHash := runSwarmExpectHash(t, args...)
+
+	checkHashLength(t, origManifestHash, encrypt)
+
+	client := swarm.NewClient(cluster.Nodes[0].URL)
+
+	// upload a new file and use its manifest to add it the original manifest.
+	t.Run("add", func(t *testing.T) {
+		humansData := []byte("Ann\nBob")
+		humansDataFilename := filepath.Join(tmp, "humans.txt")
+		err = ioutil.WriteFile(humansDataFilename, humansData, 0666)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		humansManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"up",
+			humansDataFilename,
+		)
+
+		newManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"manifest",
+			"add",
+			origManifestHash,
+			"humans.txt",
+			humansManifestHash,
+		)
+
+		checkHashLength(t, newManifestHash, encrypt)
+
+		newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+		var found bool
+		for _, e := range newManifest.Entries {
+			if e.Path == "humans.txt" {
+				found = true
+				if e.Size != int64(len(humansData)) {
+					t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size)
+				}
+				if e.ModTime.IsZero() {
+					t.Errorf("got zero mod time for humans.txt")
+				}
+				ct := "text/plain; charset=utf-8"
+				if e.ContentType != ct {
+					t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+				}
+				break
+			}
+		}
+		if !found {
+			t.Fatal("no humans.txt in new manifest")
+		}
+
+		checkFile(t, client, newManifestHash, "humans.txt", humansData)
+	})
+
+	// upload a new file and use its manifest to add it the original manifest,
+	// but ensure that the file will be in the nested manifest of the original one.
+	t.Run("add nested", func(t *testing.T) {
+		robotsData := []byte(`{"disallow": "/"}`)
+		robotsDataFilename := filepath.Join(tmp, "robots.json")
+		err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		robotsManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"up",
+			robotsDataFilename,
+		)
+
+		newManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"manifest",
+			"add",
+			origManifestHash,
+			"robots.json",
+			robotsManifestHash,
+		)
+
+		checkHashLength(t, newManifestHash, encrypt)
+
+		newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+		var found bool
+	loop:
+		for _, e := range newManifest.Entries {
+			if e.Path == "robots." {
+				nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
+				for _, e := range nestedManifest.Entries {
+					if e.Path == "json" {
+						found = true
+						if e.Size != int64(len(robotsData)) {
+							t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size)
+						}
+						if e.ModTime.IsZero() {
+							t.Errorf("got zero mod time for robots.json")
+						}
+						ct := "application/json"
+						if e.ContentType != ct {
+							t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+						}
+						break loop
+					}
+				}
+			}
+		}
+		if !found {
+			t.Fatal("no robots.json in new manifest")
+		}
+
+		checkFile(t, client, newManifestHash, "robots.json", robotsData)
+	})
+
+	// upload a new file and use its manifest to change the file it the original manifest.
+	t.Run("update", func(t *testing.T) {
+		indexData := []byte("<h1>Ethereum Swarm</h1>")
+		indexDataFilename := filepath.Join(tmp, "index.html")
+		err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		indexManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"up",
+			indexDataFilename,
+		)
+
+		newManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"manifest",
+			"update",
+			origManifestHash,
+			"index.html",
+			indexManifestHash,
+		)
+
+		checkHashLength(t, newManifestHash, encrypt)
+
+		newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+		var found bool
+		for _, e := range newManifest.Entries {
+			if e.Path == "index.html" {
+				found = true
+				if e.Size != int64(len(indexData)) {
+					t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size)
+				}
+				if e.ModTime.IsZero() {
+					t.Errorf("got zero mod time for index.html")
+				}
+				ct := "text/html; charset=utf-8"
+				if e.ContentType != ct {
+					t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+				}
+				break
+			}
+		}
+		if !found {
+			t.Fatal("no index.html in new manifest")
+		}
+
+		checkFile(t, client, newManifestHash, "index.html", indexData)
+
+		// check default entry change
+		checkFile(t, client, newManifestHash, "", indexData)
+	})
+
+	// upload a new file and use its manifest to change the file it the original manifest,
+	// but ensure that the file is in the nested manifest of the original one.
+	t.Run("update nested", func(t *testing.T) {
+		robotsData := []byte(`<string>Only humans allowed!!!</strong>`)
+		robotsDataFilename := filepath.Join(tmp, "robots.html")
+		err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		humansManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"up",
+			robotsDataFilename,
+		)
+
+		newManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"manifest",
+			"update",
+			origManifestHash,
+			"robots.html",
+			humansManifestHash,
+		)
+
+		checkHashLength(t, newManifestHash, encrypt)
+
+		newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+		var found bool
+	loop:
+		for _, e := range newManifest.Entries {
+			if e.Path == "robots." {
+				nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
+				for _, e := range nestedManifest.Entries {
+					if e.Path == "html" {
+						found = true
+						if e.Size != int64(len(robotsData)) {
+							t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size)
+						}
+						if e.ModTime.IsZero() {
+							t.Errorf("got zero mod time for robots.html")
+						}
+						ct := "text/html; charset=utf-8"
+						if e.ContentType != ct {
+							t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+						}
+						break loop
+					}
+				}
+			}
+		}
+		if !found {
+			t.Fatal("no robots.html in new manifest")
+		}
+
+		checkFile(t, client, newManifestHash, "robots.html", robotsData)
+	})
+
+	// remove a file from the manifest.
+	t.Run("remove", func(t *testing.T) {
+		newManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"manifest",
+			"remove",
+			origManifestHash,
+			"mutants.txt",
+		)
+
+		checkHashLength(t, newManifestHash, encrypt)
+
+		newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+		var found bool
+		for _, e := range newManifest.Entries {
+			if e.Path == "mutants.txt" {
+				found = true
+				break
+			}
+		}
+		if found {
+			t.Fatal("mutants.txt is not removed")
+		}
+	})
+
+	// remove a file from the manifest, but ensure that the file is in
+	// the nested manifest of the original one.
+	t.Run("remove nested", func(t *testing.T) {
+		newManifestHash := runSwarmExpectHash(t,
+			"--bzzapi",
+			cluster.Nodes[0].URL,
+			"manifest",
+			"remove",
+			origManifestHash,
+			"robots.html",
+		)
+
+		checkHashLength(t, newManifestHash, encrypt)
+
+		newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+		var found bool
+	loop:
+		for _, e := range newManifest.Entries {
+			if e.Path == "robots." {
+				nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
+				for _, e := range nestedManifest.Entries {
+					if e.Path == "html" {
+						found = true
+						break loop
+					}
+				}
+			}
+		}
+		if found {
+			t.Fatal("robots.html in not removed")
+		}
+	})
+}
+
+// TestNestedDefaultEntryUpdate tests if the default entry is updated
+// if the file in nested manifest used for it is also updated.
+func TestNestedDefaultEntryUpdate(t *testing.T) {
+	testNestedDefaultEntryUpdate(t, false)
+}
+
+// TestNestedDefaultEntryUpdateEncrypted tests if the default entry
+// of encrypted upload is updated if the file in nested manifest
+// used for it is also updated.
+func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
+	testNestedDefaultEntryUpdate(t, true)
+}
+
+func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
+	t.Parallel()
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	origDir := filepath.Join(tmp, "orig")
+	if err := os.Mkdir(origDir, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	indexData := []byte("<h1>Test</h1>")
+	indexDataFilename := filepath.Join(origDir, "index.html")
+	err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Add another file with common prefix as the default entry to test updates of
+	// default entry with nested manifests.
+	err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	args := []string{
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"--recursive",
+		"--defaultpath",
+		indexDataFilename,
+		"up",
+		origDir,
+	}
+	if encrypt {
+		args = append(args, "--encrypt")
+	}
+
+	origManifestHash := runSwarmExpectHash(t, args...)
+
+	checkHashLength(t, origManifestHash, encrypt)
+
+	client := swarm.NewClient(cluster.Nodes[0].URL)
+
+	newIndexData := []byte("<h1>Ethereum Swarm</h1>")
+	newIndexDataFilename := filepath.Join(tmp, "index.html")
+	err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newIndexManifestHash := runSwarmExpectHash(t,
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"up",
+		newIndexDataFilename,
+	)
+
+	newManifestHash := runSwarmExpectHash(t,
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"manifest",
+		"update",
+		origManifestHash,
+		"index.html",
+		newIndexManifestHash,
+	)
+
+	checkHashLength(t, newManifestHash, encrypt)
+
+	newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+	var found bool
+	for _, e := range newManifest.Entries {
+		if e.Path == "index." {
+			found = true
+			newManifest = downloadManifest(t, client, e.Hash, encrypt)
+			break
+		}
+	}
+	if !found {
+		t.Fatal("no index. path in new manifest")
+	}
+
+	found = false
+	for _, e := range newManifest.Entries {
+		if e.Path == "html" {
+			found = true
+			if e.Size != int64(len(newIndexData)) {
+				t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size)
+			}
+			if e.ModTime.IsZero() {
+				t.Errorf("got zero mod time for index.html")
+			}
+			ct := "text/html; charset=utf-8"
+			if e.ContentType != ct {
+				t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+			}
+			break
+		}
+	}
+	if !found {
+		t.Fatal("no html in new manifest")
+	}
+
+	checkFile(t, client, newManifestHash, "index.html", newIndexData)
+
+	// check default entry change
+	checkFile(t, client, newManifestHash, "", newIndexData)
+}
+
+func runSwarmExpectHash(t *testing.T, args ...string) (hash string) {
+	t.Helper()
+	hashRegexp := `[a-f\d]{64,128}`
+	up := runSwarm(t, args...)
+	_, matches := up.ExpectRegexp(hashRegexp)
+	up.ExpectExit()
+
+	if len(matches) < 1 {
+		t.Fatal("no matches found")
+	}
+	return matches[0]
+}
+
+func checkHashLength(t *testing.T, hash string, encrypted bool) {
+	t.Helper()
+	l := len(hash)
+	if encrypted && l != 128 {
+		t.Errorf("expected hash length 128, got %v", l)
+	}
+	if !encrypted && l != 64 {
+		t.Errorf("expected hash length 64, got %v", l)
+	}
+}
+
+func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) {
+	t.Helper()
+	m, isEncrypted, err := client.DownloadManifest(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if encrypted != isEncrypted {
+		t.Error("new manifest encryption flag is not correct")
+	}
+	return m
+}
+
+func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) {
+	t.Helper()
+	f, err := client.Download(hash, path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(got, expected) {
+		t.Errorf("expected file content %q, got %q", expected, got)
+	}
+}

cmd/swarm/run_test.go

@@ -17,12 +17,15 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
 	"runtime"
+	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -218,14 +221,12 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	}
 
 	// assign ports
-	httpPort, err := assignTCPPort()
+	ports, err := getAvailableTCPPorts(2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	p2pPort, err := assignTCPPort()
 	if err != nil {
 		t.Fatal(err)
 	}
+	p2pPort := ports[0]
+	httpPort := ports[1]
 
 	// start the node
 	node.Cmd = runSwarm(t,
@@ -246,6 +247,17 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 		}
 	}()
 
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// ensure that all ports have active listeners
+	// so that the next node will not get the same
+	// when calling getAvailableTCPPorts
+	err = waitTCPPorts(ctx, ports...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// wait for the node to start
 	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
 		node.Client, err = rpc.Dial(conf.IPCEndpoint())
@@ -280,14 +292,12 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	node := &testNode{Dir: dir}
 
 	// assign ports
-	httpPort, err := assignTCPPort()
+	ports, err := getAvailableTCPPorts(2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	p2pPort, err := assignTCPPort()
 	if err != nil {
 		t.Fatal(err)
 	}
+	p2pPort := ports[0]
+	httpPort := ports[1]
 
 	// start the node
 	node.Cmd = runSwarm(t,
@@ -308,6 +318,17 @@ func newTestNode(t *testing.T, dir string) *testNode {
 		}
 	}()
 
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// ensure that all ports have active listeners
+	// so that the next node will not get the same
+	// when calling getAvailableTCPPorts
+	err = waitTCPPorts(ctx, ports...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// wait for the node to start
 	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
 		node.Client, err = rpc.Dial(conf.IPCEndpoint())
@@ -343,15 +364,92 @@ func (n *testNode) Shutdown() {
 	}
 }
 
-func assignTCPPort() (string, error) {
+// getAvailableTCPPorts returns a set of ports that
+// nothing is listening on at the time.
+//
+// Function assignTCPPort cannot be called in sequence
+// and guardantee that the same port will be returned in
+// different calls as the listener is closed within the function,
+// not after all listeners are started and selected unique
+// available ports.
+func getAvailableTCPPorts(count int) (ports []string, err error) {
+	for i := 0; i < count; i++ {
 		l, err := net.Listen("tcp", "127.0.0.1:0")
 		if err != nil {
-		return "", err
+			return nil, err
 		}
-	l.Close()
+		// defer close in the loop to be sure the same port will not
+		// be selected in the next iteration
+		defer l.Close()
+
 		_, port, err := net.SplitHostPort(l.Addr().String())
 		if err != nil {
-		return "", err
+			return nil, err
+		}
+		ports = append(ports, port)
+	}
+	return ports, nil
+}
+
+// waitTCPPorts blocks until tcp connections can be
+// established on all provided ports. It runs all
+// ports dialers in parallel, and returns the first
+// encountered error.
+// See waitTCPPort also.
+func waitTCPPorts(ctx context.Context, ports ...string) error {
+	var err error
+	// mu locks err variable that is assigned in
+	// other goroutines
+	var mu sync.Mutex
+
+	// cancel is canceling all goroutines
+	// when the firs error is returned
+	// to prevent unnecessary waiting
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	for _, port := range ports {
+		wg.Add(1)
+		go func(port string) {
+			defer wg.Done()
+
+			e := waitTCPPort(ctx, port)
+
+			mu.Lock()
+			defer mu.Unlock()
+			if e != nil && err == nil {
+				err = e
+				cancel()
+			}
+		}(port)
+	}
+	wg.Wait()
+
+	return err
+}
+
+// waitTCPPort blocks until tcp connection can be established
+// ona provided port. It has a 3 minute timeout as maximum,
+// to prevent long waiting, but it can be shortened with
+// a provided context instance. Dialer has a 10 second timeout
+// in every iteration, and connection refused error will be
+// retried in 100 milliseconds periods.
+func waitTCPPort(ctx context.Context, port string) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+
+	for {
+		c, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, "tcp", "127.0.0.1:"+port)
+		if err != nil {
+			if operr, ok := err.(*net.OpError); ok {
+				if syserr, ok := operr.Err.(*os.SyscallError); ok && syserr.Err == syscall.ECONNREFUSED {
+					time.Sleep(100 * time.Millisecond)
+					continue
+				}
+			}
+			return err
+		}
+		return c.Close()
 	}
-	return port, nil
 }

cmd/swarm/upload.go

@@ -98,6 +98,17 @@ func upload(ctx *cli.Context) {
 			if !recursive {
 				return "", errors.New("Argument is a directory and recursive upload is disabled")
 			}
+			if defaultPath != "" {
+				// construct absolute default path
+				absDefaultPath, _ := filepath.Abs(defaultPath)
+				absFile, _ := filepath.Abs(file)
+				// make sure absolute directory ends with only one "/"
+				// to trim it from absolute default path and get relative default path
+				absFile = strings.TrimRight(absFile, "/") + "/"
+				if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) {
+					defaultPath = strings.TrimPrefix(absDefaultPath, absFile)
+				}
+			}
 			return client.UploadDirectory(file, defaultPath, "", toEncrypt)
 		}
 	} else {

cmd/swarm/upload_test.go

@@ -273,3 +273,84 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 		}
 	}
 }
+
+// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// default paths and with encryption.
+func TestCLISwarmUpDefaultPath(t *testing.T) {
+	testCLISwarmUpDefaultPath(false, false, t)
+	testCLISwarmUpDefaultPath(false, true, t)
+	testCLISwarmUpDefaultPath(true, false, t)
+	testCLISwarmUpDefaultPath(true, true, t)
+}
+
+func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("<h1>Test</h1>"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defaultPath := "index.html"
+	if absDefaultPath {
+		defaultPath = filepath.Join(tmp, defaultPath)
+	}
+
+	args := []string{
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"--recursive",
+		"--defaultpath",
+		defaultPath,
+		"up",
+		tmp,
+	}
+	if toEncrypt {
+		args = append(args, "--encrypt")
+	}
+
+	up := runSwarm(t, args...)
+	hashRegexp := `[a-f\d]{64,128}`
+	_, matches := up.ExpectRegexp(hashRegexp)
+	up.ExpectExit()
+	hash := matches[0]
+
+	client := swarm.NewClient(cluster.Nodes[0].URL)
+
+	m, isEncrypted, err := client.DownloadManifest(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if toEncrypt != isEncrypted {
+		t.Error("downloaded manifest is not encrypted")
+	}
+
+	var found bool
+	var entriesCount int
+	for _, e := range m.Entries {
+		entriesCount++
+		if e.Path == "" {
+			found = true
+		}
+	}
+
+	if !found {
+		t.Error("manifest default entry was not found")
+	}
+
+	if entriesCount != 3 {
+		t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3)
+	}
+}

cmd/utils/flags.go

@@ -24,7 +24,6 @@ import (
 	"math/big"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -318,9 +317,13 @@ var (
 		Usage: "Enable mining",
 	}
 	MinerThreadsFlag = cli.IntFlag{
-		Name:  "minerthreads",
+		Name:  "miner.threads",
 		Usage: "Number of CPU threads to use for mining",
-		Value: runtime.NumCPU(),
+		Value: 0,
+	}
+	MinerNotifyFlag = cli.StringFlag{
+		Name:  "miner.notify",
+		Usage: "Comma separated HTTP URL list to notify of new work packages",
 	}
 	TargetGasLimitFlag = cli.Uint64Flag{
 		Name:  "targetgaslimit",
@@ -1100,6 +1103,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
 		cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
 	}
+	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
+		cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
+	}
 	if ctx.GlobalIsSet(DocRootFlag.Name) {
 		cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name)
 	}
@@ -1300,7 +1306,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 				DatasetDir:     stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
 				DatasetsInMem:  eth.DefaultConfig.Ethash.DatasetsInMem,
 				DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
-			})
+			}, nil)
 		}
 	}
 	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {

common/mclock/mclock.go

@@ -30,3 +30,34 @@ type AbsTime time.Duration
 func Now() AbsTime {
 	return AbsTime(monotime.Now())
 }
+
+// Add returns t + d.
+func (t AbsTime) Add(d time.Duration) AbsTime {
+	return t + AbsTime(d)
+}
+
+// Clock interface makes it possible to replace the monotonic system clock with
+// a simulated clock.
+type Clock interface {
+	Now() AbsTime
+	Sleep(time.Duration)
+	After(time.Duration) <-chan time.Time
+}
+
+// System implements Clock using the system clock.
+type System struct{}
+
+// Now implements Clock.
+func (System) Now() AbsTime {
+	return AbsTime(monotime.Now())
+}
+
+// Sleep implements Clock.
+func (System) Sleep(d time.Duration) {
+	time.Sleep(d)
+}
+
+// After implements Clock.
+func (System) After(d time.Duration) <-chan time.Time {
+	return time.After(d)
+}

common/mclock/simclock.go

@@ -0,0 +1,129 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mclock
+
+import (
+	"sync"
+	"time"
+)
+
+// Simulated implements a virtual Clock for reproducible time-sensitive tests. It
+// simulates a scheduler on a virtual timescale where actual processing takes zero time.
+//
+// The virtual clock doesn't advance on its own, call Run to advance it and execute timers.
+// Since there is no way to influence the Go scheduler, testing timeout behaviour involving
+// goroutines needs special care. A good way to test such timeouts is as follows: First
+// perform the action that is supposed to time out. Ensure that the timer you want to test
+// is created. Then run the clock until after the timeout. Finally observe the effect of
+// the timeout using a channel or semaphore.
+type Simulated struct {
+	now       AbsTime
+	scheduled []event
+	mu        sync.RWMutex
+	cond      *sync.Cond
+}
+
+type event struct {
+	do func()
+	at AbsTime
+}
+
+// Run moves the clock by the given duration, executing all timers before that duration.
+func (s *Simulated) Run(d time.Duration) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.init()
+
+	end := s.now + AbsTime(d)
+	for len(s.scheduled) > 0 {
+		ev := s.scheduled[0]
+		if ev.at > end {
+			break
+		}
+		s.now = ev.at
+		ev.do()
+		s.scheduled = s.scheduled[1:]
+	}
+	s.now = end
+}
+
+func (s *Simulated) ActiveTimers() int {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return len(s.scheduled)
+}
+
+func (s *Simulated) WaitForTimers(n int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.init()
+
+	for len(s.scheduled) < n {
+		s.cond.Wait()
+	}
+}
+
+// Now implements Clock.
+func (s *Simulated) Now() AbsTime {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return s.now
+}
+
+// Sleep implements Clock.
+func (s *Simulated) Sleep(d time.Duration) {
+	<-s.After(d)
+}
+
+// After implements Clock.
+func (s *Simulated) After(d time.Duration) <-chan time.Time {
+	after := make(chan time.Time, 1)
+	s.insert(d, func() {
+		after <- (time.Time{}).Add(time.Duration(s.now))
+	})
+	return after
+}
+
+func (s *Simulated) insert(d time.Duration, do func()) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.init()
+
+	at := s.now + AbsTime(d)
+	l, h := 0, len(s.scheduled)
+	ll := h
+	for l != h {
+		m := (l + h) / 2
+		if at < s.scheduled[m].at {
+			h = m
+		} else {
+			l = m + 1
+		}
+	}
+	s.scheduled = append(s.scheduled, event{})
+	copy(s.scheduled[l+1:], s.scheduled[l:ll])
+	s.scheduled[l] = event{do: do, at: at}
+	s.cond.Broadcast()
+}
+
+func (s *Simulated) init() {
+	if s.cond == nil {
+		s.cond = sync.NewCond(&s.mu)
+	}
+}

common/prque/prque.go

@@ -0,0 +1,57 @@
+// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque".
+
+package prque
+
+import (
+	"container/heap"
+)
+
+// Priority queue data structure.
+type Prque struct {
+	cont *sstack
+}
+
+// Creates a new priority queue.
+func New(setIndex setIndexCallback) *Prque {
+	return &Prque{newSstack(setIndex)}
+}
+
+// Pushes a value with a given priority into the queue, expanding if necessary.
+func (p *Prque) Push(data interface{}, priority int64) {
+	heap.Push(p.cont, &item{data, priority})
+}
+
+// Pops the value with the greates priority off the stack and returns it.
+// Currently no shrinking is done.
+func (p *Prque) Pop() (interface{}, int64) {
+	item := heap.Pop(p.cont).(*item)
+	return item.value, item.priority
+}
+
+// Pops only the item from the queue, dropping the associated priority value.
+func (p *Prque) PopItem() interface{} {
+	return heap.Pop(p.cont).(*item).value
+}
+
+// Remove removes the element with the given index.
+func (p *Prque) Remove(i int) interface{} {
+	if i < 0 {
+		return nil
+	}
+	return heap.Remove(p.cont, i)
+}
+
+// Checks whether the priority queue is empty.
+func (p *Prque) Empty() bool {
+	return p.cont.Len() == 0
+}
+
+// Returns the number of element in the priority queue.
+func (p *Prque) Size() int {
+	return p.cont.Len()
+}
+
+// Clears the contents of the priority queue.
+func (p *Prque) Reset() {
+	*p = *New(p.cont.setIndex)
+}

common/prque/sstack.go

@@ -0,0 +1,106 @@
+// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque".
+
+package prque
+
+// The size of a block of data
+const blockSize = 4096
+
+// A prioritized item in the sorted stack.
+//
+// Note: priorities can "wrap around" the int64 range, a comes before b if (a.priority - b.priority) > 0.
+// The difference between the lowest and highest priorities in the queue at any point should be less than 2^63.
+type item struct {
+	value    interface{}
+	priority int64
+}
+
+// setIndexCallback is called when the element is moved to a new index.
+// Providing setIndexCallback is optional, it is needed only if the application needs
+// to delete elements other than the top one.
+type setIndexCallback func(a interface{}, i int)
+
+// Internal sortable stack data structure. Implements the Push and Pop ops for
+// the stack (heap) functionality and the Len, Less and Swap methods for the
+// sortability requirements of the heaps.
+type sstack struct {
+	setIndex setIndexCallback
+	size     int
+	capacity int
+	offset   int
+
+	blocks [][]*item
+	active []*item
+}
+
+// Creates a new, empty stack.
+func newSstack(setIndex setIndexCallback) *sstack {
+	result := new(sstack)
+	result.setIndex = setIndex
+	result.active = make([]*item, blockSize)
+	result.blocks = [][]*item{result.active}
+	result.capacity = blockSize
+	return result
+}
+
+// Pushes a value onto the stack, expanding it if necessary. Required by
+// heap.Interface.
+func (s *sstack) Push(data interface{}) {
+	if s.size == s.capacity {
+		s.active = make([]*item, blockSize)
+		s.blocks = append(s.blocks, s.active)
+		s.capacity += blockSize
+		s.offset = 0
+	} else if s.offset == blockSize {
+		s.active = s.blocks[s.size/blockSize]
+		s.offset = 0
+	}
+	if s.setIndex != nil {
+		s.setIndex(data.(*item).value, s.size)
+	}
+	s.active[s.offset] = data.(*item)
+	s.offset++
+	s.size++
+}
+
+// Pops a value off the stack and returns it. Currently no shrinking is done.
+// Required by heap.Interface.
+func (s *sstack) Pop() (res interface{}) {
+	s.size--
+	s.offset--
+	if s.offset < 0 {
+		s.offset = blockSize - 1
+		s.active = s.blocks[s.size/blockSize]
+	}
+	res, s.active[s.offset] = s.active[s.offset], nil
+	if s.setIndex != nil {
+		s.setIndex(res.(*item).value, -1)
+	}
+	return
+}
+
+// Returns the length of the stack. Required by sort.Interface.
+func (s *sstack) Len() int {
+	return s.size
+}
+
+// Compares the priority of two elements of the stack (higher is first).
+// Required by sort.Interface.
+func (s *sstack) Less(i, j int) bool {
+	return (s.blocks[i/blockSize][i%blockSize].priority - s.blocks[j/blockSize][j%blockSize].priority) > 0
+}
+
+// Swaps two elements in the stack. Required by sort.Interface.
+func (s *sstack) Swap(i, j int) {
+	ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize
+	a, b := s.blocks[jb][jo], s.blocks[ib][io]
+	if s.setIndex != nil {
+		s.setIndex(a.value, i)
+		s.setIndex(b.value, j)
+	}
+	s.blocks[ib][io], s.blocks[jb][jo] = a, b
+}
+
+// Resets the stack, effectively clearing its contents.
+func (s *sstack) Reset() {
+	*s = *newSstack(s.setIndex)
+}

consensus/ethash/algorithm_test.go

@@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 
 		go func(idx int) {
 			defer pend.Done()
-			ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal})
+			ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil)
 			defer ethash.Close()
 			if err := ethash.VerifySeal(nil, block.Header()); err != nil {
 				t.Errorf("proc %d: block verification failed: %v", idx, err)

consensus/ethash/consensus.go

@@ -493,7 +493,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	if !bytes.Equal(header.MixDigest[:], digest) {
 		return errInvalidMixDigest
 	}
-	target := new(big.Int).Div(maxUint256, header.Difficulty)
+	target := new(big.Int).Div(two256, header.Difficulty)
 	if new(big.Int).SetBytes(result).Cmp(target) > 0 {
 		return errInvalidPoW
 	}

consensus/ethash/ethash.go

@@ -45,11 +45,11 @@ import (
 var ErrInvalidDumpMagic = errors.New("invalid dump magic")
 
 var (
-	// maxUint256 is a big integer representing 2^256-1
+	// two256 is a big integer representing 2^256
-	maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
+	two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
 
 	// sharedEthash is a full instance that can be shared between multiple users.
-	sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal})
+	sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil)
 
 	// algorithmRevision is the data structure version used for file naming.
 	algorithmRevision = 23
@@ -447,8 +447,10 @@ type Ethash struct {
 	exitCh    chan chan error // Notification channel to exiting backend threads
 }
 
-// New creates a full sized ethash PoW scheme and starts a background thread for remote mining.
+// New creates a full sized ethash PoW scheme and starts a background thread for
-func New(config Config) *Ethash {
+// remote mining, also optionally notifying a batch of remote services of new work
+// packages.
+func New(config Config, notify []string) *Ethash {
 	if config.CachesInMem <= 0 {
 		log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem)
 		config.CachesInMem = 1
@@ -473,13 +475,13 @@ func New(config Config) *Ethash {
 		submitRateCh: make(chan *hashrate),
 		exitCh:       make(chan chan error),
 	}
-	go ethash.remote()
+	go ethash.remote(notify)
 	return ethash
 }
 
 // NewTester creates a small sized ethash PoW scheme useful only for testing
 // purposes.
-func NewTester() *Ethash {
+func NewTester(notify []string) *Ethash {
 	ethash := &Ethash{
 		config:       Config{PowMode: ModeTest},
 		caches:       newlru("cache", 1, newCache),
@@ -494,7 +496,7 @@ func NewTester() *Ethash {
 		submitRateCh: make(chan *hashrate),
 		exitCh:       make(chan chan error),
 	}
-	go ethash.remote()
+	go ethash.remote(notify)
 	return ethash
 }
 

consensus/ethash/ethash_test.go

@@ -32,17 +32,18 @@ import (
 
 // Tests that ethash works correctly in test mode.
 func TestTestMode(t *testing.T) {
-	head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
+	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
 
-	ethash := NewTester()
+	ethash := NewTester(nil)
 	defer ethash.Close()
-	block, err := ethash.Seal(nil, types.NewBlockWithHeader(head), nil)
+
+	block, err := ethash.Seal(nil, types.NewBlockWithHeader(header), nil)
 	if err != nil {
 		t.Fatalf("failed to seal block: %v", err)
 	}
-	head.Nonce = types.EncodeNonce(block.Nonce())
+	header.Nonce = types.EncodeNonce(block.Nonce())
-	head.MixDigest = block.MixDigest()
+	header.MixDigest = block.MixDigest()
-	if err := ethash.VerifySeal(nil, head); err != nil {
+	if err := ethash.VerifySeal(nil, header); err != nil {
 		t.Fatalf("unexpected verification error: %v", err)
 	}
 }
@@ -55,7 +56,7 @@ func TestCacheFileEvict(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpdir)
-	e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest})
+	e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil)
 	defer e.Close()
 
 	workers := 8
@@ -78,21 +79,21 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) {
 		if block < 0 {
 			block = 0
 		}
-		head := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)}
+		header := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)}
-		e.VerifySeal(nil, head)
+		e.VerifySeal(nil, header)
 	}
 }
 
 func TestRemoteSealer(t *testing.T) {
-	ethash := NewTester()
+	ethash := NewTester(nil)
 	defer ethash.Close()
+
 	api := &API{ethash}
 	if _, err := api.GetWork(); err != errNoMiningWork {
 		t.Error("expect to return an error indicate there is no mining work")
 	}
-
+	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
-	head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
+	block := types.NewBlockWithHeader(header)
-	block := types.NewBlockWithHeader(head)
 
 	// Push new work.
 	ethash.Seal(nil, block, nil)
@@ -108,16 +109,14 @@ func TestRemoteSealer(t *testing.T) {
 	if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res {
 		t.Error("expect to return false when submit a fake solution")
 	}
-
 	// Push new block with same block number to replace the original one.
-	head = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)}
+	header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)}
-	block = types.NewBlockWithHeader(head)
+	block = types.NewBlockWithHeader(header)
 	ethash.Seal(nil, block, nil)
 
 	if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() {
 		t.Error("expect to return the latest pushed work")
 	}
-
 	// Push block with higher block number.
 	newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)}
 	newBlock := types.NewBlockWithHeader(newHead)
@@ -130,19 +129,18 @@ func TestRemoteSealer(t *testing.T) {
 
 func TestHashRate(t *testing.T) {
 	var (
-		ethash   = NewTester()
-		api      = &API{ethash}
 		hashrate = []hexutil.Uint64{100, 200, 300}
 		expect   uint64
 		ids      = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")}
 	)
-
+	ethash := NewTester(nil)
 	defer ethash.Close()
 
 	if tot := ethash.Hashrate(); tot != 0 {
 		t.Error("expect the result should be zero")
 	}
 
+	api := &API{ethash}
 	for i := 0; i < len(hashrate); i += 1 {
 		if res := api.SubmitHashRate(hashrate[i], ids[i]); !res {
 			t.Error("remote miner submit hashrate failed")
@@ -155,9 +153,8 @@ func TestHashRate(t *testing.T) {
 }
 
 func TestClosedRemoteSealer(t *testing.T) {
-	ethash := NewTester()
+	ethash := NewTester(nil)
-	// Make sure exit channel has been listened
+	time.Sleep(1 * time.Second) // ensure exit channel is listening
-	time.Sleep(1 * time.Second)
 	ethash.Close()
 
 	api := &API{ethash}

consensus/ethash/sealer.go

@@ -17,11 +17,14 @@
 package ethash
 
 import (
+	"bytes"
 	crand "crypto/rand"
+	"encoding/json"
 	"errors"
 	"math"
 	"math/big"
 	"math/rand"
+	"net/http"
 	"runtime"
 	"sync"
 	"time"
@@ -109,7 +112,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s
 	var (
 		header  = block.Header()
 		hash    = header.HashNoNonce().Bytes()
-		target  = new(big.Int).Div(maxUint256, header.Difficulty)
+		target  = new(big.Int).Div(two256, header.Difficulty)
 		number  = header.Number.Uint64()
 		dataset = ethash.dataset(number)
 	)
@@ -161,40 +164,65 @@ search:
 	runtime.KeepAlive(dataset)
 }
 
-// remote starts a standalone goroutine to handle remote mining related stuff.
+// remote is a standalone goroutine to handle remote mining related stuff.
-func (ethash *Ethash) remote() {
+func (ethash *Ethash) remote(notify []string) {
 	var (
 		works = make(map[common.Hash]*types.Block)
 		rates = make(map[common.Hash]hashrate)
-		currentWork *types.Block
-	)
 
-	// getWork returns a work package for external miner.
+		currentBlock *types.Block
+		currentWork  [3]string
+
+		notifyTransport = &http.Transport{}
+		notifyClient    = &http.Client{
+			Transport: notifyTransport,
+			Timeout:   time.Second,
+		}
+		notifyReqs = make([]*http.Request, len(notify))
+	)
+	// notifyWork notifies all the specified mining endpoints of the availability of
+	// new work to be processed.
+	notifyWork := func() {
+		work := currentWork
+		blob, _ := json.Marshal(work)
+
+		for i, url := range notify {
+			// Terminate any previously pending request and create the new work
+			if notifyReqs[i] != nil {
+				notifyTransport.CancelRequest(notifyReqs[i])
+			}
+			notifyReqs[i], _ = http.NewRequest("POST", url, bytes.NewReader(blob))
+			notifyReqs[i].Header.Set("Content-Type", "application/json")
+
+			// Push the new work concurrently to all the remote nodes
+			go func(req *http.Request, url string) {
+				res, err := notifyClient.Do(req)
+				if err != nil {
+					log.Warn("Failed to notify remote miner", "err", err)
+				} else {
+					log.Trace("Notified remote miner", "miner", url, "hash", log.Lazy{Fn: func() common.Hash { return common.HexToHash(work[0]) }}, "target", work[2])
+					res.Body.Close()
+				}
+			}(notifyReqs[i], url)
+		}
+	}
+	// makeWork creates a work package for external miner.
 	//
 	// The work package consists of 3 strings:
 	//   result[0], 32 bytes hex encoded current block header pow-hash
 	//   result[1], 32 bytes hex encoded seed hash used for DAG
 	//   result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
-	getWork := func() ([3]string, error) {
+	makeWork := func(block *types.Block) {
-		var res [3]string
+		hash := block.HashNoNonce()
-		if currentWork == nil {
-			return res, errNoMiningWork
-		}
-		res[0] = currentWork.HashNoNonce().Hex()
-		res[1] = common.BytesToHash(SeedHash(currentWork.NumberU64())).Hex()
 
-		// Calculate the "target" to be returned to the external sealer.
+		currentWork[0] = hash.Hex()
-		n := big.NewInt(1)
+		currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
-		n.Lsh(n, 255)
+		currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex()
-		n.Div(n, currentWork.Difficulty())
-		n.Lsh(n, 1)
-		res[2] = common.BytesToHash(n.Bytes()).Hex()
 
 		// Trace the seal work fetched by remote sealer.
-		works[currentWork.HashNoNonce()] = currentWork
+		currentBlock = block
-		return res, nil
+		works[hash] = block
 	}
-
 	// submitWork verifies the submitted pow solution, returning
 	// whether the solution was accepted or not (not can be both a bad pow as well as
 	// any other error, like no pending work or stale mining result).
@@ -238,21 +266,23 @@ func (ethash *Ethash) remote() {
 	for {
 		select {
 		case block := <-ethash.workCh:
-			if currentWork != nil && block.ParentHash() != currentWork.ParentHash() {
+			if currentBlock != nil && block.ParentHash() != currentBlock.ParentHash() {
 				// Start new round mining, throw out all previous work.
 				works = make(map[common.Hash]*types.Block)
 			}
 			// Update current work with new received block.
 			// Note same work can be past twice, happens when changing CPU threads.
-			currentWork = block
+			makeWork(block)
+
+			// Notify and requested URLs of the new work availability
+			notifyWork()
 
 		case work := <-ethash.fetchWorkCh:
 			// Return current mining work to remote miner.
-			miningWork, err := getWork()
+			if currentBlock == nil {
-			if err != nil {
+				work.errc <- errNoMiningWork
-				work.errc <- err
 			} else {
-				work.res <- miningWork
+				work.res <- currentWork
 			}
 
 		case result := <-ethash.submitWorkCh:

consensus/ethash/sealer_test.go

@@ -0,0 +1,115 @@
+package ethash
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"math/big"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// Tests whether remote HTTP servers are correctly notified of new work.
+func TestRemoteNotify(t *testing.T) {
+	// Start a simple webserver to capture notifications
+	sink := make(chan [3]string)
+
+	server := &http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			blob, err := ioutil.ReadAll(req.Body)
+			if err != nil {
+				t.Fatalf("failed to read miner notification: %v", err)
+			}
+			var work [3]string
+			if err := json.Unmarshal(blob, &work); err != nil {
+				t.Fatalf("failed to unmarshal miner notification: %v", err)
+			}
+			sink <- work
+		}),
+	}
+	// Open a custom listener to extract its local address
+	listener, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("failed to open notification server: %v", err)
+	}
+	defer listener.Close()
+
+	go server.Serve(listener)
+
+	// Create the custom ethash engine
+	ethash := NewTester([]string{"http://" + listener.Addr().String()})
+	defer ethash.Close()
+
+	// Stream a work task and ensure the notification bubbles out
+	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
+	block := types.NewBlockWithHeader(header)
+
+	ethash.Seal(nil, block, nil)
+	select {
+	case work := <-sink:
+		if want := header.HashNoNonce().Hex(); work[0] != want {
+			t.Errorf("work packet hash mismatch: have %s, want %s", work[0], want)
+		}
+		if want := common.BytesToHash(SeedHash(header.Number.Uint64())).Hex(); work[1] != want {
+			t.Errorf("work packet seed mismatch: have %s, want %s", work[1], want)
+		}
+		target := new(big.Int).Div(new(big.Int).Lsh(big.NewInt(1), 256), header.Difficulty)
+		if want := common.BytesToHash(target.Bytes()).Hex(); work[2] != want {
+			t.Errorf("work packet target mismatch: have %s, want %s", work[2], want)
+		}
+	case <-time.After(time.Second):
+		t.Fatalf("notification timed out")
+	}
+}
+
+// Tests that pushing work packages fast to the miner doesn't cause any daa race
+// issues in the notifications.
+func TestRemoteMultiNotify(t *testing.T) {
+	// Start a simple webserver to capture notifications
+	sink := make(chan [3]string, 1024)
+
+	server := &http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			blob, err := ioutil.ReadAll(req.Body)
+			if err != nil {
+				t.Fatalf("failed to read miner notification: %v", err)
+			}
+			var work [3]string
+			if err := json.Unmarshal(blob, &work); err != nil {
+				t.Fatalf("failed to unmarshal miner notification: %v", err)
+			}
+			sink <- work
+		}),
+	}
+	// Open a custom listener to extract its local address
+	listener, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("failed to open notification server: %v", err)
+	}
+	defer listener.Close()
+
+	go server.Serve(listener)
+
+	// Create the custom ethash engine
+	ethash := NewTester([]string{"http://" + listener.Addr().String()})
+	defer ethash.Close()
+
+	// Stream a lot of work task and ensure all the notifications bubble out
+	for i := 0; i < cap(sink); i++ {
+		header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)}
+		block := types.NewBlockWithHeader(header)
+
+		ethash.Seal(nil, block, nil)
+	}
+	for i := 0; i < cap(sink); i++ {
+		select {
+		case <-sink:
+		case <-time.After(250 * time.Millisecond):
+			t.Fatalf("notification %d timed out", i)
+		}
+	}
+}

contracts/chequebook/cheque_test.go

@@ -46,7 +46,7 @@ func newTestBackend() *backends.SimulatedBackend {
 		addr0: {Balance: big.NewInt(1000000000)},
 		addr1: {Balance: big.NewInt(1000000000)},
 		addr2: {Balance: big.NewInt(1000000000)},
-	})
+	}, 10000000)
 }
 
 func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) {

contracts/chequebook/gencode.go

@@ -40,7 +40,7 @@ var (
 )
 
 func main() {
-	backend := backends.NewSimulatedBackend(testAlloc)
+	backend := backends.NewSimulatedBackend(testAlloc, uint64(100000000))
 	auth := bind.NewKeyedTransactor(testKey)
 
 	// Deploy the contract, get the code.

contracts/ens/ens_test.go

@@ -35,7 +35,7 @@ var (
 )
 
 func TestENS(t *testing.T) {
-	contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}})
+	contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, 10000000)
 	transactOpts := bind.NewKeyedTransactor(key)
 
 	ensAddr, ens, err := DeployENS(transactOpts, contractBackend)

core/blockchain.go

@@ -899,9 +899,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil {
 		return NonStatTy, err
 	}
-	// Write other block data using a batch.
+	rawdb.WriteBlock(bc.db, block)
-	batch := bc.db.NewBatch()
-	rawdb.WriteBlock(batch, block)
 
 	root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number()))
 	if err != nil {
@@ -955,6 +953,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 			}
 		}
 	}
+
+	// Write other block data using a batch.
+	batch := bc.db.NewBatch()
 	rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts)
 
 	// If the total difficulty is higher than our known, add it to the canonical chain

core/tx_cacher.go

@@ -22,7 +22,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
-// senderCacher is a concurrent tranaction sender recoverer anc cacher.
+// senderCacher is a concurrent transaction sender recoverer anc cacher.
 var senderCacher = newTxSenderCacher(runtime.NumCPU())
 
 // txSenderCacherRequest is a request for recovering transaction senders with a
@@ -45,7 +45,7 @@ type txSenderCacher struct {
 }
 
 // newTxSenderCacher creates a new transaction sender background cacher and starts
-// as many procesing goroutines as allowed by the GOMAXPROCS on construction.
+// as many processing goroutines as allowed by the GOMAXPROCS on construction.
 func newTxSenderCacher(threads int) *txSenderCacher {
 	cacher := &txSenderCacher{
 		tasks:   make(chan *txSenderCacherRequest, threads),

core/vm/evm.go

@@ -184,7 +184,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 			precompiles = PrecompiledContractsByzantium
 		}
 		if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 {
-			// Calling a non existing account, don't do antything, but ping the tracer
+			// Calling a non existing account, don't do anything, but ping the tracer
 			if evm.vmConfig.Debug && evm.depth == 0 {
 				evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value)
 				evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil)
@@ -427,7 +427,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I
 
 // Create2 creates a new contract using code as deployment code.
 //
-// The different between Create2 with Create is Create2 uses sha3(msg.sender ++ salt ++ init_code)[12:]
+// The different between Create2 with Create is Create2 uses sha3(0xff ++ msg.sender ++ salt ++ sha3(init_code))[12:]
 // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at.
 func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) {
 	contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), code)

crypto/crypto.go

@@ -78,8 +78,8 @@ func CreateAddress(b common.Address, nonce uint64) common.Address {
 
 // CreateAddress2 creates an ethereum address given the address bytes, initial
 // contract code and a salt.
-func CreateAddress2(b common.Address, salt common.Hash, code []byte) common.Address {
+func CreateAddress2(b common.Address, salt [32]byte, code []byte) common.Address {
-	return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt.Bytes(), code)[12:])
+	return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], Keccak256(code))[12:])
 }
 
 // ToECDSA creates a private key with the given D value.

eth/api_tracer.go

@@ -297,7 +297,9 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				database.TrieDB().Reference(root, common.Hash{})
 			}
 			// Dereference all past tries we ourselves are done working with
+			if proot != (common.Hash{}) {
 				database.TrieDB().Dereference(proot)
+			}
 			proot = root
 
 			// TODO(karalabe): Do we need the preimages? Won't they accumulate too much?
@@ -526,7 +528,9 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 			return nil, err
 		}
 		database.TrieDB().Reference(root, common.Hash{})
+		if proot != (common.Hash{}) {
 			database.TrieDB().Dereference(proot)
+		}
 		proot = root
 	}
 	nodes, imgs := database.TrieDB().Size()

eth/backend.go

@@ -124,7 +124,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		chainConfig:    chainConfig,
 		eventMux:       ctx.EventMux,
 		accountManager: ctx.AccountManager,
-		engine:         CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb),
+		engine:         CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb),
 		shutdownChan:   make(chan bool),
 		networkID:      config.NetworkId,
 		gasPrice:       config.GasPrice,
@@ -210,7 +210,7 @@ func CreateDB(ctx *node.ServiceContext, config *Config, name string) (ethdb.Data
 }
 
 // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service
-func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chainConfig *params.ChainConfig, db ethdb.Database) consensus.Engine {
+func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, db ethdb.Database) consensus.Engine {
 	// If proof-of-authority is requested, set it up
 	if chainConfig.Clique != nil {
 		return clique.New(chainConfig.Clique, db)
@@ -222,7 +222,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai
 		return ethash.NewFaker()
 	case ethash.ModeTest:
 		log.Warn("Ethash used in test mode")
-		return ethash.NewTester()
+		return ethash.NewTester(nil)
 	case ethash.ModeShared:
 		log.Warn("Ethash used in shared mode")
 		return ethash.NewShared()
@@ -234,7 +234,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai
 			DatasetDir:     config.DatasetDir,
 			DatasetsInMem:  config.DatasetsInMem,
 			DatasetsOnDisk: config.DatasetsOnDisk,
-		})
+		}, notify)
 		engine.SetThreads(-1) // Disable CPU mining
 		return engine
 	}

eth/config.go

@@ -97,6 +97,7 @@ type Config struct {
 	// Mining-related options
 	Etherbase    common.Address `toml:",omitempty"`
 	MinerThreads int            `toml:",omitempty"`
+	MinerNotify  []string       `toml:",omitempty"`
 	ExtraData    []byte         `toml:",omitempty"`
 	GasPrice     *big.Int
 

internal/web3ext/web3ext.go

@@ -152,6 +152,16 @@ web3._extend({
 			call: 'admin_removePeer',
 			params: 1
 		}),
+		new web3._extend.Method({
+			name: 'addTrustedPeer',
+			call: 'admin_addTrustedPeer',
+			params: 1
+		}),
+		new web3._extend.Method({
+			name: 'removeTrustedPeer',
+			call: 'admin_removeTrustedPeer',
+			params: 1
+		}),
 		new web3._extend.Method({
 			name: 'exportChain',
 			call: 'admin_exportChain',

les/backend.go

@@ -102,7 +102,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) {
 		peers:            peers,
 		reqDist:          newRequestDistributor(peers, quitSync),
 		accountManager:   ctx.AccountManager,
-		engine:           eth.CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb),
+		engine:           eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb),
 		shutdownChan:     make(chan bool),
 		networkId:        config.NetworkId,
 		bloomRequests:    make(chan chan *bloombits.Retrieval),

les/freeclient.go

@@ -0,0 +1,278 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package les implements the Light Ethereum Subprotocol.
+package les
+
+import (
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/common/prque"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// freeClientPool implements a client database that limits the connection time
+// of each client and manages accepting/rejecting incoming connections and even
+// kicking out some connected clients. The pool calculates recent usage time
+// for each known client (a value that increases linearly when the client is
+// connected and decreases exponentially when not connected). Clients with lower
+// recent usage are preferred, unknown nodes have the highest priority. Already
+// connected nodes receive a small bias in their favor in order to avoid accepting
+// and instantly kicking out clients.
+//
+// Note: the pool can use any string for client identification. Using signature
+// keys for that purpose would not make sense when being known has a negative
+// value for the client. Currently the LES protocol manager uses IP addresses
+// (without port address) to identify clients.
+type freeClientPool struct {
+	db     ethdb.Database
+	lock   sync.Mutex
+	clock  mclock.Clock
+	closed bool
+
+	connectedLimit, totalLimit int
+
+	addressMap            map[string]*freeClientPoolEntry
+	connPool, disconnPool *prque.Prque
+	startupTime           mclock.AbsTime
+	logOffsetAtStartup    int64
+}
+
+const (
+	recentUsageExpTC     = time.Hour   // time constant of the exponential weighting window for "recent" server usage
+	fixedPointMultiplier = 0x1000000   // constant to convert logarithms to fixed point format
+	connectedBias        = time.Minute // this bias is applied in favor of already connected clients in order to avoid kicking them out very soon
+)
+
+// newFreeClientPool creates a new free client pool
+func newFreeClientPool(db ethdb.Database, connectedLimit, totalLimit int, clock mclock.Clock) *freeClientPool {
+	pool := &freeClientPool{
+		db:             db,
+		clock:          clock,
+		addressMap:     make(map[string]*freeClientPoolEntry),
+		connPool:       prque.New(poolSetIndex),
+		disconnPool:    prque.New(poolSetIndex),
+		connectedLimit: connectedLimit,
+		totalLimit:     totalLimit,
+	}
+	pool.loadFromDb()
+	return pool
+}
+
+func (f *freeClientPool) stop() {
+	f.lock.Lock()
+	f.closed = true
+	f.saveToDb()
+	f.lock.Unlock()
+}
+
+// connect should be called after a successful handshake. If the connection was
+// rejected, there is no need to call disconnect.
+//
+// Note: the disconnectFn callback should not block.
+func (f *freeClientPool) connect(address string, disconnectFn func()) bool {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	if f.closed {
+		return false
+	}
+	e := f.addressMap[address]
+	now := f.clock.Now()
+	var recentUsage int64
+	if e == nil {
+		e = &freeClientPoolEntry{address: address, index: -1}
+		f.addressMap[address] = e
+	} else {
+		if e.connected {
+			log.Debug("Client already connected", "address", address)
+			return false
+		}
+		recentUsage = int64(math.Exp(float64(e.logUsage-f.logOffset(now)) / fixedPointMultiplier))
+	}
+	e.linUsage = recentUsage - int64(now)
+	// check whether (linUsage+connectedBias) is smaller than the highest entry in the connected pool
+	if f.connPool.Size() == f.connectedLimit {
+		i := f.connPool.PopItem().(*freeClientPoolEntry)
+		if e.linUsage+int64(connectedBias)-i.linUsage < 0 {
+			// kick it out and accept the new client
+			f.connPool.Remove(i.index)
+			f.calcLogUsage(i, now)
+			i.connected = false
+			f.disconnPool.Push(i, -i.logUsage)
+			log.Debug("Client kicked out", "address", i.address)
+			i.disconnectFn()
+		} else {
+			// keep the old client and reject the new one
+			f.connPool.Push(i, i.linUsage)
+			log.Debug("Client rejected", "address", address)
+			return false
+		}
+	}
+	f.disconnPool.Remove(e.index)
+	e.connected = true
+	e.disconnectFn = disconnectFn
+	f.connPool.Push(e, e.linUsage)
+	if f.connPool.Size()+f.disconnPool.Size() > f.totalLimit {
+		f.disconnPool.Pop()
+	}
+	log.Debug("Client accepted", "address", address)
+	return true
+}
+
+// disconnect should be called when a connection is terminated. If the disconnection
+// was initiated by the pool itself using disconnectFn then calling disconnect is
+// not necessary but permitted.
+func (f *freeClientPool) disconnect(address string) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	if f.closed {
+		return
+	}
+	e := f.addressMap[address]
+	now := f.clock.Now()
+	if !e.connected {
+		log.Debug("Client already disconnected", "address", address)
+		return
+	}
+
+	f.connPool.Remove(e.index)
+	f.calcLogUsage(e, now)
+	e.connected = false
+	f.disconnPool.Push(e, -e.logUsage)
+	log.Debug("Client disconnected", "address", address)
+}
+
+// logOffset calculates the time-dependent offset for the logarithmic
+// representation of recent usage
+func (f *freeClientPool) logOffset(now mclock.AbsTime) int64 {
+	// Note: fixedPointMultiplier acts as a multiplier here; the reason for dividing the divisor
+	// is to avoid int64 overflow. We assume that int64(recentUsageExpTC) >> fixedPointMultiplier.
+	logDecay := int64((time.Duration(now - f.startupTime)) / (recentUsageExpTC / fixedPointMultiplier))
+	return f.logOffsetAtStartup + logDecay
+}
+
+// calcLogUsage converts recent usage from linear to logarithmic representation
+// when disconnecting a peer or closing the client pool
+func (f *freeClientPool) calcLogUsage(e *freeClientPoolEntry, now mclock.AbsTime) {
+	dt := e.linUsage + int64(now)
+	if dt < 1 {
+		dt = 1
+	}
+	e.logUsage = int64(math.Log(float64(dt))*fixedPointMultiplier) + f.logOffset(now)
+}
+
+// freeClientPoolStorage is the RLP representation of the pool's database storage
+type freeClientPoolStorage struct {
+	LogOffset uint64
+	List      []*freeClientPoolEntry
+}
+
+// loadFromDb restores pool status from the database storage
+// (automatically called at initialization)
+func (f *freeClientPool) loadFromDb() {
+	enc, err := f.db.Get([]byte("freeClientPool"))
+	if err != nil {
+		return
+	}
+	var storage freeClientPoolStorage
+	err = rlp.DecodeBytes(enc, &storage)
+	if err != nil {
+		log.Error("Failed to decode client list", "err", err)
+		return
+	}
+	f.logOffsetAtStartup = int64(storage.LogOffset)
+	f.startupTime = f.clock.Now()
+	for _, e := range storage.List {
+		log.Debug("Loaded free client record", "address", e.address, "logUsage", e.logUsage)
+		f.addressMap[e.address] = e
+		f.disconnPool.Push(e, -e.logUsage)
+	}
+}
+
+// saveToDb saves pool status to the database storage
+// (automatically called during shutdown)
+func (f *freeClientPool) saveToDb() {
+	now := f.clock.Now()
+	storage := freeClientPoolStorage{
+		LogOffset: uint64(f.logOffset(now)),
+		List:      make([]*freeClientPoolEntry, len(f.addressMap)),
+	}
+	i := 0
+	for _, e := range f.addressMap {
+		if e.connected {
+			f.calcLogUsage(e, now)
+		}
+		storage.List[i] = e
+		i++
+	}
+	enc, err := rlp.EncodeToBytes(storage)
+	if err != nil {
+		log.Error("Failed to encode client list", "err", err)
+	} else {
+		f.db.Put([]byte("freeClientPool"), enc)
+	}
+}
+
+// freeClientPoolEntry represents a client address known by the pool.
+// When connected, recent usage is calculated as linUsage + int64(clock.Now())
+// When disconnected, it is calculated as exp(logUsage - logOffset) where logOffset
+// also grows linearly with time while the server is running.
+// Conversion between linear and logarithmic representation happens when connecting
+// or disconnecting the node.
+//
+// Note: linUsage and logUsage are values used with constantly growing offsets so
+// even though they are close to each other at any time they may wrap around int64
+// limits over time. Comparison should be performed accordingly.
+type freeClientPoolEntry struct {
+	address            string
+	connected          bool
+	disconnectFn       func()
+	linUsage, logUsage int64
+	index              int
+}
+
+func (e *freeClientPoolEntry) EncodeRLP(w io.Writer) error {
+	return rlp.Encode(w, []interface{}{e.address, uint64(e.logUsage)})
+}
+
+func (e *freeClientPoolEntry) DecodeRLP(s *rlp.Stream) error {
+	var entry struct {
+		Address  string
+		LogUsage uint64
+	}
+	if err := s.Decode(&entry); err != nil {
+		return err
+	}
+	e.address = entry.Address
+	e.logUsage = int64(entry.LogUsage)
+	e.connected = false
+	e.index = -1
+	return nil
+}
+
+// poolSetIndex callback is used by both priority queues to set/update the index of
+// the element in the queue. Index is needed to remove elements other than the top one.
+func poolSetIndex(a interface{}, i int) {
+	a.(*freeClientPoolEntry).index = i
+}

les/freeclient_test.go

@@ -0,0 +1,139 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package light implements on-demand retrieval capable state and chain objects
+// for the Ethereum Light Client.
+package les
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+func TestFreeClientPoolL10C100(t *testing.T) {
+	testFreeClientPool(t, 10, 100)
+}
+
+func TestFreeClientPoolL40C200(t *testing.T) {
+	testFreeClientPool(t, 40, 200)
+}
+
+func TestFreeClientPoolL100C300(t *testing.T) {
+	testFreeClientPool(t, 100, 300)
+}
+
+const testFreeClientPoolTicks = 500000
+
+func testFreeClientPool(t *testing.T, connLimit, clientCount int) {
+	var (
+		clock     mclock.Simulated
+		db        = ethdb.NewMemDatabase()
+		pool      = newFreeClientPool(db, connLimit, 10000, &clock)
+		connected = make([]bool, clientCount)
+		connTicks = make([]int, clientCount)
+		disconnCh = make(chan int, clientCount)
+	)
+	peerId := func(i int) string {
+		return fmt.Sprintf("test peer #%d", i)
+	}
+	disconnFn := func(i int) func() {
+		return func() {
+			disconnCh <- i
+		}
+	}
+
+	// pool should accept new peers up to its connected limit
+	for i := 0; i < connLimit; i++ {
+		if pool.connect(peerId(i), disconnFn(i)) {
+			connected[i] = true
+		} else {
+			t.Fatalf("Test peer #%d rejected", i)
+		}
+	}
+	// since all accepted peers are new and should not be kicked out, the next one should be rejected
+	if pool.connect(peerId(connLimit), disconnFn(connLimit)) {
+		connected[connLimit] = true
+		t.Fatalf("Peer accepted over connected limit")
+	}
+
+	// randomly connect and disconnect peers, expect to have a similar total connection time at the end
+	for tickCounter := 0; tickCounter < testFreeClientPoolTicks; tickCounter++ {
+		clock.Run(1 * time.Second)
+
+		i := rand.Intn(clientCount)
+		if connected[i] {
+			pool.disconnect(peerId(i))
+			connected[i] = false
+			connTicks[i] += tickCounter
+		} else {
+			if pool.connect(peerId(i), disconnFn(i)) {
+				connected[i] = true
+				connTicks[i] -= tickCounter
+			}
+		}
+	pollDisconnects:
+		for {
+			select {
+			case i := <-disconnCh:
+				pool.disconnect(peerId(i))
+				if connected[i] {
+					connTicks[i] += tickCounter
+					connected[i] = false
+				}
+			default:
+				break pollDisconnects
+			}
+		}
+	}
+
+	expTicks := testFreeClientPoolTicks * connLimit / clientCount
+	expMin := expTicks - expTicks/10
+	expMax := expTicks + expTicks/10
+
+	// check if the total connected time of peers are all in the expected range
+	for i, c := range connected {
+		if c {
+			connTicks[i] += testFreeClientPoolTicks
+		}
+		if connTicks[i] < expMin || connTicks[i] > expMax {
+			t.Errorf("Total connected time of test node #%d (%d) outside expected range (%d to %d)", i, connTicks[i], expMin, expMax)
+		}
+	}
+
+	// a previously unknown peer should be accepted now
+	if !pool.connect("newPeer", func() {}) {
+		t.Fatalf("Previously unknown peer rejected")
+	}
+
+	// close and restart pool
+	pool.stop()
+	pool = newFreeClientPool(db, connLimit, 10000, &clock)
+
+	// try connecting all known peers (connLimit should be filled up)
+	for i := 0; i < clientCount; i++ {
+		pool.connect(peerId(i), func() {})
+	}
+	// expect pool to remember known nodes and kick out one of them to accept a new one
+	if !pool.connect("newPeer2", func() {}) {
+		t.Errorf("Previously unknown peer rejected after restarting pool")
+	}
+	pool.stop()
+}

les/handler.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/mclock"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -104,6 +105,7 @@ type ProtocolManager struct {
 	odr         *LesOdr
 	server      *LesServer
 	serverPool  *serverPool
+	clientPool  *freeClientPool
 	lesTopic    discv5.Topic
 	reqDist     *requestDistributor
 	retriever   *retrieveManager
@@ -226,6 +228,7 @@ func (pm *ProtocolManager) Start(maxPeers int) {
 	if pm.lightSync {
 		go pm.syncer()
 	} else {
+		pm.clientPool = newFreeClientPool(pm.chainDb, maxPeers, 10000, mclock.System{})
 		go func() {
 			for range pm.newPeerCh {
 			}
@@ -243,6 +246,9 @@ func (pm *ProtocolManager) Stop() {
 	pm.noMorePeers <- struct{}{}
 
 	close(pm.quitSync) // quits syncer, fetcher
+	if pm.clientPool != nil {
+		pm.clientPool.stop()
+	}
 
 	// Disconnect existing sessions.
 	// This also closes the gate for any new registrations on the peer set.
@@ -264,7 +270,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
 	// Ignore maxPeers if this is a trusted peer
-	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
+	// In server mode we try to check into the client pool after handshake
+	if pm.lightSync && pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 
@@ -282,6 +289,19 @@ func (pm *ProtocolManager) handle(p *peer) error {
 		p.Log().Debug("Light Ethereum handshake failed", "err", err)
 		return err
 	}
+
+	if !pm.lightSync && !p.Peer.Info().Network.Trusted {
+		addr, ok := p.RemoteAddr().(*net.TCPAddr)
+		// test peer address is not a tcp address, don't use client pool if can not typecast
+		if ok {
+			id := addr.IP.String()
+			if !pm.clientPool.connect(id, func() { go pm.removePeer(p.id) }) {
+				return p2p.DiscTooManyPeers
+			}
+			defer pm.clientPool.disconnect(id)
+		}
+	}
+
 	if rw, ok := p.rw.(*meteredMsgReadWriter); ok {
 		rw.Init(p.version)
 	}

miner/agent.go

@@ -1,111 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package miner
-
-import (
-	"sync"
-	"sync/atomic"
-
-	"github.com/ethereum/go-ethereum/consensus"
-	"github.com/ethereum/go-ethereum/log"
-)
-
-type CpuAgent struct {
-	mu sync.Mutex
-
-	workCh        chan *Work
-	stop          chan struct{}
-	quitCurrentOp chan struct{}
-	returnCh      chan<- *Result
-
-	chain  consensus.ChainReader
-	engine consensus.Engine
-
-	started int32 // started indicates whether the agent is currently started
-}
-
-func NewCpuAgent(chain consensus.ChainReader, engine consensus.Engine) *CpuAgent {
-	agent := &CpuAgent{
-		chain:  chain,
-		engine: engine,
-		stop:   make(chan struct{}, 1),
-		workCh: make(chan *Work, 1),
-	}
-	return agent
-}
-
-func (self *CpuAgent) Work() chan<- *Work            { return self.workCh }
-func (self *CpuAgent) SetReturnCh(ch chan<- *Result) { self.returnCh = ch }
-
-func (self *CpuAgent) Start() {
-	if !atomic.CompareAndSwapInt32(&self.started, 0, 1) {
-		return // agent already started
-	}
-	go self.update()
-}
-
-func (self *CpuAgent) Stop() {
-	if !atomic.CompareAndSwapInt32(&self.started, 1, 0) {
-		return // agent already stopped
-	}
-	self.stop <- struct{}{}
-done:
-	// Empty work channel
-	for {
-		select {
-		case <-self.workCh:
-		default:
-			break done
-		}
-	}
-}
-
-func (self *CpuAgent) update() {
-out:
-	for {
-		select {
-		case work := <-self.workCh:
-			self.mu.Lock()
-			if self.quitCurrentOp != nil {
-				close(self.quitCurrentOp)
-			}
-			self.quitCurrentOp = make(chan struct{})
-			go self.mine(work, self.quitCurrentOp)
-			self.mu.Unlock()
-		case <-self.stop:
-			self.mu.Lock()
-			if self.quitCurrentOp != nil {
-				close(self.quitCurrentOp)
-				self.quitCurrentOp = nil
-			}
-			self.mu.Unlock()
-			break out
-		}
-	}
-}
-
-func (self *CpuAgent) mine(work *Work, stop <-chan struct{}) {
-	if result, err := self.engine.Seal(self.chain, work.Block, stop); result != nil {
-		log.Info("Successfully sealed new block", "number", result.Number(), "hash", result.Hash())
-		self.returnCh <- &Result{work, result}
-	} else {
-		if err != nil {
-			log.Warn("Block sealing failed", "err", err)
-		}
-		self.returnCh <- nil
-	}
-}

miner/miner.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/downloader"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
@@ -36,10 +34,8 @@ import (
 
 // Backend wraps all methods required for mining.
 type Backend interface {
-	AccountManager() *accounts.Manager
 	BlockChain() *core.BlockChain
 	TxPool() *core.TxPool
-	ChainDb() ethdb.Database
 }
 
 // Miner creates blocks and searches for proof-of-work values.
@@ -49,6 +45,7 @@ type Miner struct {
 	coinbase common.Address
 	eth      Backend
 	engine   consensus.Engine
+	exitCh   chan struct{}
 
 	canStart    int32 // can start indicates whether we can start the mining operation
 	shouldStart int32 // should start indicates whether we should start after sync
@@ -59,10 +56,10 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con
 		eth:      eth,
 		mux:      mux,
 		engine:   engine,
+		exitCh:   make(chan struct{}),
 		worker:   newWorker(config, engine, eth, mux),
 		canStart: 1,
 	}
-	miner.Register(NewCpuAgent(eth.BlockChain(), engine))
 	go miner.update()
 
 	return miner
@@ -74,8 +71,14 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con
 // and halt your mining operation for as long as the DOS continues.
 func (self *Miner) update() {
 	events := self.mux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{})
-out:
+	defer events.Unsubscribe()
-	for ev := range events.Chan() {
+
+	for {
+		select {
+		case ev := <-events.Chan():
+			if ev == nil {
+				return
+			}
 			switch ev.Data.(type) {
 			case downloader.StartEvent:
 				atomic.StoreInt32(&self.canStart, 0)
@@ -92,10 +95,11 @@ out:
 				if shouldStart {
 					self.Start(self.coinbase)
 				}
-			// unsubscribe. we're only interested in this event once
-			events.Unsubscribe()
 				// stop immediately and ignore all further pending events
-			break out
+				return
+			}
+		case <-self.exitCh:
+			return
 		}
 	}
 }
@@ -109,7 +113,6 @@ func (self *Miner) Start(coinbase common.Address) {
 		return
 	}
 	self.worker.start()
-	self.worker.commitNewWork()
 }
 
 func (self *Miner) Stop() {
@@ -117,12 +120,9 @@ func (self *Miner) Stop() {
 	atomic.StoreInt32(&self.shouldStart, 0)
 }
 
-func (self *Miner) Register(agent Agent) {
+func (self *Miner) Close() {
-	self.worker.register(agent)
+	self.worker.close()
-}
+	close(self.exitCh)
-
-func (self *Miner) Unregister(agent Agent) {
-	self.worker.unregister(agent)
 }
 
 func (self *Miner) Mining() bool {

miner/worker_test.go

@@ -0,0 +1,212 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package miner
+
+import (
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/clique"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+var (
+	// Test chain configurations
+	testTxPoolConfig  core.TxPoolConfig
+	ethashChainConfig *params.ChainConfig
+	cliqueChainConfig *params.ChainConfig
+
+	// Test accounts
+	testBankKey, _  = crypto.GenerateKey()
+	testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
+	testBankFunds   = big.NewInt(1000000000000000000)
+
+	acc1Key, _ = crypto.GenerateKey()
+	acc1Addr   = crypto.PubkeyToAddress(acc1Key.PublicKey)
+
+	// Test transactions
+	pendingTxs []*types.Transaction
+	newTxs     []*types.Transaction
+)
+
+func init() {
+	testTxPoolConfig = core.DefaultTxPoolConfig
+	testTxPoolConfig.Journal = ""
+	ethashChainConfig = params.TestChainConfig
+	cliqueChainConfig = params.TestChainConfig
+	cliqueChainConfig.Clique = &params.CliqueConfig{
+		Period: 1,
+		Epoch:  30000,
+	}
+	tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey)
+	pendingTxs = append(pendingTxs, tx1)
+	tx2, _ := types.SignTx(types.NewTransaction(1, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey)
+	newTxs = append(newTxs, tx2)
+}
+
+// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing.
+type testWorkerBackend struct {
+	db         ethdb.Database
+	txPool     *core.TxPool
+	chain      *core.BlockChain
+	testTxFeed event.Feed
+}
+
+func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend {
+	var (
+		db    = ethdb.NewMemDatabase()
+		gspec = core.Genesis{
+			Config: chainConfig,
+			Alloc:  core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}},
+		}
+	)
+
+	switch engine.(type) {
+	case *clique.Clique:
+		gspec.ExtraData = make([]byte, 32+common.AddressLength+65)
+		copy(gspec.ExtraData[32:], testBankAddress[:])
+	case *ethash.Ethash:
+	default:
+		t.Fatal("unexpect consensus engine type")
+	}
+	gspec.MustCommit(db)
+
+	chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
+	txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain)
+
+	return &testWorkerBackend{
+		db:     db,
+		chain:  chain,
+		txPool: txpool,
+	}
+}
+
+func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain }
+func (b *testWorkerBackend) TxPool() *core.TxPool         { return b.txPool }
+func (b *testWorkerBackend) PostChainEvents(events []interface{}) {
+	b.chain.PostChainEvents(events, nil)
+}
+
+func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) {
+	backend := newTestWorkerBackend(t, chainConfig, engine)
+	backend.txPool.AddLocals(pendingTxs)
+	w := newWorker(chainConfig, engine, backend, new(event.TypeMux))
+	w.setEtherbase(testBankAddress)
+	return w, backend
+}
+
+func TestPendingStateAndBlockEthash(t *testing.T) {
+	testPendingStateAndBlock(t, ethashChainConfig, ethash.NewFaker())
+}
+func TestPendingStateAndBlockClique(t *testing.T) {
+	testPendingStateAndBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase()))
+}
+
+func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) {
+	defer engine.Close()
+
+	w, b := newTestWorker(t, chainConfig, engine)
+	defer w.close()
+
+	// Ensure snapshot has been updated.
+	time.Sleep(100 * time.Millisecond)
+	block, state := w.pending()
+	if block.NumberU64() != 1 {
+		t.Errorf("block number mismatch, has %d, want %d", block.NumberU64(), 1)
+	}
+	if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(1000)) != 0 {
+		t.Errorf("account balance mismatch, has %d, want %d", balance, 1000)
+	}
+	b.txPool.AddLocals(newTxs)
+	// Ensure the new tx events has been processed
+	time.Sleep(100 * time.Millisecond)
+	block, state = w.pending()
+	if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(2000)) != 0 {
+		t.Errorf("account balance mismatch, has %d, want %d", balance, 2000)
+	}
+}
+
+func TestEmptyWorkEthash(t *testing.T) {
+	testEmptyWork(t, ethashChainConfig, ethash.NewFaker())
+}
+func TestEmptyWorkClique(t *testing.T) {
+	testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase()))
+}
+
+func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) {
+	defer engine.Close()
+
+	w, _ := newTestWorker(t, chainConfig, engine)
+	defer w.close()
+
+	var (
+		taskCh    = make(chan struct{}, 2)
+		taskIndex int
+	)
+
+	checkEqual := func(t *testing.T, task *task, index int) {
+		receiptLen, balance := 0, big.NewInt(0)
+		if index == 1 {
+			receiptLen, balance = 1, big.NewInt(1000)
+		}
+		if len(task.receipts) != receiptLen {
+			t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen)
+		}
+		if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 {
+			t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance)
+		}
+	}
+
+	w.newTaskHook = func(task *task) {
+		if task.block.NumberU64() == 1 {
+			checkEqual(t, task, taskIndex)
+			taskIndex += 1
+			taskCh <- struct{}{}
+		}
+	}
+	w.fullTaskInterval = func() {
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// Ensure worker has finished initialization
+	for {
+		b := w.pendingBlock()
+		if b != nil && b.NumberU64() == 1 {
+			break
+		}
+	}
+
+	w.start()
+	for i := 0; i < 2; i += 1 {
+		to := time.NewTimer(time.Second)
+		select {
+		case <-taskCh:
+		case <-to.C:
+			t.Error("new task timeout")
+		}
+	}
+}

node/api.go

@@ -59,7 +59,7 @@ func (api *PrivateAdminAPI) AddPeer(url string) (bool, error) {
 	return true, nil
 }
 
-// RemovePeer disconnects from a a remote node if the connection exists
+// RemovePeer disconnects from a remote node if the connection exists
 func (api *PrivateAdminAPI) RemovePeer(url string) (bool, error) {
 	// Make sure the server is running, fail otherwise
 	server := api.node.Server()
@@ -75,6 +75,37 @@ func (api *PrivateAdminAPI) RemovePeer(url string) (bool, error) {
 	return true, nil
 }
 
+// AddTrustedPeer allows a remote node to always connect, even if slots are full
+func (api *PrivateAdminAPI) AddTrustedPeer(url string) (bool, error) {
+	// Make sure the server is running, fail otherwise
+	server := api.node.Server()
+	if server == nil {
+		return false, ErrNodeStopped
+	}
+	node, err := discover.ParseNode(url)
+	if err != nil {
+		return false, fmt.Errorf("invalid enode: %v", err)
+	}
+	server.AddTrustedPeer(node)
+	return true, nil
+}
+
+// RemoveTrustedPeer removes a remote node from the trusted peer set, but it
+// does not disconnect it automatically.
+func (api *PrivateAdminAPI) RemoveTrustedPeer(url string) (bool, error) {
+	// Make sure the server is running, fail otherwise
+	server := api.node.Server()
+	if server == nil {
+		return false, ErrNodeStopped
+	}
+	node, err := discover.ParseNode(url)
+	if err != nil {
+		return false, fmt.Errorf("invalid enode: %v", err)
+	}
+	server.RemoveTrustedPeer(node)
+	return true, nil
+}
+
 // PeerEvents creates an RPC subscription which receives peer events from the
 // node's p2p.Server
 func (api *PrivateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) {

p2p/discover/table.go

@@ -160,7 +160,7 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) {
 
 	// Find all non-empty buckets and get a fresh slice of their entries.
 	var buckets [][]*Node
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		if len(b.entries) > 0 {
 			buckets = append(buckets, b.entries[:])
 		}
@@ -508,7 +508,7 @@ func (tab *Table) copyLiveNodes() {
 	defer tab.mutex.Unlock()
 
 	now := time.Now()
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		for _, n := range b.entries {
 			if now.Sub(n.addedAt) >= seedMinTableTime {
 				tab.db.updateNode(n)
@@ -524,7 +524,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
 	// obviously correct. I believe that tree-based buckets would make
 	// this easier to implement efficiently.
 	close := &nodesByDistance{target: target}
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		for _, n := range b.entries {
 			close.push(n, nresults)
 		}
@@ -533,7 +533,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
 }
 
 func (tab *Table) len() (n int) {
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		n += len(b.entries)
 	}
 	return n

p2p/discv5/net.go

@@ -678,7 +678,7 @@ func (net *Network) refresh(done chan<- struct{}) {
 	}
 	if len(seeds) == 0 {
 		log.Trace("no seed nodes found")
-		close(done)
+		time.AfterFunc(time.Second*10, func() { close(done) })
 		return
 	}
 	for _, n := range seeds {
@@ -1228,7 +1228,7 @@ func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) {
 	if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash {
 		return nil, errors.New("topic hash mismatch")
 	}
-	if data.Idx < 0 || int(data.Idx) >= len(data.Topics) {
+	if int(data.Idx) < 0 || int(data.Idx) >= len(data.Topics) {
 		return nil, errors.New("topic index out of range")
 	}
 	return pongpkt.data.(*pong), nil

p2p/discv5/net_test.go

@@ -355,7 +355,7 @@ func (tn *preminedTestnet) mine(target NodeID) {
 	fmt.Printf("	target: %#v,\n", tn.target)
 	fmt.Printf("	targetSha: %#v,\n", tn.targetSha)
 	fmt.Printf("	dists: [%d][]NodeID{\n", len(tn.dists))
-	for ld, ns := range tn.dists {
+	for ld, ns := range &tn.dists {
 		if len(ns) == 0 {
 			continue
 		}

p2p/discv5/table.go

@@ -81,7 +81,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash {
 	if printTable {
 		fmt.Println()
 	}
-	for i, b := range tab.buckets {
+	for i, b := range &tab.buckets {
 		entries += len(b.entries)
 		if printTable {
 			for _, e := range b.entries {
@@ -93,7 +93,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash {
 	prefix := binary.BigEndian.Uint64(tab.self.sha[0:8])
 	dist := ^uint64(0)
 	entry := int(randUint(uint32(entries + 1)))
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		if entry < len(b.entries) {
 			n := b.entries[entry]
 			dist = binary.BigEndian.Uint64(n.sha[0:8]) ^ prefix
@@ -121,7 +121,7 @@ func (tab *Table) readRandomNodes(buf []*Node) (n int) {
 	// TODO: tree-based buckets would help here
 	// Find all non-empty buckets and get a fresh slice of their entries.
 	var buckets [][]*Node
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		if len(b.entries) > 0 {
 			buckets = append(buckets, b.entries[:])
 		}
@@ -175,7 +175,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
 	// obviously correct. I believe that tree-based buckets would make
 	// this easier to implement efficiently.
 	close := &nodesByDistance{target: target}
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		for _, n := range b.entries {
 			close.push(n, nresults)
 		}

p2p/peer.go

@@ -165,7 +165,7 @@ func (p *Peer) String() string {
 
 // Inbound returns true if the peer is an inbound connection
 func (p *Peer) Inbound() bool {
-	return p.rw.flags&inboundConn != 0
+	return p.rw.is(inboundConn)
 }
 
 func newPeer(conn *conn, protocols []Protocol) *Peer {

p2p/server.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -169,6 +170,8 @@ type Server struct {
 	quit          chan struct{}
 	addstatic     chan *discover.Node
 	removestatic  chan *discover.Node
+	addtrusted    chan *discover.Node
+	removetrusted chan *discover.Node
 	posthandshake chan *conn
 	addpeer       chan *conn
 	delpeer       chan peerDrop
@@ -185,7 +188,7 @@ type peerDrop struct {
 	requested bool // true if signaled by the peer
 }
 
-type connFlag int
+type connFlag int32
 
 const (
 	dynDialedConn connFlag = 1 << iota
@@ -250,7 +253,23 @@ func (f connFlag) String() string {
 }
 
 func (c *conn) is(f connFlag) bool {
-	return c.flags&f != 0
+	flags := connFlag(atomic.LoadInt32((*int32)(&c.flags)))
+	return flags&f != 0
+}
+
+func (c *conn) set(f connFlag, val bool) {
+	for {
+		oldFlags := connFlag(atomic.LoadInt32((*int32)(&c.flags)))
+		flags := oldFlags
+		if val {
+			flags |= f
+		} else {
+			flags &= ^f
+		}
+		if atomic.CompareAndSwapInt32((*int32)(&c.flags), int32(oldFlags), int32(flags)) {
+			return
+		}
+	}
 }
 
 // Peers returns all connected peers.
@@ -300,6 +319,23 @@ func (srv *Server) RemovePeer(node *discover.Node) {
 	}
 }
 
+// AddTrustedPeer adds the given node to a reserved whitelist which allows the
+// node to always connect, even if the slot are full.
+func (srv *Server) AddTrustedPeer(node *discover.Node) {
+	select {
+	case srv.addtrusted <- node:
+	case <-srv.quit:
+	}
+}
+
+// RemoveTrustedPeer removes the given node from the trusted peer set.
+func (srv *Server) RemoveTrustedPeer(node *discover.Node) {
+	select {
+	case srv.removetrusted <- node:
+	case <-srv.quit:
+	}
+}
+
 // SubscribePeers subscribes the given channel to peer events
 func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription {
 	return srv.peerFeed.Subscribe(ch)
@@ -411,6 +447,8 @@ func (srv *Server) Start() (err error) {
 	srv.posthandshake = make(chan *conn)
 	srv.addstatic = make(chan *discover.Node)
 	srv.removestatic = make(chan *discover.Node)
+	srv.addtrusted = make(chan *discover.Node)
+	srv.removetrusted = make(chan *discover.Node)
 	srv.peerOp = make(chan peerOpFunc)
 	srv.peerOpDone = make(chan struct{})
 
@@ -547,8 +585,7 @@ func (srv *Server) run(dialstate dialer) {
 		queuedTasks  []task // tasks that can't run yet
 	)
 	// Put trusted nodes into a map to speed up checks.
-	// Trusted peers are loaded on startup and cannot be
+	// Trusted peers are loaded on startup or added via AddTrustedPeer RPC.
-	// modified while the server is running.
 	for _, n := range srv.TrustedNodes {
 		trusted[n.ID] = true
 	}
@@ -600,12 +637,32 @@ running:
 		case n := <-srv.removestatic:
 			// This channel is used by RemovePeer to send a
 			// disconnect request to a peer and begin the
-			// stop keeping the node connected
+			// stop keeping the node connected.
 			srv.log.Trace("Removing static node", "node", n)
 			dialstate.removeStatic(n)
 			if p, ok := peers[n.ID]; ok {
 				p.Disconnect(DiscRequested)
 			}
+		case n := <-srv.addtrusted:
+			// This channel is used by AddTrustedPeer to add an enode
+			// to the trusted node set.
+			srv.log.Trace("Adding trusted node", "node", n)
+			trusted[n.ID] = true
+			// Mark any already-connected peer as trusted
+			if p, ok := peers[n.ID]; ok {
+				p.rw.set(trustedConn, true)
+			}
+		case n := <-srv.removetrusted:
+			// This channel is used by RemoveTrustedPeer to remove an enode
+			// from the trusted node set.
+			srv.log.Trace("Removing trusted node", "node", n)
+			if _, ok := trusted[n.ID]; ok {
+				delete(trusted, n.ID)
+			}
+			// Unmark any already-connected peer as trusted
+			if p, ok := peers[n.ID]; ok {
+				p.rw.set(trustedConn, false)
+			}
 		case op := <-srv.peerOp:
 			// This channel is used by Peers and PeerCount.
 			op(peers)

p2p/server_test.go

@@ -148,7 +148,8 @@ func TestServerDial(t *testing.T) {
 
 	// tell the server to connect
 	tcpAddr := listener.Addr().(*net.TCPAddr)
-	srv.AddPeer(&discover.Node{ID: remid, IP: tcpAddr.IP, TCP: uint16(tcpAddr.Port)})
+	node := &discover.Node{ID: remid, IP: tcpAddr.IP, TCP: uint16(tcpAddr.Port)}
+	srv.AddPeer(node)
 
 	select {
 	case conn := <-accepted:
@@ -170,6 +171,29 @@ func TestServerDial(t *testing.T) {
 			if !reflect.DeepEqual(peers, []*Peer{peer}) {
 				t.Errorf("Peers mismatch: got %v, want %v", peers, []*Peer{peer})
 			}
+
+			// Test AddTrustedPeer/RemoveTrustedPeer and changing Trusted flags
+			// Particularly for race conditions on changing the flag state.
+			if peer := srv.Peers()[0]; peer.Info().Network.Trusted {
+				t.Errorf("peer is trusted prematurely: %v", peer)
+			}
+			done := make(chan bool)
+			go func() {
+				srv.AddTrustedPeer(node)
+				if peer := srv.Peers()[0]; !peer.Info().Network.Trusted {
+					t.Errorf("peer is not trusted after AddTrustedPeer: %v", peer)
+				}
+				srv.RemoveTrustedPeer(node)
+				if peer := srv.Peers()[0]; peer.Info().Network.Trusted {
+					t.Errorf("peer is trusted after RemoveTrustedPeer: %v", peer)
+				}
+				done <- true
+			}()
+			// Trigger potential race conditions
+			peer = srv.Peers()[0]
+			_ = peer.Inbound()
+			_ = peer.Info()
+			<-done
 		case <-time.After(1 * time.Second):
 			t.Error("server did not launch peer within one second")
 		}
@@ -351,7 +375,8 @@ func TestServerAtCap(t *testing.T) {
 		}
 	}
 	// Try inserting a non-trusted connection.
-	c := newconn(randomID())
+	anotherID := randomID()
+	c := newconn(anotherID)
 	if err := srv.checkpoint(c, srv.posthandshake); err != DiscTooManyPeers {
 		t.Error("wrong error for insert:", err)
 	}
@@ -364,6 +389,87 @@ func TestServerAtCap(t *testing.T) {
 		t.Error("Server did not set trusted flag")
 	}
 
+	// Remove from trusted set and try again
+	srv.RemoveTrustedPeer(&discover.Node{ID: trustedID})
+	c = newconn(trustedID)
+	if err := srv.checkpoint(c, srv.posthandshake); err != DiscTooManyPeers {
+		t.Error("wrong error for insert:", err)
+	}
+
+	// Add anotherID to trusted set and try again
+	srv.AddTrustedPeer(&discover.Node{ID: anotherID})
+	c = newconn(anotherID)
+	if err := srv.checkpoint(c, srv.posthandshake); err != nil {
+		t.Error("unexpected error for trusted conn @posthandshake:", err)
+	}
+	if !c.is(trustedConn) {
+		t.Error("Server did not set trusted flag")
+	}
+}
+
+func TestServerPeerLimits(t *testing.T) {
+	srvkey := newkey()
+
+	clientid := randomID()
+	clientnode := &discover.Node{ID: clientid}
+
+	var tp *setupTransport = &setupTransport{
+		id: clientid,
+		phs: &protoHandshake{
+			ID: clientid,
+			// Force "DiscUselessPeer" due to unmatching caps
+			// Caps: []Cap{discard.cap()},
+		},
+	}
+	var flags connFlag = dynDialedConn
+	var dialDest *discover.Node = &discover.Node{ID: clientid}
+
+	srv := &Server{
+		Config: Config{
+			PrivateKey: srvkey,
+			MaxPeers:   0,
+			NoDial:     true,
+			Protocols:  []Protocol{discard},
+		},
+		newTransport: func(fd net.Conn) transport { return tp },
+		log:          log.New(),
+	}
+	if err := srv.Start(); err != nil {
+		t.Fatalf("couldn't start server: %v", err)
+	}
+	defer srv.Stop()
+
+	// Check that server is full (MaxPeers=0)
+	conn, _ := net.Pipe()
+	srv.SetupConn(conn, flags, dialDest)
+	if tp.closeErr != DiscTooManyPeers {
+		t.Errorf("unexpected close error: %q", tp.closeErr)
+	}
+	conn.Close()
+
+	srv.AddTrustedPeer(clientnode)
+
+	// Check that server allows a trusted peer despite being full.
+	conn, _ = net.Pipe()
+	srv.SetupConn(conn, flags, dialDest)
+	if tp.closeErr == DiscTooManyPeers {
+		t.Errorf("failed to bypass MaxPeers with trusted node: %q", tp.closeErr)
+	}
+
+	if tp.closeErr != DiscUselessPeer {
+		t.Errorf("unexpected close error: %q", tp.closeErr)
+	}
+	conn.Close()
+
+	srv.RemoveTrustedPeer(clientnode)
+
+	// Check that server is full again.
+	conn, _ = net.Pipe()
+	srv.SetupConn(conn, flags, dialDest)
+	if tp.closeErr != DiscTooManyPeers {
+		t.Errorf("unexpected close error: %q", tp.closeErr)
+	}
+	conn.Close()
 }
 
 func TestServerSetupConn(t *testing.T) {

rpc/client_example_test.go

@@ -66,7 +66,7 @@ func subscribeBlocks(client *rpc.Client, subch chan Block) {
 	defer cancel()
 
 	// Subscribe to new blocks.
-	sub, err := client.EthSubscribe(ctx, subch, "newBlocks")
+	sub, err := client.EthSubscribe(ctx, subch, "newHeads")
 	if err != nil {
 		fmt.Println("subscribe error:", err)
 		return

swarm/README.md

@@ -7,6 +7,21 @@ Swarm is a distributed storage platform and content distribution service, a nati
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethersphere/orange-lounge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
+## Table of Contents
+
+*  [Building the source](#building-the-source)
+*  [Running Swarm](#running-swarm)
+*  [Documentation](#documentation)
+*  [Developers Guide](#developers-guide)
+   *  [Go Environment](#development-environment)
+   *  [Vendored Dependencies](#vendored-dependencies)
+   *  [Testing](#testing)
+   *  [Profiling Swarm](#profiling-swarm)
+   *  [Metrics and Instrumentation in Swarm](#metrics-and-instrumentation-in-swarm)
+*  [Public Gateways](#public-gateways)
+*  [Swarm Dapps](#swarm-dapps)
+*  [Contributing](#contributing)
+*  [License](#license)
 
 ## Building the source
 
@@ -16,13 +31,187 @@ Building Swarm requires Go (version 1.10 or later).
 
     go install github.com/ethereum/go-ethereum/cmd/swarm
 
+## Running Swarm
+
+Going through all the possible command line flags is out of scope here, but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own Swarm node.
+
+To run Swarm you need an Ethereum account. You can create a new account by running the following command:
+
+    geth account new
+
+You will be prompted for a password:
+
+    Your new account is locked with a password. Please give a password. Do not forget this password.
+    Passphrase:
+    Repeat passphrase:
+
+Once you have specified the password, the output will be the Ethereum address representing that account. For example:
+
+    Address: {2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1}
+
+Using this account, connect to Swarm with
+
+    swarm --bzzaccount <your-account-here>
+
+    # in our example
+
+    swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1
+
+
+### Verifying that your local Swarm node is running
+
+When running, Swarm is accessible through an HTTP API on port 8500.
+
+Confirm that it is up and running by pointing your browser to http://localhost:8500
+
+### Ethereum Name Service resolution
+
+The Ethereum Name Service is the Ethereum equivalent of DNS in the classic web. In order to use ENS to resolve names to Swarm content hashes (e.g. `bzz://theswarm.eth`), `swarm` has to connect to a `geth` instance, which is synced with the Ethereum mainnet. This is done using the `--ens-api` flag.
+
+    swarm --bzzaccount <your-account-here> \
+          --ens-api '$HOME/.ethereum/geth.ipc'
+
+    # in our example
+
+    swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 \
+          --ens-api '$HOME/.ethereum/geth.ipc'
+
+For more information on usage, features or command line flags, please consult the Documentation.
+
 
 ## Documentation
 
 Swarm documentation can be found at [https://swarm-guide.readthedocs.io](https://swarm-guide.readthedocs.io).
 
 
-## Contribution
+## Developers Guide
+
+### Go Environment
+
+We assume that you have Go v1.10 installed, and `GOPATH` is set.
+
+You must have your working copy under `$GOPATH/src/github.com/ethereum/go-ethereum`.
+
+Most likely you will be working from your fork of `go-ethereum`, let's say from `github.com/nirname/go-ethereum`. Clone or move your fork into the right place:
+
+```
+git clone git@github.com:nirname/go-ethereum.git $GOPATH/src/github.com/ethereum/go-ethereum
+```
+
+
+### Vendored Dependencies
+
+All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them.
+
+If you want to add a new dependency, run `govendor fetch <import-path>`, then commit the result.
+
+If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`.
+
+
+### Testing
+
+This section explains how to run unit, integration, and end-to-end tests in your development sandbox.
+
+Testing one library:
+
+```
+go test -v -cpu 4 ./swarm/api
+```
+
+Note: Using options -cpu (number of cores allowed) and -v (logging even if no error) is recommended.
+
+Testing only some methods:
+
+```
+go test -v -cpu 4 ./eth -run TestMethod
+```
+
+Note: here all tests with prefix TestMethod will be run, so if you got TestMethod, TestMethod1, then both!
+
+Running benchmarks:
+
+```
+go test -v -cpu 4 -bench . -run BenchmarkJoin
+```
+
+
+### Profiling Swarm
+
+This section explains how to add Go `pprof` profiler to Swarm
+
+If `swarm` is started with the `--pprof` option, a debugging HTTP server is made available on port 6060.
+
+You can bring up http://localhost:6060/debug/pprof to see the heap, running routines etc.
+
+By clicking full goroutine stack dump (clicking http://localhost:6060/debug/pprof/goroutine?debug=2) you can generate trace that is useful for debugging.
+
+
+### Metrics and Instrumentation in Swarm
+
+This section explains how to visualize and use existing Swarm metrics and how to instrument Swarm with a new metric.
+
+Swarm metrics system is based on the `go-metrics` library.
+
+The most common types of measurements we use in Swarm are `counters` and `resetting timers`. Consult the `go-metrics` documentation for full reference of available types.
+
+```
+# incrementing a counter
+metrics.GetOrRegisterCounter("network.stream.received_chunks", nil).Inc(1)
+
+# measuring latency with a resetting timer
+start := time.Now()
+t := metrics.GetOrRegisterResettingTimer("http.request.GET.time"), nil)
+...
+t := UpdateSince(start)
+```
+
+#### Visualizing metrics
+
+Swarm supports an InfluxDB exporter. Consult the help section to learn about the command line arguments used to configure it:
+
+```
+swarm --help | grep metrics
+```
+
+We use Grafana and InfluxDB to visualise metrics reported by Swarm. We keep our Grafana dashboards under version control at `./swarm/grafana_dashboards`. You could use them or design your own.
+
+We have built a tool to help with automatic start of Grafana and InfluxDB and provisioning of dashboards at https://github.com/nonsense/stateth , which requires that you have Docker installed.
+
+Once you have `stateth` installed, and you have Docker running locally, you have to:
+
+1. Run `stateth` and keep it running in the background
+```
+stateth --rm --grafana-dashboards-folder $GOPATH/src/github.com/ethereum/go-ethereum/swarm/grafana_dashboards --influxdb-database metrics
+```
+
+2. Run `swarm` with at least the following params:
+```
+--metrics \
+--metrics.influxdb.export \
+--metrics.influxdb.endpoint "http://localhost:8086" \
+--metrics.influxdb.username "admin" \
+--metrics.influxdb.password "admin" \
+--metrics.influxdb.database "metrics"
+```
+
+3. Open Grafana at http://localhost:3000 and view the dashboards to gain insight into Swarm.
+
+
+## Public Gateways
+
+Swarm offers a local HTTP proxy API that Dapps can use to interact with Swarm. The Ethereum Foundation is hosting a public gateway, which allows free access so that people can try Swarm without running their own node.
+
+The Swarm public gateways are temporary and users should not rely on their existence for production services.
+
+The Swarm public gateway can be found at https://swarm-gateways.net and is always running the latest `stable` Swarm release.
+
+## Swarm Dapps
+
+You can find a few reference Swarm decentralised applications at: https://swarm-gateways.net/bzz:/swarmapps.eth
+
+Their source code can be found at: https://github.com/ethersphere/swarm-dapps
+
+## Contributing
 
 Thank you for considering to help out with the source code! We welcome contributions from
 anyone on the internet, and are grateful for even the smallest of fixes!

swarm/api/api.go

@@ -339,8 +339,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
 	if err != nil {
 		apiGetNotFound.Inc(1)
 		status = http.StatusNotFound
-		log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
+		return nil, "", http.StatusNotFound, nil, err
-		return
 	}
 
 	log.Debug("trie getting entry", "key", manifestAddr, "path", path)
@@ -526,6 +525,10 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err
 
 			return nil
 		})
+		// close tar writer before closing pipew
+		// to flush remaining data to pipew
+		// regardless of error value
+		tw.Close()
 		if err != nil {
 			apiGetTarFail.Inc(1)
 			pipew.CloseWithError(err)
@@ -701,11 +704,12 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []
 	return fkey, newMkey.String(), nil
 }
 
-func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) {
+func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath, defaultPath string, mw *ManifestWriter) (storage.Address, error) {
 	apiUploadTarCount.Inc(1)
 	var contentKey storage.Address
 	tr := tar.NewReader(bodyReader)
 	defer bodyReader.Close()
+	var defaultPathFound bool
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
@@ -734,6 +738,25 @@ func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestP
 			apiUploadTarFail.Inc(1)
 			return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err)
 		}
+		if hdr.Name == defaultPath {
+			entry := &ManifestEntry{
+				Hash:        contentKey.Hex(),
+				Path:        "", // default entry
+				ContentType: hdr.Xattrs["user.swarm.content-type"],
+				Mode:        hdr.Mode,
+				Size:        hdr.Size,
+				ModTime:     hdr.ModTime,
+			}
+			contentKey, err = mw.AddEntry(ctx, nil, entry)
+			if err != nil {
+				apiUploadTarFail.Inc(1)
+				return nil, fmt.Errorf("error adding default manifest entry from tar stream: %s", err)
+			}
+			defaultPathFound = true
+		}
+	}
+	if defaultPath != "" && !defaultPathFound {
+		return contentKey, fmt.Errorf("default path %q not found", defaultPath)
 	}
 	return contentKey, nil
 }

swarm/api/client/client.go

@@ -138,7 +138,7 @@ func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, er
 	if file.Size <= 0 {
 		return "", errors.New("file size must be greater than zero")
 	}
-	return c.TarUpload(manifest, &FileUploader{file}, toEncrypt)
+	return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt)
 }
 
 // Download downloads a file with the given path from the swarm manifest with
@@ -175,7 +175,15 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo
 	} else if !stat.IsDir() {
 		return "", fmt.Errorf("not a directory: %s", dir)
 	}
-	return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt)
+	if defaultPath != "" {
+		if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil {
+			if os.IsNotExist(err) {
+				return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir)
+			}
+			return "", fmt.Errorf("default path: %v", err)
+		}
+	}
+	return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt)
 }
 
 // DownloadDirectory downloads the files contained in a swarm manifest under
@@ -390,20 +398,10 @@ func (u UploaderFunc) Upload(upload UploadFn) error {
 // a file to the default path
 type DirectoryUploader struct {
 	Dir string
-	DefaultPath string
 }
 
 // Upload performs the upload of the directory and default path
 func (d *DirectoryUploader) Upload(upload UploadFn) error {
-	if d.DefaultPath != "" {
-		file, err := Open(d.DefaultPath)
-		if err != nil {
-			return err
-		}
-		if err := upload(file); err != nil {
-			return err
-		}
-	}
 	return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -441,7 +439,7 @@ type UploadFn func(file *File) error
 
 // TarUpload uses the given Uploader to upload files to swarm as a tar stream,
 // returning the resulting manifest hash
-func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) {
+func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) {
 	reqR, reqW := io.Pipe()
 	defer reqR.Close()
 	addr := hash
@@ -458,6 +456,11 @@ func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (stri
 		return "", err
 	}
 	req.Header.Set("Content-Type", "application/x-tar")
+	if defaultPath != "" {
+		q := req.URL.Query()
+		q.Set("defaultpath", defaultPath)
+		req.URL.RawQuery = q.Encode()
+	}
 
 	// use 'Expect: 100-continue' so we don't send the request body if
 	// the server refuses the request

swarm/api/client/client_test.go

@@ -194,7 +194,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
 
 	// upload the directory
 	client := NewClient(srv.URL)
-	defaultPath := filepath.Join(dir, testDirFiles[0])
+	defaultPath := testDirFiles[0]
 	hash, err := client.UploadDirectory(dir, defaultPath, "", false)
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)

swarm/api/config.go

@@ -63,6 +63,7 @@ type Config struct {
 	SwapEnabled       bool
 	SyncEnabled       bool
 	DeliverySkipCheck bool
+	LightNodeEnabled  bool
 	SyncUpdateDelay   time.Duration
 	SwapAPI           string
 	Cors              string

swarm/api/http/error.go

@@ -1,208 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-/*
-Show nicely (but simple) formatted HTML error pages (or respond with JSON
-if the appropriate `Accept` header is set)) for the http package.
-*/
-package http
-
-import (
-	"encoding/json"
-	"fmt"
-	"html/template"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/ethereum/go-ethereum/swarm/api"
-	l "github.com/ethereum/go-ethereum/swarm/log"
-)
-
-//templateMap holds a mapping of an HTTP error code to a template
-var templateMap map[int]*template.Template
-var caseErrors []CaseError
-
-//metrics variables
-var (
-	htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
-	jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
-)
-
-//parameters needed for formatting the correct HTML page
-type ResponseParams struct {
-	Msg       string
-	Code      int
-	Timestamp string
-	template  *template.Template
-	Details   template.HTML
-}
-
-//a custom error case struct that would be used to store validators and
-//additional error info to display with client responses.
-type CaseError struct {
-	Validator func(*Request) bool
-	Msg       func(*Request) string
-}
-
-//we init the error handling right on boot time, so lookup and http response is fast
-func init() {
-	initErrHandling()
-}
-
-func initErrHandling() {
-	//pages are saved as strings - get these strings
-	genErrPage := GetGenericErrorPage()
-	notFoundPage := GetNotFoundErrorPage()
-	multipleChoicesPage := GetMultipleChoicesErrorPage()
-	//map the codes to the available pages
-	tnames := map[int]string{
-		0: genErrPage, //default
-		http.StatusBadRequest:          genErrPage,
-		http.StatusNotFound:            notFoundPage,
-		http.StatusMultipleChoices:     multipleChoicesPage,
-		http.StatusInternalServerError: genErrPage,
-	}
-	templateMap = make(map[int]*template.Template)
-	for code, tname := range tnames {
-		//assign formatted HTML to the code
-		templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname))
-	}
-
-	caseErrors = []CaseError{
-		{
-			Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") },
-			Msg: func(r *Request) string {
-				uriCopy := r.uri
-				uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x")
-				return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
-			Please click <a href='%[1]s'>here</a> if your browser does not redirect you.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uriCopy.String())
-			},
-		}}
-}
-
-//ValidateCaseErrors is a method that process the request object through certain validators
-//that assert if certain conditions are met for further information to log as an error
-func ValidateCaseErrors(r *Request) string {
-	for _, err := range caseErrors {
-		if err.Validator(r) {
-			return err.Msg(r)
-		}
-	}
-
-	return ""
-}
-
-//ShowMultipeChoices is used when a user requests a resource in a manifest which results
-//in ambiguous results. It returns a HTML page with clickable links of each of the entry
-//in the manifest which fits the request URI ambiguity.
-//For example, if the user requests bzz:/<hash>/read and that manifest contains entries
-//"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
-//This only applies if the manifest has no default entry
-func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) {
-	msg := ""
-	if list.Entries == nil {
-		Respond(w, req, "Could not resolve", http.StatusInternalServerError)
-		return
-	}
-	//make links relative
-	//requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt"
-	//to get clickable links, need to remove the ambiguous path, i.e. "read"
-	idx := strings.LastIndex(req.RequestURI, "/")
-	if idx == -1 {
-		Respond(w, req, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-	//remove ambiguous part
-	base := req.RequestURI[:idx+1]
-	for _, e := range list.Entries {
-		//create clickable link for each entry
-		msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
-	}
-	Respond(w, req, msg, http.StatusMultipleChoices)
-}
-
-//Respond is used to show an HTML page to a client.
-//If there is an `Accept` header of `application/json`, JSON will be returned instead
-//The function just takes a string message which will be displayed in the error page.
-//The code is used to evaluate which template will be displayed
-//(and return the correct HTTP status code)
-func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
-	additionalMessage := ValidateCaseErrors(req)
-	switch code {
-	case http.StatusInternalServerError:
-		log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code)
-	case http.StatusMultipleChoices:
-		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
-		listURI := api.URI{
-			Scheme: "bzz-list",
-			Addr:   req.uri.Addr,
-			Path:   req.uri.Path,
-		}
-		additionalMessage = fmt.Sprintf(`<a href="/%s">multiple choices</a>`, listURI.String())
-	default:
-		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
-	}
-
-	if code >= 400 {
-		w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
-		w.Header().Del("ETag")
-	}
-
-	respond(w, &req.Request, &ResponseParams{
-		Code:      code,
-		Msg:       msg,
-		Details:   template.HTML(additionalMessage),
-		Timestamp: time.Now().Format(time.RFC1123),
-		template:  getTemplate(code),
-	})
-}
-
-//evaluate if client accepts html or json response
-func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
-	w.WriteHeader(params.Code)
-	if r.Header.Get("Accept") == "application/json" {
-		respondJSON(w, params)
-	} else {
-		respondHTML(w, params)
-	}
-}
-
-//return a HTML page
-func respondHTML(w http.ResponseWriter, params *ResponseParams) {
-	htmlCounter.Inc(1)
-	err := params.template.Execute(w, params)
-	if err != nil {
-		log.Error(err.Error())
-	}
-}
-
-//return JSON
-func respondJSON(w http.ResponseWriter, params *ResponseParams) {
-	jsonCounter.Inc(1)
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(params)
-}
-
-//get the HTML template for a given code
-func getTemplate(code int) *template.Template {
-	if val, tmpl := templateMap[code]; tmpl {
-		return val
-	}
-	return templateMap[0]
-}

swarm/api/http/middleware.go

@@ -0,0 +1,95 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+	"runtime/debug"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/swarm/spancontext"
+	"github.com/pborman/uuid"
+)
+
+// Adapt chains h (main request handler) main handler to adapters (middleware handlers)
+// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first)
+func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
+	for i := range adapters {
+		adapter := adapters[len(adapters)-1-i]
+		h = adapter(h)
+	}
+	return h
+}
+
+type Adapter func(http.Handler) http.Handler
+
+func SetRequestID(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8]))
+		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
+		log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func ParseURI(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
+			return
+		}
+		if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") {
+			uri.Addr = strings.TrimPrefix(uri.Addr, "0x")
+
+			msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
+			Please click <a href='%[1]s'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uri.String())
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(msg))
+			return
+		}
+
+		ctx := r.Context()
+		r = r.WithContext(SetURI(ctx, uri))
+		log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func InitLoggingResponseWriter(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		writer := newLoggingResponseWriter(w)
+		h.ServeHTTP(writer, r)
+		log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
+	})
+}
+
+func InstrumentOpenTracing(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri := GetURI(r.Context())
+		if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
+			h.ServeHTTP(w, r) // soft fail
+			return
+		}
+		spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme)
+		ctx, sp := spancontext.StartSpan(r.Context(), spanName)
+		defer sp.Finish()
+		h.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func RecoverPanic(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header)
+			}
+		}()
+		h.ServeHTTP(w, r)
+	})
+}

swarm/api/http/response.go

@@ -0,0 +1,133 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+)
+
+var (
+	htmlCounter      = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
+	jsonCounter      = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
+	plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil)
+)
+
+type ResponseParams struct {
+	Msg       template.HTML
+	Code      int
+	Timestamp string
+	template  *template.Template
+	Details   template.HTML
+}
+
+// ShowMultipleChoices is used when a user requests a resource in a manifest which results
+// in ambiguous results. It returns a HTML page with clickable links of each of the entry
+// in the manifest which fits the request URI ambiguity.
+// For example, if the user requests bzz:/<hash>/read and that manifest contains entries
+// "readme.md" and "readinglist.txt", a HTML page is returned with this two links.
+// This only applies if the manifest has no default entry
+func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
+	log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	msg := ""
+	if list.Entries == nil {
+		RespondError(w, r, "Could not resolve", http.StatusInternalServerError)
+		return
+	}
+	requestUri := strings.TrimPrefix(r.RequestURI, "/")
+
+	uri, err := api.Parse(requestUri)
+	if err != nil {
+		RespondError(w, r, "Bad Request", http.StatusBadRequest)
+	}
+
+	uri.Scheme = "bzz-list"
+	msg += fmt.Sprintf("Disambiguation:<br/>Your request may refer to multiple choices.<br/>Click <a class=\"orange\" href='"+"/"+uri.String()+"'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout(\"location.href='%s';\",5000);</script><br/>", "/"+uri.String())
+	RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices)
+}
+
+func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) {
+	log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	respond(w, r, &ResponseParams{
+		Code:      code,
+		Msg:       template.HTML(msg),
+		Timestamp: time.Now().Format(time.RFC1123),
+		template:  TemplatesMap[templateName],
+	})
+}
+
+func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
+	log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	RespondTemplate(w, r, "error", msg, code)
+}
+
+func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
+
+	w.WriteHeader(params.Code)
+
+	if params.Code >= 400 {
+		w.Header().Del("Cache-Control")
+		w.Header().Del("ETag")
+	}
+
+	acceptHeader := r.Header.Get("Accept")
+	// this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8"
+	if strings.Contains(acceptHeader, "application/json") {
+		if err := respondJSON(w, r, params); err != nil {
+			RespondError(w, r, "Internal server error", http.StatusInternalServerError)
+		}
+	} else if strings.Contains(acceptHeader, "text/html") {
+		respondHTML(w, r, params)
+	} else {
+		respondPlaintext(w, r, params) //returns nice errors for curl
+	}
+}
+
+func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
+	htmlCounter.Inc(1)
+	log.Debug("respondHTML", "ruid", GetRUID(r.Context()))
+	err := params.template.Execute(w, params)
+	if err != nil {
+		log.Error(err.Error())
+	}
+}
+
+func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	jsonCounter.Inc(1)
+	log.Debug("respondJSON", "ruid", GetRUID(r.Context()))
+	w.Header().Set("Content-Type", "application/json")
+	return json.NewEncoder(w).Encode(params)
+}
+
+func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	plaintextCounter.Inc(1)
+	log.Debug("respondPlaintext", "ruid", GetRUID(r.Context()))
+	w.Header().Set("Content-Type", "text/plain")
+	strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n"
+	strToWrite += "Message: " + string(params.Msg) + "\n"
+	strToWrite += "Timestamp: " + params.Timestamp + "\n"
+	_, err := w.Write([]byte(strToWrite))
+	return err
+}

swarm/api/http/response_test.go

@@ -44,7 +44,7 @@ func TestError(t *testing.T) {
 	defer resp.Body.Close()
 	respbody, err = ioutil.ReadAll(resp.Body)
 
-	if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") {
+	if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") {
 		t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
 	}
 

swarm/api/http/sctx.go

@@ -0,0 +1,38 @@
+package http
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/sctx"
+)
+
+type contextKey int
+
+const (
+	uriKey contextKey = iota
+)
+
+func GetRUID(ctx context.Context) string {
+	v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string)
+	if ok {
+		return v
+	}
+	return "xxxxxxxx"
+}
+
+func SetRUID(ctx context.Context, ruid string) context.Context {
+	return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid)
+}
+
+func GetURI(ctx context.Context) *api.URI {
+	v, ok := ctx.Value(uriKey).(*api.URI)
+	if ok {
+		return v
+	}
+	return nil
+}
+
+func SetURI(ctx context.Context, uri *api.URI) context.Context {
+	return context.WithValue(ctx, uriKey, uri)
+}

swarm/api/http/server.go

@@ -41,12 +41,9 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/log"
-	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/mru"
-	opentracing "github.com/opentracing/opentracing-go"
 
-	"github.com/pborman/uuid"
 	"github.com/rs/cors"
 )
 
@@ -72,6 +69,17 @@ var (
 	getListFail     = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
 )
 
+type methodHandler map[string]http.Handler
+
+func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	v, ok := m[r.Method]
+	if ok {
+		v.ServeHTTP(rw, r)
+		return
+	}
+	rw.WriteHeader(http.StatusMethodNotAllowed)
+}
+
 func NewServer(api *api.API, corsString string) *Server {
 	var allowedOrigins []string
 	for _, domain := range strings.Split(corsString, ",") {
@@ -84,20 +92,79 @@ func NewServer(api *api.API, corsString string) *Server {
 		AllowedHeaders: []string{"*"},
 	})
 
-	mux := http.NewServeMux()
 	server := &Server{api: api}
-	mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz))
-	mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw))
-	mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable))
-	mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash))
-	mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList))
-	mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource))
 
-	mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths))
+	defaultMiddlewares := []Adapter{
-	mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths))
+		RecoverPanic,
-	mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths))
+		SetRequestID,
+		InitLoggingResponseWriter,
+		ParseURI,
+		InstrumentOpenTracing,
+	}
 
+	mux := http.NewServeMux()
+	mux.Handle("/bzz:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleBzzGet),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostFiles),
+			defaultMiddlewares...,
+		),
+		"DELETE": Adapt(
+			http.HandlerFunc(server.HandleDelete),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-raw:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostRaw),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-immutable:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-hash:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-list:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGetList),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-resource:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGetResource),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostResource),
+			defaultMiddlewares...,
+		),
+	})
+
+	mux.Handle("/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleRootPaths),
+			SetRequestID,
+			InitLoggingResponseWriter,
+		),
+	})
 	server.Handler = c.Handler(mux)
+
 	return server
 }
 
@@ -105,139 +172,6 @@ func (s *Server) ListenAndServe(addr string) error {
 	return http.ListenAndServe(addr, s)
 }
 
-func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		if r.RequestURI == "/" {
-			if strings.Contains(r.Header.Get("Accept"), "text/html") {
-				err := landingPageTemplate.Execute(w, nil)
-				if err != nil {
-					log.Error(fmt.Sprintf("error rendering landing page: %s", err))
-				}
-				return
-			}
-			if strings.Contains(r.Header.Get("Accept"), "application/json") {
-				w.Header().Set("Content-Type", "application/json")
-				w.WriteHeader(http.StatusOK)
-				json.NewEncoder(w).Encode("Welcome to Swarm!")
-				return
-			}
-		}
-
-		if r.URL.Path == "/robots.txt" {
-			w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
-			fmt.Fprintf(w, "User-agent: *\nDisallow: /")
-			return
-		}
-		Respond(w, r, "Bad Request", http.StatusBadRequest)
-	default:
-		Respond(w, r, "Not Found", http.StatusNotFound)
-	}
-}
-
-func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetBzz")
-		if r.Header.Get("Accept") == "application/x-tar" {
-			reader, err := s.api.GetDirectoryTar(r.Context(), r.uri)
-			if err != nil {
-				Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
-			}
-			defer reader.Close()
-
-			w.Header().Set("Content-Type", "application/x-tar")
-			w.WriteHeader(http.StatusOK)
-			io.Copy(w, reader)
-			return
-		}
-		s.HandleGetFile(w, r)
-	case http.MethodPost:
-		log.Debug("handlePostFiles")
-		s.HandlePostFiles(w, r)
-	case http.MethodDelete:
-		log.Debug("handleBzzDelete")
-		s.HandleDelete(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetRaw")
-		s.HandleGet(w, r)
-	case http.MethodPost:
-		log.Debug("handlePostRaw")
-		s.HandlePostRaw(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetHash")
-		s.HandleGetList(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetHash")
-		s.HandleGet(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetHash")
-		s.HandleGetList(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetResource")
-		s.HandleGetResource(w, r)
-	case http.MethodPost:
-		log.Debug("handlePostResource")
-		s.HandlePostResource(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc {
-	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-		defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
-		req := &Request{Request: *r, ruid: uuid.New()[:8]}
-		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
-		log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
-
-		// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
-		w := newLoggingResponseWriter(rw)
-		if parseBzzUri {
-			uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
-			if err != nil {
-				Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
-				return
-			}
-			req.uri = uri
-
-			log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
-		}
-
-		h(w, req) // call original
-		log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
-	})
-}
-
 // browser API for registering bzz url scheme handlers:
 // https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
 // electron (chromium) api for registering bzz url scheme handlers:
@@ -247,59 +181,81 @@ type Server struct {
 	api *api.API
 }
 
-// Request wraps http.Request and also includes the parsed bzz URI
+func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
-type Request struct {
+	log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()))
-	http.Request
+	if r.Header.Get("Accept") == "application/x-tar" {
+		uri := GetURI(r.Context())
+		reader, err := s.api.GetDirectoryTar(r.Context(), uri)
+		if err != nil {
+			RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
+		}
+		defer reader.Close()
 
-	uri  *api.URI
+		w.Header().Set("Content-Type", "application/x-tar")
-	ruid string // request unique id
+		w.WriteHeader(http.StatusOK)
+		io.Copy(w, reader)
+		return
+	}
+
+	s.HandleGetFile(w, r)
+}
+
+func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) {
+	switch r.RequestURI {
+	case "/":
+		RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200)
+		return
+	case "/robots.txt":
+		w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+		fmt.Fprintf(w, "User-agent: *\nDisallow: /")
+	case "/favicon.ico":
+		w.WriteHeader(http.StatusOK)
+		w.Write(faviconBytes)
+	default:
+		RespondError(w, r, "Not Found", http.StatusNotFound)
+	}
 }
 
 // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage address as a text/plain response
-func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.post.raw", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.post.raw", "ruid", ruid)
 
 	postRawCount.Inc(1)
 
-	ctx := r.Context()
-	var sp opentracing.Span
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.post.raw")
-	defer sp.Finish()
-
 	toEncrypt := false
-	if r.uri.Addr == "encrypt" {
+	uri := GetURI(r.Context())
+	if uri.Addr == "encrypt" {
 		toEncrypt = true
 	}
 
-	if r.uri.Path != "" {
+	if uri.Path != "" {
 		postRawFail.Inc(1)
-		Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
+		RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
 		return
 	}
 
-	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
+	if uri.Addr != "" && uri.Addr != "encrypt" {
 		postRawFail.Inc(1)
-		Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
+		RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
 		return
 	}
 
 	if r.Header.Get("Content-Length") == "" {
 		postRawFail.Inc(1)
-		Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest)
+		RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest)
 		return
 	}
 
-	addr, _, err := s.api.Store(ctx, r.Body, r.ContentLength, toEncrypt)
+	addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
 	if err != nil {
 		postRawFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
+		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	log.Debug("stored content", "ruid", r.ruid, "key", addr)
+	log.Debug("stored content", "ruid", ruid, "key", addr)
 
 	w.Header().Set("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
@@ -311,55 +267,49 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 // (either a tar archive or multipart form), adds those files either to an
 // existing manifest or to a new manifest under <path> and returns the
 // resulting manifest hash as a text/plain response
-func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.post.files", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.post.files", "ruid", ruid)
 	postFilesCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.post.files")
-	defer sp.Finish()
-
 	contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
 	if err != nil {
 		postFilesFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusBadRequest)
+		RespondError(w, r, err.Error(), http.StatusBadRequest)
 		return
 	}
 
 	toEncrypt := false
-	if r.uri.Addr == "encrypt" {
+	uri := GetURI(r.Context())
+	if uri.Addr == "encrypt" {
 		toEncrypt = true
 	}
 
 	var addr storage.Address
-	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
+	if uri.Addr != "" && uri.Addr != "encrypt" {
-		addr, err = s.api.Resolve(r.Context(), r.uri)
+		addr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			postFilesFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
 			return
 		}
-		log.Debug("resolved key", "ruid", r.ruid, "key", addr)
+		log.Debug("resolved key", "ruid", ruid, "key", addr)
 	} else {
 		addr, err = s.api.NewManifest(r.Context(), toEncrypt)
 		if err != nil {
 			postFilesFail.Inc(1)
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
-		log.Debug("new manifest", "ruid", r.ruid, "key", addr)
+		log.Debug("new manifest", "ruid", ruid, "key", addr)
 	}
 
-	newAddr, err := s.api.UpdateManifest(ctx, addr, func(mw *api.ManifestWriter) error {
+	newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
 		switch contentType {
-
 		case "application/x-tar":
 			_, err := s.handleTarUpload(r, mw)
 			if err != nil {
-				Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
+				RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
 				return err
 			}
 			return nil
@@ -372,30 +322,33 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 	})
 	if err != nil {
 		postFilesFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
+		RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
 		return
 	}
 
-	log.Debug("stored content", "ruid", r.ruid, "key", newAddr)
+	log.Debug("stored content", "ruid", ruid, "key", newAddr)
 
 	w.Header().Set("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
 	fmt.Fprint(w, newAddr)
 }
 
-func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) {
+func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) {
-	log.Debug("handle.tar.upload", "ruid", r.ruid)
+	log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()))
 
-	key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw)
+	defaultPath := r.URL.Query().Get("defaultpath")
+
+	key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, defaultPath, mw)
 	if err != nil {
 		return nil, err
 	}
 	return key, nil
 }
 
-func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
+func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api.ManifestWriter) error {
-	log.Debug("handle.multipart.upload", "ruid", req.ruid)
+	ruid := GetRUID(r.Context())
-	mr := multipart.NewReader(req.Body, boundary)
+	log.Debug("handle.multipart.upload", "ruid", ruid)
+	mr := multipart.NewReader(r.Body, boundary)
 	for {
 		part, err := mr.NextPart()
 		if err == io.EOF {
@@ -435,48 +388,52 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma
 		if name == "" {
 			name = part.FormName()
 		}
-		path := path.Join(req.uri.Path, name)
+		uri := GetURI(r.Context())
+		path := path.Join(uri.Path, name)
 		entry := &api.ManifestEntry{
 			Path:        path,
 			ContentType: part.Header.Get("Content-Type"),
 			Size:        size,
 			ModTime:     time.Now(),
 		}
-		log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
+		log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path)
-		contentKey, err := mw.AddEntry(req.Context(), reader, entry)
+		contentKey, err := mw.AddEntry(r.Context(), reader, entry)
 		if err != nil {
 			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
 		}
-		log.Debug("stored content", "ruid", req.ruid, "key", contentKey)
+		log.Debug("stored content", "ruid", ruid, "key", contentKey)
 	}
 }
 
-func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
+func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) error {
-	log.Debug("handle.direct.upload", "ruid", req.ruid)
+	ruid := GetRUID(r.Context())
-	key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{
+	log.Debug("handle.direct.upload", "ruid", ruid)
-		Path:        req.uri.Path,
+	key, err := mw.AddEntry(r.Context(), r.Body, &api.ManifestEntry{
-		ContentType: req.Header.Get("Content-Type"),
+		Path:        GetURI(r.Context()).Path,
+		ContentType: r.Header.Get("Content-Type"),
 		Mode:        0644,
-		Size:        req.ContentLength,
+		Size:        r.ContentLength,
 		ModTime:     time.Now(),
 	})
 	if err != nil {
 		return err
 	}
-	log.Debug("stored content", "ruid", req.ruid, "key", key)
+	log.Debug("stored content", "ruid", ruid, "key", key)
 	return nil
 }
 
 // HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
 // <path> from <manifest> and returns the resulting manifest hash as a
 // text/plain response
-func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.delete", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.delete", "ruid", ruid)
 	deleteCount.Inc(1)
-	newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path)
+	newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path)
 	if err != nil {
 		deleteFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
+		RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
 		return
 	}
 
@@ -519,27 +476,20 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) {
 //
 // The POST request admits a JSON structure as defined in the mru package: `mru.updateRequestJSON`
 // The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content
-func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.post.resource", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
-
+	log.Debug("handle.post.resource", "ruid", ruid)
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.post.resource")
-	defer sp.Finish()
-
 	var err error
 
 	// Creation and update must send mru.updateRequestJSON JSON structure
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
+		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	var updateRequest mru.Request
 	if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON
-		Respond(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
+		RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
 		return
 	}
 
@@ -548,7 +498,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// to update this resource
 		// Check this early, to avoid creating a resource and then not being able to set its first update.
 		if err = updateRequest.Verify(); err != nil {
-			Respond(w, r, err.Error(), http.StatusForbidden)
+			RespondError(w, r, err.Error(), http.StatusForbidden)
 			return
 		}
 	}
@@ -557,7 +507,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		err = s.api.ResourceCreate(r.Context(), &updateRequest)
 		if err != nil {
 			code, err2 := s.translateResourceError(w, r, "resource creation fail", err)
-			Respond(w, r, err2.Error(), code)
+			RespondError(w, r, err2.Error(), code)
 			return
 		}
 	}
@@ -565,7 +515,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 	if updateRequest.IsUpdate() {
 		_, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate)
 		if err != nil {
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
 	}
@@ -579,7 +529,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// metadata chunk (rootAddr)
 		m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex())
 		if err != nil {
-			Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
+			RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
 			return
 		}
 
@@ -589,7 +539,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// \TODO update manifest key automatically in ENS
 		outdata, err := json.Marshal(m)
 		if err != nil {
-			Respond(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
+			RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
 			return
 		}
 		fmt.Fprint(w, string(outdata))
@@ -604,17 +554,19 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 // bzz-resource://<id>/meta - get metadata and next version information
 // <id> = ens name or hash
 // TODO: Enable pass maxPeriod parameter
-func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get.resource", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.resource", "ruid", ruid)
 	var err error
 
 	// resolve the content key.
-	manifestAddr := r.uri.Address()
+	manifestAddr := uri.Address()
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 			return
 		}
 	} else {
@@ -625,25 +577,25 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 	rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr)
 	if err != nil {
 		getFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound)
 		return
 	}
 
-	log.Debug("handle.get.resource: resolved", "ruid", r.ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
+	log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
 
 	// determine if the query specifies period and version or it is a metadata query
 	var params []string
-	if len(r.uri.Path) > 0 {
+	if len(uri.Path) > 0 {
-		if r.uri.Path == "meta" {
+		if uri.Path == "meta" {
 			unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr)
 			if err != nil {
 				getFail.Inc(1)
-				Respond(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
+				RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
 				return
 			}
 			rawResponse, err := unsignedUpdateRequest.MarshalJSON()
 			if err != nil {
-				Respond(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
+				RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
 				return
 			}
 			w.Header().Add("Content-type", "application/json")
@@ -653,7 +605,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 
 		}
 
-		params = strings.Split(r.uri.Path, "/")
+		params = strings.Split(uri.Path, "/")
 
 	}
 	var name string
@@ -689,17 +641,17 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 	// any error from the switch statement will end up here
 	if err != nil {
 		code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err)
-		Respond(w, r, err2.Error(), code)
+		RespondError(w, r, err2.Error(), code)
 		return
 	}
 
 	// All ok, serve the retrieved update
-	log.Debug("Found update", "name", name, "ruid", r.ruid)
+	log.Debug("Found update", "name", name, "ruid", ruid)
 	w.Header().Set("Content-Type", "application/octet-stream")
-	http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data))
+	http.ServeContent(w, r, "", now, bytes.NewReader(data))
 }
 
-func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) (int, error) {
+func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) {
 	code := 0
 	defaultErr := fmt.Errorf("%s: %v", supErr, err)
 	rsrcErr, ok := err.(*mru.Error)
@@ -725,46 +677,41 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
 //   given storage key
 // - bzz-hash://<key> and responds with the hash of the content stored
 //   at the given storage key as a text/plain response
-func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get", "ruid", ruid, "uri", uri)
 	getCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.get")
-	defer sp.Finish()
-
 	var err error
-	addr := r.uri.Address()
+	addr := uri.Address()
 	if addr == nil {
-		addr, err = s.api.Resolve(r.Context(), r.uri)
+		addr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 			return
 		}
 	} else {
 		w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
 	}
 
-	log.Debug("handle.get: resolved", "ruid", r.ruid, "key", addr)
+	log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
 
 	// if path is set, interpret <key> as a manifest and return the
 	// raw entry at the given path
-	if r.uri.Path != "" {
+	if uri.Path != "" {
 		walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
 		if err != nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
+			RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
 			return
 		}
 		var entry *api.ManifestEntry
 		walker.Walk(func(e *api.ManifestEntry) error {
 			// if the entry matches the path, set entry and stop
 			// the walk
-			if e.Path == r.uri.Path {
+			if e.Path == uri.Path {
 				entry = e
 				// return an error to cancel the walk
 				return errors.New("found")
@@ -778,7 +725,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 			// if the manifest's path is a prefix of the
 			// requested path, recurse into it by returning
 			// nil and continuing the walk
-			if strings.HasPrefix(r.uri.Path, e.Path) {
+			if strings.HasPrefix(uri.Path, e.Path) {
 				return nil
 			}
 
@@ -786,7 +733,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 		})
 		if entry == nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
 			return
 		}
 		addr = storage.Address(common.Hex2Bytes(entry.Hash))
@@ -796,23 +743,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
 	if noneMatchEtag != "" {
 		if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) {
-			Respond(w, r, "Not Modified", http.StatusNotModified)
+			w.WriteHeader(http.StatusNotModified)
 			return
 		}
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	reader, isEncrypted := s.api.Retrieve(ctx, addr)
+	reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
-	if _, err := reader.Size(ctx, nil); err != nil {
+	if _, err := reader.Size(r.Context(), nil); err != nil {
 		getFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
 		return
 	}
 
 	w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
 
 	switch {
-	case r.uri.Raw():
+	case uri.Raw():
 		// allow the request to overwrite the content type using a query
 		// parameter
 		contentType := "application/octet-stream"
@@ -820,8 +767,8 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 			contentType = typ
 		}
 		w.Header().Set("Content-Type", contentType)
-		http.ServeContent(w, &r.Request, "", time.Now(), reader)
+		http.ServeContent(w, r, "", time.Now(), reader)
-	case r.uri.Hash():
+	case uri.Hash():
 		w.Header().Set("Content-Type", "text/plain")
 		w.WriteHeader(http.StatusOK)
 		fmt.Fprint(w, addr)
@@ -831,35 +778,30 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 // HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
 // a list of all files contained in <manifest> under <path> grouped into
 // common prefixes using "/" as a delimiter
-func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
 	getListCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.get.list")
-	defer sp.Finish()
-
 	// ensure the root path has a trailing slash so that relative URLs work
-	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+	if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 
-	addr, err := s.api.Resolve(r.Context(), r.uri)
+	addr, err := s.api.Resolve(r.Context(), uri)
 	if err != nil {
 		getListFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 		return
 	}
-	log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr)
+	log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
 
-	list, err := s.api.GetManifestList(ctx, addr, r.uri.Path)
+	list, err := s.api.GetManifestList(r.Context(), addr, uri.Path)
 	if err != nil {
 		getListFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
+		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
@@ -867,11 +809,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	// HTML index with relative URLs
 	if strings.Contains(r.Header.Get("Accept"), "text/html") {
 		w.Header().Set("Content-Type", "text/html")
-		err := htmlListTemplate.Execute(w, &htmlListData{
+		err := TemplatesMap["bzz-list"].Execute(w, &htmlListData{
 			URI: &api.URI{
 				Scheme: "bzz",
-				Addr:   r.uri.Addr,
+				Addr:   uri.Addr,
-				Path:   r.uri.Path,
+				Path:   uri.Path,
 			},
 			List: &list,
 		})
@@ -888,45 +830,40 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 
 // HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
 // with the content of the file at <path> from the given <manifest>
-func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get.file", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.file", "ruid", ruid)
 	getFileCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.get.file")
-	defer sp.Finish()
-
 	// ensure the root path has a trailing slash so that relative URLs work
-	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+	if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 	var err error
-	manifestAddr := r.uri.Address()
+	manifestAddr := uri.Address()
 
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			getFileFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 			return
 		}
 	} else {
 		w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
 	}
 
-	log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr)
+	log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
-	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path)
+	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path)
 
 	etag := common.Bytes2Hex(contentKey)
 	noneMatchEtag := r.Header.Get("If-None-Match")
 	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key.
 	if noneMatchEtag != "" {
 		if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) {
-			Respond(w, r, "Not Modified", http.StatusNotModified)
+			w.WriteHeader(http.StatusNotModified)
 			return
 		}
 	}
@@ -935,10 +872,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 		switch status {
 		case http.StatusNotFound:
 			getFileNotFound.Inc(1)
-			Respond(w, r, err.Error(), http.StatusNotFound)
+			RespondError(w, r, err.Error(), http.StatusNotFound)
 		default:
 			getFileFail.Inc(1)
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		}
 		return
 	}
@@ -946,28 +883,28 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	//the request results in ambiguous files
 	//e.g. /read with readme.md and readinglist.txt available in manifest
 	if status == http.StatusMultipleChoices {
-		list, err := s.api.GetManifestList(ctx, manifestAddr, r.uri.Path)
+		list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path)
 		if err != nil {
 			getFileFail.Inc(1)
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
-		log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", r.ruid)
+		log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", ruid)
 		//show a nice page links to available entries
 		ShowMultipleChoices(w, r, list)
 		return
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	if _, err := reader.Size(ctx, nil); err != nil {
+	if _, err := reader.Size(r.Context(), nil); err != nil {
 		getFileNotFound.Inc(1)
-		Respond(w, r, fmt.Sprintf("file not found %s: %s", r.uri, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", contentType)
-	http.ServeContent(w, &r.Request, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
+	http.ServeContent(w, r, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
 }
 
 // The size of buffer used for bufio.Reader on LazyChunkReader passed to

swarm/api/manifest.go

@@ -106,13 +106,18 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC
 }
 
 // AddEntry stores the given data and adds the resulting key to the manifest
-func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (storage.Address, error) {
+func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (key storage.Address, err error) {
-	key, _, err := m.api.Store(ctx, data, e.Size, m.trie.encrypted)
+	entry := newManifestTrieEntry(e, nil)
+	if data != nil {
+		key, _, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted)
 		if err != nil {
 			return nil, err
 		}
-	entry := newManifestTrieEntry(e, nil)
 		entry.Hash = key.Hex()
+	}
+	if entry.Hash == "" {
+		return key, errors.New("missing entry hash")
+	}
 	m.trie.addEntry(entry, m.quitC)
 	return key, nil
 }
@@ -159,7 +164,7 @@ func (m *ManifestWalker) Walk(walkFn WalkFn) error {
 }
 
 func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
-	for _, entry := range trie.entries {
+	for _, entry := range &trie.entries {
 		if entry == nil {
 			continue
 		}
@@ -308,7 +313,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 }
 
 func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
-	for _, e := range mt.entries {
+	for _, e := range &mt.entries {
 		if e != nil {
 			cnt++
 			entry = e
@@ -362,7 +367,7 @@ func (mt *manifestTrie) recalcAndStore() error {
 	buffer.WriteString(`{"entries":[`)
 
 	list := &Manifest{}
-	for _, entry := range mt.entries {
+	for _, entry := range &mt.entries {
 		if entry != nil {
 			if entry.Hash == "" { // TODO: paralellize
 				err := entry.subtrie.recalcAndStore()

swarm/bmt/bmt.go

@@ -55,9 +55,6 @@ Two implementations are provided:
 */
 
 const (
-	// SegmentCount is the maximum number of segments of the underlying chunk
-	// Should be equal to max-chunk-data-size / hash-size
-	SegmentCount = 128
 	// PoolSize is the maximum number of bmt trees used by the hashers, i.e,
 	// the maximum number of concurrent BMT hashing operations performed by the same hasher
 	PoolSize = 8
@@ -318,7 +315,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) {
 // with every full segment calls writeSection in a go routine
 func (h *Hasher) Write(b []byte) (int, error) {
 	l := len(b)
-	if l == 0 {
+	if l == 0 || l > h.pool.Size {
 		return 0, nil
 	}
 	t := h.getTree()

swarm/bmt/bmt_test.go

@@ -34,6 +34,13 @@ import (
 // the actual data length generated (could be longer than max datalength of the BMT)
 const BufferSize = 4128
 
+const (
+	// segmentCount is the maximum number of segments of the underlying chunk
+	// Should be equal to max-chunk-data-size / hash-size
+	// Currently set to 128 == 4096 (default chunk size) / 32 (sha3.keccak256 size)
+	segmentCount = 128
+)
+
 var counts = []int{1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 32, 37, 42, 53, 63, 64, 65, 111, 127, 128}
 
 // calculates the Keccak256 SHA3 hash of the data
@@ -224,14 +231,14 @@ func TestHasherReuse(t *testing.T) {
 // tests if bmt reuse is not corrupting result
 func testHasherReuse(poolsize int, t *testing.T) {
 	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, SegmentCount, poolsize)
+	pool := NewTreePool(hasher, segmentCount, poolsize)
 	defer pool.Drain(0)
 	bmt := New(pool)
 
 	for i := 0; i < 100; i++ {
 		data := newData(BufferSize)
 		n := rand.Intn(bmt.Size())
-		err := testHasherCorrectness(bmt, hasher, data, n, SegmentCount)
+		err := testHasherCorrectness(bmt, hasher, data, n, segmentCount)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -241,7 +248,7 @@ func testHasherReuse(poolsize int, t *testing.T) {
 // Tests if pool can be cleanly reused even in concurrent use by several hasher
 func TestBMTConcurrentUse(t *testing.T) {
 	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, SegmentCount, PoolSize)
+	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	defer pool.Drain(0)
 	cycles := 100
 	errc := make(chan error)
@@ -451,7 +458,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) {
 func benchmarkBMT(t *testing.B, n int) {
 	data := newData(n)
 	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, SegmentCount, PoolSize)
+	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	bmt := New(pool)
 
 	t.ReportAllocs()
@@ -465,7 +472,7 @@ func benchmarkBMT(t *testing.B, n int) {
 func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
 	data := newData(n)
 	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, SegmentCount, PoolSize)
+	pool := NewTreePool(hasher, segmentCount, PoolSize)
 	bmt := New(pool).NewAsyncWriter(double)
 	idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
 	shuffle(len(idxs), func(i int, j int) {
@@ -483,7 +490,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
 func benchmarkPool(t *testing.B, poolsize, n int) {
 	data := newData(n)
 	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, SegmentCount, poolsize)
+	pool := NewTreePool(hasher, segmentCount, poolsize)
 	cycles := 100
 
 	t.ReportAllocs()

swarm/chunk/chunk.go

@@ -0,0 +1,5 @@
+package chunk
+
+const (
+	DefaultSize = 4096
+)

swarm/network/protocol.go

@@ -94,12 +94,14 @@ type BzzConfig struct {
 	UnderlayAddr []byte // node's underlay address
 	HiveParams   *HiveParams
 	NetworkID    uint64
+	LightNode    bool
 }
 
 // Bzz is the swarm protocol bundle
 type Bzz struct {
 	*Hive
 	NetworkID    uint64
+	LightNode    bool
 	localAddr    *BzzAddr
 	mtx          sync.Mutex
 	handshakes   map[discover.NodeID]*HandshakeMsg
@@ -116,6 +118,7 @@ func NewBzz(config *BzzConfig, kad Overlay, store state.Store, streamerSpec *pro
 	return &Bzz{
 		Hive:         NewHive(config.HiveParams, kad, store),
 		NetworkID:    config.NetworkID,
+		LightNode:    config.LightNode,
 		localAddr:    &BzzAddr{config.OverlayAddr, config.UnderlayAddr},
 		handshakes:   make(map[discover.NodeID]*HandshakeMsg),
 		streamerRun:  streamerRun,
@@ -209,7 +212,11 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*
 			localAddr:  b.localAddr,
 			BzzAddr:    handshake.peerAddr,
 			lastActive: time.Now(),
+			LightNode:  handshake.LightNode,
 		}
+
+		log.Debug("peer created", "addr", handshake.peerAddr.String())
+
 		return run(peer)
 	}
 }
@@ -228,6 +235,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error
 		return err
 	}
 	handshake.peerAddr = rsh.(*HandshakeMsg).Addr
+	handshake.LightNode = rsh.(*HandshakeMsg).LightNode
 	return nil
 }
 
@@ -263,6 +271,7 @@ type BzzPeer struct {
 	localAddr       *BzzAddr  // local Peers address
 	*BzzAddr                  // remote address -> implements Addr interface = protocols.Peer
 	lastActive      time.Time // time is updated whenever mutexes are releasing
+	LightNode       bool
 }
 
 func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer {
@@ -294,6 +303,7 @@ type HandshakeMsg struct {
 	Version   uint64
 	NetworkID uint64
 	Addr      *BzzAddr
+	LightNode bool
 
 	// peerAddr is the address received in the peer handshake
 	peerAddr *BzzAddr
@@ -305,7 +315,7 @@ type HandshakeMsg struct {
 
 // String pretty prints the handshake
 func (bh *HandshakeMsg) String() string {
-	return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v", bh.Version, bh.NetworkID, bh.Addr)
+	return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, LightNode: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.LightNode, bh.peerAddr)
 }
 
 // Perform initiates the handshake and validates the remote handshake message
@@ -338,6 +348,7 @@ func (b *Bzz) GetHandshake(peerID discover.NodeID) (*HandshakeMsg, bool) {
 			Version:   uint64(BzzSpec.Version),
 			NetworkID: b.NetworkID,
 			Addr:      b.localAddr,
+			LightNode: b.LightNode,
 			init:      make(chan bool, 1),
 			done:      make(chan struct{}),
 		}

swarm/network/protocol_test.go

@@ -30,6 +30,11 @@ import (
 	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
 )
 
+const (
+	TestProtocolVersion   = 5
+	TestProtocolNetworkID = 3
+)
+
 var (
 	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
 )
@@ -127,23 +132,30 @@ type bzzTester struct {
 	*p2ptest.ProtocolTester
 	addr *BzzAddr
 	cs   map[string]chan bool
+	bzz  *Bzz
 }
 
-func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr) *bzzTester {
+func newBzz(addr *BzzAddr, lightNode bool) *Bzz {
 	config := &BzzConfig{
 		OverlayAddr:  addr.Over(),
 		UnderlayAddr: addr.Under(),
 		HiveParams:   NewHiveParams(),
 		NetworkID:    DefaultNetworkID,
+		LightNode:    lightNode,
 	}
 	kad := NewKademlia(addr.OAddr, NewKadParams())
 	bzz := NewBzz(config, kad, nil, nil, nil)
+	return bzz
+}
 
-	s := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, bzz.runBzz)
+func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *bzzTester {
+	bzz := newBzz(addr, lightNode)
+	pt := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), n, bzz.runBzz)
 
 	return &bzzTester{
 		addr:           addr,
-		ProtocolTester: s,
+		ProtocolTester: pt,
+		bzz:            bzz,
 	}
 }
 
@@ -184,22 +196,24 @@ func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptes
 	return nil
 }
 
-func correctBzzHandshake(addr *BzzAddr) *HandshakeMsg {
+func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg {
 	return &HandshakeMsg{
-		Version:   5,
+		Version:   TestProtocolVersion,
-		NetworkID: DefaultNetworkID,
+		NetworkID: TestProtocolNetworkID,
 		Addr:      addr,
+		LightNode: lightNode,
 	}
 }
 
 func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
+	lightNode := false
 	addr := RandomAddr()
-	s := newBzzHandshakeTester(t, 1, addr)
+	s := newBzzHandshakeTester(t, 1, addr, lightNode)
 	id := s.IDs[0]
 
 	err := s.testHandshake(
-		correctBzzHandshake(addr),
+		correctBzzHandshake(addr, lightNode),
-		&HandshakeMsg{Version: 5, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
+		&HandshakeMsg{Version: TestProtocolVersion, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
 		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): network id mismatch 321 (!= 3)")},
 	)
 
@@ -209,14 +223,15 @@ func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
 }
 
 func TestBzzHandshakeVersionMismatch(t *testing.T) {
+	lightNode := false
 	addr := RandomAddr()
-	s := newBzzHandshakeTester(t, 1, addr)
+	s := newBzzHandshakeTester(t, 1, addr, lightNode)
 	id := s.IDs[0]
 
 	err := s.testHandshake(
-		correctBzzHandshake(addr),
+		correctBzzHandshake(addr, lightNode),
-		&HandshakeMsg{Version: 0, NetworkID: 3, Addr: NewAddrFromNodeID(id)},
+		&HandshakeMsg{Version: 0, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)},
-		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= 5)")},
+		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= %d)", TestProtocolVersion)},
 	)
 
 	if err != nil {
@@ -225,16 +240,49 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) {
 }
 
 func TestBzzHandshakeSuccess(t *testing.T) {
+	lightNode := false
 	addr := RandomAddr()
-	s := newBzzHandshakeTester(t, 1, addr)
+	s := newBzzHandshakeTester(t, 1, addr, lightNode)
 	id := s.IDs[0]
 
 	err := s.testHandshake(
-		correctBzzHandshake(addr),
+		correctBzzHandshake(addr, lightNode),
-		&HandshakeMsg{Version: 5, NetworkID: 3, Addr: NewAddrFromNodeID(id)},
+		&HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)},
 	)
 
 	if err != nil {
 		t.Fatal(err)
 	}
 }
+
+func TestBzzHandshakeLightNode(t *testing.T) {
+	var lightNodeTests = []struct {
+		name      string
+		lightNode bool
+	}{
+		{"on", true},
+		{"off", false},
+	}
+
+	for _, test := range lightNodeTests {
+		t.Run(test.name, func(t *testing.T) {
+			randomAddr := RandomAddr()
+			pt := newBzzHandshakeTester(t, 1, randomAddr, false)
+			id := pt.IDs[0]
+			addr := NewAddrFromNodeID(id)
+
+			err := pt.testHandshake(
+				correctBzzHandshake(randomAddr, false),
+				&HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: addr, LightNode: test.lightNode},
+			)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if pt.bzz.handshakes[id].LightNode != test.lightNode {
+				t.Fatalf("peer LightNode flag is %v, should be %v", pt.bzz.handshakes[id].LightNode, test.lightNode)
+			}
+		})
+	}
+}

swarm/network/simulation/simulation.go

@@ -62,6 +62,8 @@ type Simulation struct {
 // where all "global" state related to the service should be kept.
 // All cleanups needed for constructed service and any other constructed
 // objects should ne provided in a single returned cleanup function.
+// Returned cleanup function will be called by Close function
+// after network shutdown.
 type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error)
 
 // New creates a new Simulation instance with new
@@ -161,6 +163,18 @@ var maxParallelCleanups = 10
 // simulation.
 func (s *Simulation) Close() {
 	close(s.done)
+
+	// Close all connections before calling the Network Shutdown.
+	// It is possible that p2p.Server.Stop will block if there are
+	// existing connections.
+	for _, c := range s.Net.Conns {
+		if c.Up {
+			s.Net.Disconnect(c.One, c.Other)
+		}
+	}
+	s.shutdownWG.Wait()
+	s.Net.Shutdown()
+
 	sem := make(chan struct{}, maxParallelCleanups)
 	s.mu.RLock()
 	cleanupFuncs := make([]func(), len(s.cleanupFuncs))
@@ -170,16 +184,19 @@ func (s *Simulation) Close() {
 		}
 	}
 	s.mu.RUnlock()
+	var cleanupWG sync.WaitGroup
 	for _, cleanup := range cleanupFuncs {
-		s.shutdownWG.Add(1)
+		cleanupWG.Add(1)
 		sem <- struct{}{}
 		go func(cleanup func()) {
-			defer s.shutdownWG.Done()
+			defer cleanupWG.Done()
 			defer func() { <-sem }()
 
 			cleanup()
 		}(cleanup)
 	}
+	cleanupWG.Wait()
+
 	if s.httpSrv != nil {
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 		defer cancel()
@@ -189,8 +206,6 @@ func (s *Simulation) Close() {
 		}
 		close(s.runC)
 	}
-	s.shutdownWG.Wait()
-	s.Net.Shutdown()
 }
 
 // Done returns a channel that is closed when the simulation

swarm/network/simulation/simulation_test.go

@@ -68,7 +68,7 @@ func TestRun(t *testing.T) {
 		defer cancel()
 
 		r := sim.Run(ctx, func(ctx context.Context, sim *Simulation) error {
-			time.Sleep(100 * time.Millisecond)
+			time.Sleep(time.Second)
 			return nil
 		})
 

swarm/network/stream/delivery.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/p2p/discover"
+	cp "github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/spancontext"
@@ -229,6 +230,11 @@ R:
 	for req := range d.receiveC {
 		processReceivedChunksCount.Inc(1)
 
+		if len(req.SData) > cp.DefaultSize+8 {
+			log.Warn("received chunk is bigger than expected", "len", len(req.SData))
+			continue R
+		}
+
 		// this should be has locally
 		chunk, err := d.db.Get(context.TODO(), req.Addr)
 		if err == nil {
@@ -244,6 +250,7 @@ R:
 			continue R
 		default:
 		}
+
 		chunk.SData = req.SData
 		d.db.Put(context.TODO(), chunk)
 

swarm/network/stream/delivery_test.go

@@ -393,6 +393,11 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 			return err
 		}
 
+		log.Debug("Waiting for kademlia")
+		if _, err := sim.WaitTillHealthy(ctx, 2); err != nil {
+			return err
+		}
+
 		//each of the nodes (except pivot node) subscribes to the stream of the next node
 		for j, node := range nodeIDs[0 : nodes-1] {
 			sid := nodeIDs[j+1]
@@ -424,11 +429,6 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 			}
 		}()
 
-		log.Debug("Waiting for kademlia")
-		if _, err := sim.WaitTillHealthy(ctx, 2); err != nil {
-			return err
-		}
-
 		log.Debug("Watching for disconnections")
 		disconnections := sim.PeerEvents(
 			context.Background(),

swarm/network/stream/snapshot_sync_test.go

@@ -246,6 +246,8 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
 					if err != nil {
 						log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id))
 						localSuccess = false
+						// Do not get crazy with logging the warn message
+						time.Sleep(500 * time.Millisecond)
 					} else {
 						log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id))
 					}
@@ -426,6 +428,8 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error {
 					if err != nil {
 						log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id))
 						localSuccess = false
+						// Do not get crazy with logging the warn message
+						time.Sleep(500 * time.Millisecond)
 					} else {
 						log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id))
 					}

swarm/sctx/sctx.go

@@ -0,0 +1,7 @@
+package sctx
+
+type ContextKey int
+
+const (
+	HTTPRequestIDKey ContextKey = iota
+)

swarm/storage/chunker.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -69,10 +70,6 @@ var (
 	errOperationTimedOut    = errors.New("operation timed out")
 )
 
-const (
-	DefaultChunkSize int64 = 4096
-)
-
 type ChunkerParams struct {
 	chunkSize int64
 	hashSize  int64
@@ -136,7 +133,7 @@ type TreeChunker struct {
 func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
 	jp := &JoinerParams{
 		ChunkerParams: ChunkerParams{
-			chunkSize: DefaultChunkSize,
+			chunkSize: chunk.DefaultSize,
 			hashSize:  int64(len(addr)),
 		},
 		addr:   addr,
@@ -156,7 +153,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (
 	tsp := &TreeSplitterParams{
 		SplitterParams: SplitterParams{
 			ChunkerParams: ChunkerParams{
-				chunkSize: DefaultChunkSize,
+				chunkSize: chunk.DefaultSize,
 				hashSize:  putter.RefSize(),
 			},
 			reader: data,

swarm/storage/hasherstore.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/storage/encryption"
 )
 
@@ -57,7 +58,7 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool)
 	refSize := int64(hashSize)
 	if toEncrypt {
 		refSize += encryption.KeyLength
-		chunkEncryption = newChunkEncryption(DefaultChunkSize, refSize)
+		chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize)
 	}
 
 	return &hasherStore{
@@ -190,9 +191,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp
 
 	// removing extra bytes which were just added for padding
 	length := ChunkData(decryptedSpan).Size()
-	for length > DefaultChunkSize {
+	for length > chunk.DefaultSize {
-		length = length + (DefaultChunkSize - 1)
+		length = length + (chunk.DefaultSize - 1)
-		length = length / DefaultChunkSize
+		length = length / chunk.DefaultSize
 		length *= h.refSize
 	}
 

swarm/storage/ldbstore_test.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 
@@ -184,7 +185,7 @@ func testIterator(t *testing.T, mock bool) {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
 
-	chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount)
+	chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount)
 
 	wg := &sync.WaitGroup{}
 	wg.Add(len(chunks))
@@ -294,7 +295,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < n; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}
@@ -344,7 +345,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < n; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}
@@ -398,7 +399,7 @@ func TestLDBStoreAddRemove(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < n; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}
@@ -460,7 +461,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < capacity; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}

swarm/storage/localstore.go

@@ -98,20 +98,16 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) {
 // After the LDBStore.Put, it is ensured that the MemStore
 // contains the chunk with the same data, but nil ReqC channel.
 func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk) {
-	if l := len(chunk.SData); l < 9 {
-		log.Debug("incomplete chunk data", "addr", chunk.Addr, "length", l)
-		chunk.SetErrored(ErrChunkInvalid)
-		chunk.markAsStored()
-		return
-	}
 	valid := true
+	// ls.Validators contains a list of one validator per chunk type.
+	// if one validator succeeds, then the chunk is valid
 	for _, v := range ls.Validators {
 		if valid = v.Validate(chunk.Addr, chunk.SData); valid {
 			break
 		}
 	}
 	if !valid {
-		log.Trace("invalid content address", "addr", chunk.Addr)
+		log.Trace("invalid chunk", "addr", chunk.Addr, "len", len(chunk.SData))
 		chunk.SetErrored(ErrChunkInvalid)
 		chunk.markAsStored()
 		return

swarm/storage/localstore_test.go

@@ -20,6 +20,8 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 )
 
 var (
@@ -61,7 +63,7 @@ func TestValidator(t *testing.T) {
 	// add content address validator and check puts
 	// bad should fail, good should pass
 	store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
-	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk = chunks[0]
 	badChunk = chunks[1]
 	copy(badChunk.SData, goodChunk.SData)
@@ -79,7 +81,7 @@ func TestValidator(t *testing.T) {
 	var negV boolTestValidator
 	store.Validators = append(store.Validators, negV)
 
-	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk = chunks[0]
 	badChunk = chunks[1]
 	copy(badChunk.SData, goodChunk.SData)
@@ -97,7 +99,7 @@ func TestValidator(t *testing.T) {
 	var posV boolTestValidator = true
 	store.Validators = append(store.Validators, posV)
 
-	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk = chunks[0]
 	badChunk = chunks[1]
 	copy(badChunk.SData, goodChunk.SData)

swarm/storage/mru/handler.go

@@ -21,17 +21,15 @@ package mru
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"sync"
 	"time"
 	"unsafe"
 
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-const chunkSize = 4096 // temporary until we implement FileStore in the resourcehandler
-
 type Handler struct {
 	chunkStore      *storage.NetStore
 	HashSize        int
@@ -66,8 +64,7 @@ func init() {
 }
 
 // NewHandler creates a new Mutable Resource API
-func NewHandler(params *HandlerParams) (*Handler, error) {
+func NewHandler(params *HandlerParams) *Handler {
-
 	rh := &Handler{
 		resources:       make(map[uint64]*resource),
 		storeTimeout:    defaultStoreTimeout,
@@ -82,7 +79,7 @@ func NewHandler(params *HandlerParams) (*Handler, error) {
 		hashPool.Put(hashfunc)
 	}
 
-	return rh, nil
+	return rh
 }
 
 // SetStore sets the store backend for the Mutable Resource API
@@ -94,9 +91,8 @@ func (h *Handler) SetStore(store *storage.NetStore) {
 // If it looks like a resource update, the chunk address is checked against the ownerAddr of the update's signature
 // It implements the storage.ChunkValidator interface
 func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
-
 	dataLength := len(data)
-	if dataLength < minimumChunkLength {
+	if dataLength < minimumChunkLength || dataLength > chunk.DefaultSize+8 {
 		return false
 	}
 
@@ -106,7 +102,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
 		rootAddr, _ := metadataHash(data)
 		valid := bytes.Equal(chunkAddr, rootAddr)
 		if !valid {
-			log.Debug(fmt.Sprintf("Invalid root metadata chunk with address: %s", chunkAddr.Hex()))
+			log.Debug("Invalid root metadata chunk with address", "addr", chunkAddr.Hex())
 		}
 		return valid
 	}
@@ -118,7 +114,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
 	// First, deserialize the chunk
 	var r SignedResourceUpdate
 	if err := r.fromChunk(chunkAddr, data); err != nil {
-		log.Debug("Invalid resource chunk with address %s: %s ", chunkAddr.Hex(), err.Error())
+		log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error())
 		return false
 	}
 
@@ -126,7 +122,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
 	// that was used to retrieve this chunk
 	// if this validation fails, someone forged a chunk.
 	if !bytes.Equal(chunkAddr, r.updateHeader.UpdateAddr()) {
-		log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr %s", chunkAddr.Hex())
+		log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr", "addr", chunkAddr.Hex())
 		return false
 	}
 
@@ -134,7 +130,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
 	// If it fails, it means either the signature is not valid, data is corrupted
 	// or someone is trying to update someone else's resource.
 	if err := r.Verify(); err != nil {
-		log.Debug("Invalid signature: %v", err)
+		log.Debug("Invalid signature", "err", err)
 		return false
 	}
 
@@ -172,11 +168,6 @@ func (h *Handler) GetVersion(rootAddr storage.Address) (uint32, error) {
 	return rsrc.version, nil
 }
 
-// \TODO should be hashsize * branches from the chosen chunker, implement with FileStore
-func (h *Handler) chunkSize() int64 {
-	return chunkSize
-}
-
 // New creates a new metadata chunk out of the request passed in.
 func (h *Handler) New(ctx context.Context, request *Request) error {
 
@@ -469,7 +460,7 @@ func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAd
 	log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.SData, "multihash", r.multihash)
 
 	// update our resources map entry if the new update is older than the one we have, if we have it.
-	if rsrc != nil && r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version) {
+	if rsrc != nil && (r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version)) {
 		rsrc.period = r.period
 		rsrc.version = r.version
 		rsrc.data = make([]byte, len(r.data))

swarm/storage/mru/resource_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
@@ -776,14 +777,11 @@ func TestValidatorInStore(t *testing.T) {
 
 	// set up resource handler and add is as a validator to the localstore
 	rhParams := &HandlerParams{}
-	rh, err := NewHandler(rhParams)
+	rh := NewHandler(rhParams)
-	if err != nil {
-		t.Fatal(err)
-	}
 	store.Validators = append(store.Validators, rh)
 
 	// create content addressed chunks, one good, one faulty
-	chunks := storage.GenerateRandomChunks(storage.DefaultChunkSize, 2)
+	chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk := chunks[0]
 	badChunk := chunks[1]
 	badChunk.SData = goodChunk.SData

swarm/storage/mru/testutil.go

@@ -38,10 +38,7 @@ func (t *TestHandler) Close() {
 // NewTestHandler creates Handler object to be used for testing purposes.
 func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) {
 	path := filepath.Join(datadir, testDbDirName)
-	rh, err := NewHandler(params)
+	rh := NewHandler(params)
-	if err != nil {
-		return nil, fmt.Errorf("resource handler create fail: %v", err)
-	}
 	localstoreparams := storage.NewDefaultLocalStoreParams()
 	localstoreparams.Init(path)
 	localStore, err := storage.NewLocalStore(localstoreparams, nil)

swarm/storage/mru/update.go

@@ -20,6 +20,7 @@ import (
 	"encoding/binary"
 	"errors"
 
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/multihash"
 )
@@ -42,7 +43,7 @@ const chunkPrefixLength = 2 + 2
 //
 // Minimum size is Header + 1 (minimum data length, enforced)
 const minimumUpdateDataLength = updateHeaderLength + 1
-const maxUpdateDataLength = chunkSize - signatureLength - updateHeaderLength - chunkPrefixLength
+const maxUpdateDataLength = chunk.DefaultSize - signatureLength - updateHeaderLength - chunkPrefixLength
 
 // binaryPut serializes the resource update information into the given slice
 func (r *resourceUpdate) binaryPut(serializedData []byte) error {

swarm/storage/pyramid.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 )
 
@@ -101,11 +102,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
 	New chunks to store are store using the putter which the caller provides.
 */
 func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
-	return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx)
+	return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx)
 }
 
 func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
-	return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx)
+	return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx)
 }
 
 // Entry to create a tree node

swarm/storage/types.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/swarm/bmt"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 )
 
 const MaxPO = 16
@@ -114,7 +115,9 @@ func MakeHashFunc(hash string) SwarmHasher {
 	case "BMT":
 		return func() SwarmHash {
 			hasher := sha3.NewKeccak256
-			pool := bmt.NewTreePool(hasher, bmt.SegmentCount, bmt.PoolSize)
+			hasherSize := hasher().Size()
+			segmentCount := chunk.DefaultSize / hasherSize
+			pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize)
 			return bmt.New(pool)
 		}
 	}
@@ -230,8 +233,8 @@ func GenerateRandomChunk(dataSize int64) *Chunk {
 func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) {
 	var i int
 	hasher := MakeHashFunc(DefaultHash)()
-	if dataSize > DefaultChunkSize {
+	if dataSize > chunk.DefaultSize {
-		dataSize = DefaultChunkSize
+		dataSize = chunk.DefaultSize
 	}
 
 	for i = 0; i < count; i++ {
@@ -345,6 +348,10 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
 
 // Validate that the given key is a valid content address for the given data
 func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool {
+	if l := len(data); l < 9 || l > chunk.DefaultSize+8 {
+		return false
+	}
+
 	hasher := v.Hasher()
 	hasher.ResetWithLength(data[:8])
 	hasher.Write(data[8:])

swarm/swarm.go

@@ -143,6 +143,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 		OverlayAddr:  addr.OAddr,
 		UnderlayAddr: addr.UAddr,
 		HiveParams:   config.HiveParams,
+		LightNode:    config.LightNodeEnabled,
 	}
 
 	stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db"))
@@ -194,18 +195,13 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 	var resourceHandler *mru.Handler
 	rhparams := &mru.HandlerParams{}
 
-	resourceHandler, err = mru.NewHandler(rhparams)
+	resourceHandler = mru.NewHandler(rhparams)
-	if err != nil {
-		return nil, err
-	}
 	resourceHandler.SetStore(netStore)
 
-	var validators []storage.ChunkValidator
+	self.lstore.Validators = []storage.ChunkValidator{
-	validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)))
+		storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)),
-	if resourceHandler != nil {
+		resourceHandler,
-		validators = append(validators, resourceHandler)
 	}
-	self.lstore.Validators = validators
 
 	// setup local store
 	log.Debug(fmt.Sprintf("Set up local storage"))

trie/database.go

@@ -51,7 +51,7 @@ const secureKeyLength = 11 + 32
 
 // DatabaseReader wraps the Get and Has method of a backing store for the trie.
 type DatabaseReader interface {
-	// Get retrieves the value associated with key form the database.
+	// Get retrieves the value associated with key from the database.
 	Get(key []byte) (value []byte, err error)
 
 	// Has retrieves whether a key is present in the database.
@@ -431,6 +431,11 @@ func (db *Database) reference(child common.Hash, parent common.Hash) {
 
 // Dereference removes an existing reference from a root node.
 func (db *Database) Dereference(root common.Hash) {
+	// Sanity check to ensure that the meta-root is not removed
+	if root == (common.Hash{}) {
+		log.Error("Attempted to dereference the trie cache meta root")
+		return
+	}
 	db.lock.Lock()
 	defer db.lock.Unlock()
 

trie/node.go

@@ -55,7 +55,7 @@ var nilValueNode = valueNode(nil)
 func (n *fullNode) EncodeRLP(w io.Writer) error {
 	var nodes [17]node
 
-	for i, child := range n.Children {
+	for i, child := range &n.Children {
 		if child != nil {
 			nodes[i] = child
 		} else {
@@ -98,7 +98,7 @@ func (n valueNode) String() string  { return n.fstring("") }
 
 func (n *fullNode) fstring(ind string) string {
 	resp := fmt.Sprintf("[\n%s  ", ind)
-	for i, node := range n.Children {
+	for i, node := range &n.Children {
 		if node == nil {
 			resp += fmt.Sprintf("%s: <nil> ", indices[i])
 		} else {

trie/trie.go

@@ -356,7 +356,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 		// value that is left in n or -2 if n contains at least two
 		// values.
 		pos := -1
-		for i, cld := range n.Children {
+		for i, cld := range &n.Children {
 			if cld != nil {
 				if pos == -1 {
 					pos = i

whisper/whisperv5/filter.go

@@ -220,7 +220,7 @@ func matchSingleTopic(topic TopicType, bt []byte) bool {
 		bt = bt[:TopicLength]
 	}
 
-	if len(bt) < TopicLength {
+	if len(bt) == 0 {
 		return false
 	}
 

whisper/whisperv5/filter_test.go

@@ -829,16 +829,16 @@ func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) {
 	}
 }
 
-func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) {
+func TestMatchSingleTopic_PartialTopic_ReturnTrue(t *testing.T) {
 	bt := []byte("tes")
-	topic := BytesToTopic(bt)
+	topic := BytesToTopic([]byte("test"))
 
-	if matchSingleTopic(topic, bt) {
+	if !matchSingleTopic(topic, bt) {
 		t.FailNow()
 	}
 }
 
-func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) {
+func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) {
 	bt := []byte("test")
 	topic := BytesToTopic([]byte("not_equal"))
 

whisper/whisperv6/doc.go

@@ -33,7 +33,6 @@ particularly the notion of singular endpoints.
 package whisperv6
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -79,12 +78,6 @@ const (
 	DefaultSyncAllowance = 10 // seconds
 )
 
-type unknownVersionError uint64
-
-func (e unknownVersionError) Error() string {
-	return fmt.Sprintf("invalid envelope version %d", uint64(e))
-}
-
 // MailServer represents a mail server, capable of
 // archiving the old messages for subsequent delivery
 // to the peers. Any implementation must ensure that both

whisper/whisperv6/filter.go

@@ -250,23 +250,6 @@ func (f *Filter) MatchEnvelope(envelope *Envelope) bool {
 	return f.PoW <= 0 || envelope.pow >= f.PoW
 }
 
-func matchSingleTopic(topic TopicType, bt []byte) bool {
-	if len(bt) > TopicLength {
-		bt = bt[:TopicLength]
-	}
-
-	if len(bt) < TopicLength {
-		return false
-	}
-
-	for j, b := range bt {
-		if topic[j] != b {
-			return false
-		}
-	}
-	return true
-}
-
 // IsPubKeyEqual checks that two public keys are equal
 func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool {
 	if !ValidatePublicKey(a) {