Merge pull request #1 from ethereum/master

Merge latest geth changes

.travis.yml

@@ -8,7 +8,6 @@ matrix:
       sudo: required
       go: 1.7.x
       script:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
@@ -20,7 +19,6 @@ matrix:
       sudo: required
       go: 1.8.x
       script:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
@@ -33,7 +31,6 @@ matrix:
       sudo: required
       go: 1.9.x
       script:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
@@ -42,7 +39,6 @@ matrix:
 
     - os: osx
       go: 1.9.x
-      sudo: required
       script:
         - brew update
         - brew install caskroom/cask/brew-cask
@@ -53,15 +49,12 @@ matrix:
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      sudo: required
       go: 1.9.x
       env:
         - lint
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
-        - sudo modprobe fuse
-        - sudo chmod 666 /dev/fuse
-        - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA and Linux Azure uploads
@@ -72,6 +65,8 @@ matrix:
       env:
         - ubuntu-ppa
         - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
       addons:
         apt:
           packages:
@@ -104,12 +99,13 @@ matrix:
     # This builder does the Linux Azure MIPS xgo uploads
     - os: linux
       dist: trusty
-      sudo: required
       services:
         - docker
       go: 1.9.x
       env:
         - azure-linux-mips
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v
         - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done
@@ -146,6 +142,8 @@ matrix:
       env:
         - azure-android
         - maven-android
+      git:
+        submodules: false # avoid cloning ethereum/tests
       before_install:
         - curl https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
@@ -169,6 +167,8 @@ matrix:
         - azure-osx
         - azure-ios
         - cocoapods-ios
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
@@ -193,15 +193,11 @@ matrix:
       go: 1.9.x
       env:
         - azure-purge
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
 
-install:
-  - go get golang.org/x/tools/cmd/cover
-script:
-  - go run build/ci.go install
-  - go run build/ci.go test -coverage
-
 notifications:
   webhooks:
     urls:

cmd/swarm/config.go

@@ -0,0 +1,321 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strconv"
+	"unicode"
+
+	cli "gopkg.in/urfave/cli.v1"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/naoina/toml"
+
+	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
+)
+
+var (
+	//flag definition for the dumpconfig command
+	DumpConfigCommand = cli.Command{
+		Action:      utils.MigrateFlags(dumpConfig),
+		Name:        "dumpconfig",
+		Usage:       "Show configuration values",
+		ArgsUsage:   "",
+		Flags:       app.Flags,
+		Category:    "MISCELLANEOUS COMMANDS",
+		Description: `The dumpconfig command shows configuration values.`,
+	}
+
+	//flag definition for the config file command
+	SwarmTomlConfigPathFlag = cli.StringFlag{
+		Name:  "config",
+		Usage: "TOML configuration file",
+	}
+)
+
+//constants for environment variables
+const (
+	SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
+	SWARM_ENV_ACCOUNT         = "SWARM_ACCOUNT"
+	SWARM_ENV_LISTEN_ADDR     = "SWARM_LISTEN_ADDR"
+	SWARM_ENV_PORT            = "SWARM_PORT"
+	SWARM_ENV_NETWORK_ID      = "SWARM_NETWORK_ID"
+	SWARM_ENV_SWAP_ENABLE     = "SWARM_SWAP_ENABLE"
+	SWARM_ENV_SWAP_API        = "SWARM_SWAP_API"
+	SWARM_ENV_SYNC_ENABLE     = "SWARM_SYNC_ENABLE"
+	SWARM_ENV_ENS_API         = "SWARM_ENS_API"
+	SWARM_ENV_ENS_ADDR        = "SWARM_ENS_ADDR"
+	SWARM_ENV_CORS            = "SWARM_CORS"
+	SWARM_ENV_BOOTNODES       = "SWARM_BOOTNODES"
+	GETH_ENV_DATADIR          = "GETH_DATADIR"
+)
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+	NormFieldName: func(rt reflect.Type, key string) string {
+		return key
+	},
+	FieldToKey: func(rt reflect.Type, field string) string {
+		return field
+	},
+	MissingField: func(rt reflect.Type, field string) error {
+		link := ""
+		if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+			link = fmt.Sprintf(", check github.com/ethereum/go-ethereum/swarm/api/config.go for available fields")
+		}
+		return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+	},
+}
+
+//before booting the swarm node, build the configuration
+func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) {
+	//check for deprecated flags
+	checkDeprecated(ctx)
+	//start by creating a default config
+	config = bzzapi.NewDefaultConfig()
+	//first load settings from config file (if provided)
+	config, err = configFileOverride(config, ctx)
+	//override settings provided by environment variables
+	config = envVarsOverride(config)
+	//override settings provided by command line
+	config = cmdLineOverride(config, ctx)
+
+	return
+}
+
+//finally, after the configuration build phase is finished, initialize
+func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
+	//at this point, all vars should be set in the Config
+	//get the account for the provided swarm account
+	prvkey := getAccount(config.BzzAccount, ctx, stack)
+	//set the resolved config path (geth --datadir)
+	config.Path = stack.InstanceDir()
+	//finally, initialize the configuration
+	config.Init(prvkey)
+	//configuration phase completed here
+	log.Debug("Starting Swarm with the following parameters:")
+	//after having created the config, print it to screen
+	log.Debug(printConfig(config))
+}
+
+//override the current config with whatever is in the config file, if a config file has been provided
+func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) {
+	var err error
+
+	//only do something if the -config flag has been set
+	if ctx.GlobalIsSet(SwarmTomlConfigPathFlag.Name) {
+		var filepath string
+		if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" {
+			utils.Fatalf("Config file flag provided with invalid file path")
+		}
+		f, err := os.Open(filepath)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+
+		//decode the TOML file into a Config struct
+		//note that we are decoding into the existing defaultConfig;
+		//if an entry is not present in the file, the default entry is kept
+		err = tomlSettings.NewDecoder(f).Decode(&config)
+		// Add file name to errors that have a line number.
+		if _, ok := err.(*toml.LineError); ok {
+			err = errors.New(filepath + ", " + err.Error())
+		}
+	}
+	return config, err
+}
+
+//override the current config with whatever is provided through the command line
+//most values are not allowed a zero value (empty string), if not otherwise noted
+func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Config {
+
+	if keyid := ctx.GlobalString(SwarmAccountFlag.Name); keyid != "" {
+		currentConfig.BzzAccount = keyid
+	}
+
+	if chbookaddr := ctx.GlobalString(ChequebookAddrFlag.Name); chbookaddr != "" {
+		currentConfig.Contract = common.HexToAddress(chbookaddr)
+	}
+
+	if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
+		if id, _ := strconv.Atoi(networkid); id != 0 {
+			currentConfig.NetworkId = uint64(id)
+		}
+	}
+
+	if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
+		if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" {
+			currentConfig.Path = datadir
+		}
+	}
+
+	bzzport := ctx.GlobalString(SwarmPortFlag.Name)
+	if len(bzzport) > 0 {
+		currentConfig.Port = bzzport
+	}
+
+	if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" {
+		currentConfig.ListenAddr = bzzaddr
+	}
+
+	if ctx.GlobalIsSet(SwarmSwapEnabledFlag.Name) {
+		currentConfig.SwapEnabled = true
+	}
+
+	if ctx.GlobalIsSet(SwarmSyncEnabledFlag.Name) {
+		currentConfig.SyncEnabled = true
+	}
+
+	currentConfig.SwapApi = ctx.GlobalString(SwarmSwapAPIFlag.Name)
+	if currentConfig.SwapEnabled && currentConfig.SwapApi == "" {
+		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
+	}
+
+	//EnsApi can be set to "", so can't check for empty string, as it is allowed!
+	if ctx.GlobalIsSet(EnsAPIFlag.Name) {
+		currentConfig.EnsApi = ctx.GlobalString(EnsAPIFlag.Name)
+	}
+
+	if ensaddr := ctx.GlobalString(EnsAddrFlag.Name); ensaddr != "" {
+		currentConfig.EnsRoot = common.HexToAddress(ensaddr)
+	}
+
+	if cors := ctx.GlobalString(CorsStringFlag.Name); cors != "" {
+		currentConfig.Cors = cors
+	}
+
+	if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
+		currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name)
+	}
+
+	return currentConfig
+
+}
+
+//override the current config with whatver is provided in environment variables
+//most values are not allowed a zero value (empty string), if not otherwise noted
+func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
+
+	if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" {
+		currentConfig.BzzAccount = keyid
+	}
+
+	if chbookaddr := os.Getenv(SWARM_ENV_CHEQUEBOOK_ADDR); chbookaddr != "" {
+		currentConfig.Contract = common.HexToAddress(chbookaddr)
+	}
+
+	if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
+		if id, _ := strconv.Atoi(networkid); id != 0 {
+			currentConfig.NetworkId = uint64(id)
+		}
+	}
+
+	if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
+		currentConfig.Path = datadir
+	}
+
+	bzzport := os.Getenv(SWARM_ENV_PORT)
+	if len(bzzport) > 0 {
+		currentConfig.Port = bzzport
+	}
+
+	if bzzaddr := os.Getenv(SWARM_ENV_LISTEN_ADDR); bzzaddr != "" {
+		currentConfig.ListenAddr = bzzaddr
+	}
+
+	if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
+		if swap, err := strconv.ParseBool(swapenable); err != nil {
+			currentConfig.SwapEnabled = swap
+		}
+	}
+
+	if syncenable := os.Getenv(SWARM_ENV_SYNC_ENABLE); syncenable != "" {
+		if sync, err := strconv.ParseBool(syncenable); err != nil {
+			currentConfig.SyncEnabled = sync
+		}
+	}
+
+	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
+		currentConfig.SwapApi = swapapi
+	}
+
+	if currentConfig.SwapEnabled && currentConfig.SwapApi == "" {
+		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
+	}
+
+	//EnsApi can be set to "", so can't check for empty string, as it is allowed
+	if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists == true {
+		currentConfig.EnsApi = ensapi
+	}
+
+	if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" {
+		currentConfig.EnsRoot = common.HexToAddress(ensaddr)
+	}
+
+	if cors := os.Getenv(SWARM_ENV_CORS); cors != "" {
+		currentConfig.Cors = cors
+	}
+
+	if bootnodes := os.Getenv(SWARM_ENV_BOOTNODES); bootnodes != "" {
+		currentConfig.BootNodes = bootnodes
+	}
+
+	return currentConfig
+}
+
+// dumpConfig is the dumpconfig command.
+// writes a default config to STDOUT
+func dumpConfig(ctx *cli.Context) error {
+	cfg, err := buildConfig(ctx)
+	if err != nil {
+		utils.Fatalf(fmt.Sprintf("Uh oh - dumpconfig triggered an error %v", err))
+	}
+	comment := ""
+	out, err := tomlSettings.Marshal(&cfg)
+	if err != nil {
+		return err
+	}
+	io.WriteString(os.Stdout, comment)
+	os.Stdout.Write(out)
+	return nil
+}
+
+//deprecated flags checked here
+func checkDeprecated(ctx *cli.Context) {
+	// exit if the deprecated --ethapi flag is set
+	if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" {
+		utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.")
+	}
+}
+
+//print a Config as string
+func printConfig(config *bzzapi.Config) string {
+	out, err := tomlSettings.Marshal(&config)
+	if err != nil {
+		return (fmt.Sprintf("Something is not right with the configuration: %v", err))
+	}
+	return string(out)
+}

cmd/swarm/config_test.go

@@ -0,0 +1,459 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/api"
+
+	"github.com/docker/docker/pkg/reexec"
+)
+
+func TestDumpConfig(t *testing.T) {
+	swarm := runSwarm(t, "dumpconfig")
+	defaultConf := api.NewDefaultConfig()
+	out, err := tomlSettings.Marshal(&defaultConf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	swarm.Expect(string(out))
+	swarm.ExpectExit()
+}
+
+func TestFailsSwapEnabledNoSwapApi(t *testing.T) {
+	flags := []string{
+		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
+		fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545",
+		fmt.Sprintf("--%s", SwarmSwapEnabledFlag.Name),
+	}
+
+	swarm := runSwarm(t, flags...)
+	swarm.Expect("Fatal: " + SWARM_ERR_SWAP_SET_NO_API + "\n")
+	swarm.ExpectExit()
+}
+
+func TestFailsNoBzzAccount(t *testing.T) {
+	flags := []string{
+		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
+		fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545",
+	}
+
+	swarm := runSwarm(t, flags...)
+	swarm.Expect("Fatal: " + SWARM_ERR_NO_BZZACCOUNT + "\n")
+	swarm.ExpectExit()
+}
+
+func TestCmdLineOverrides(t *testing.T) {
+	dir, err := ioutil.TempDir("", "bzztest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	conf, account := getTestAccount(t, dir)
+	node := &testNode{Dir: dir}
+
+	// assign ports
+	httpPort, err := assignTCPPort()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	flags := []string{
+		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
+		fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort,
+		fmt.Sprintf("--%s", SwarmSyncEnabledFlag.Name),
+		fmt.Sprintf("--%s", CorsStringFlag.Name), "*",
+		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
+		"--datadir", dir,
+		"--ipcpath", conf.IPCPath,
+	}
+	node.Cmd = runSwarm(t, flags...)
+	node.Cmd.InputLine(testPassphrase)
+	defer func() {
+		if t.Failed() {
+			node.Shutdown()
+		}
+	}()
+	// wait for the node to start
+	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
+		node.Client, err = rpc.Dial(conf.IPCEndpoint())
+		if err == nil {
+			break
+		}
+	}
+	if node.Client == nil {
+		t.Fatal(err)
+	}
+
+	// load info
+	var info swarm.Info
+	if err := node.Client.Call(&info, "bzz_info"); err != nil {
+		t.Fatal(err)
+	}
+
+	if info.Port != httpPort {
+		t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
+	}
+
+	if info.NetworkId != 42 {
+		t.Fatalf("Expected network ID to be %d, got %d", 42, info.NetworkId)
+	}
+
+	if info.SyncEnabled != true {
+		t.Fatal("Expected Sync to be enabled, but is false")
+	}
+
+	if info.Cors != "*" {
+		t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors)
+	}
+
+	node.Shutdown()
+}
+
+func TestFileOverrides(t *testing.T) {
+
+	// assign ports
+	httpPort, err := assignTCPPort()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//create a config file
+	//first, create a default conf
+	defaultConf := api.NewDefaultConfig()
+	//change some values in order to test if they have been loaded
+	defaultConf.SyncEnabled = true
+	defaultConf.NetworkId = 54
+	defaultConf.Port = httpPort
+	defaultConf.StoreParams.DbCapacity = 9000000
+	defaultConf.ChunkerParams.Branches = 64
+	defaultConf.HiveParams.CallInterval = 6000000000
+	defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
+	defaultConf.SyncParams.KeyBufferSize = 512
+	//create a TOML string
+	out, err := tomlSettings.Marshal(&defaultConf)
+	if err != nil {
+		t.Fatalf("Error creating TOML file in TestFileOverride: %v", err)
+	}
+	//create file
+	f, err := ioutil.TempFile("", "testconfig.toml")
+	if err != nil {
+		t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
+	}
+	//write file
+	_, err = f.WriteString(string(out))
+	if err != nil {
+		t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
+	}
+	f.Sync()
+
+	dir, err := ioutil.TempDir("", "bzztest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	conf, account := getTestAccount(t, dir)
+	node := &testNode{Dir: dir}
+
+	flags := []string{
+		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
+		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
+		"--ens-api", "",
+		"--ipcpath", conf.IPCPath,
+		"--datadir", dir,
+	}
+	node.Cmd = runSwarm(t, flags...)
+	node.Cmd.InputLine(testPassphrase)
+	defer func() {
+		if t.Failed() {
+			node.Shutdown()
+		}
+	}()
+	// wait for the node to start
+	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
+		node.Client, err = rpc.Dial(conf.IPCEndpoint())
+		if err == nil {
+			break
+		}
+	}
+	if node.Client == nil {
+		t.Fatal(err)
+	}
+
+	// load info
+	var info swarm.Info
+	if err := node.Client.Call(&info, "bzz_info"); err != nil {
+		t.Fatal(err)
+	}
+
+	if info.Port != httpPort {
+		t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
+	}
+
+	if info.NetworkId != 54 {
+		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
+	}
+
+	if info.SyncEnabled != true {
+		t.Fatal("Expected Sync to be enabled, but is false")
+	}
+
+	if info.StoreParams.DbCapacity != 9000000 {
+		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
+	}
+
+	if info.ChunkerParams.Branches != 64 {
+		t.Fatalf("Expected chunker params branches to be %d, got %d", 64, info.ChunkerParams.Branches)
+	}
+
+	if info.HiveParams.CallInterval != 6000000000 {
+		t.Fatalf("Expected HiveParams CallInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.CallInterval))
+	}
+
+	if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second {
+		t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval)
+	}
+
+	if info.SyncParams.KeyBufferSize != 512 {
+		t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
+	}
+
+	node.Shutdown()
+}
+
+func TestEnvVars(t *testing.T) {
+	// assign ports
+	httpPort, err := assignTCPPort()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envVars := os.Environ()
+	envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmPortFlag.EnvVar, httpPort))
+	envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmNetworkIdFlag.EnvVar, "999"))
+	envVars = append(envVars, fmt.Sprintf("%s=%s", CorsStringFlag.EnvVar, "*"))
+	envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmSyncEnabledFlag.EnvVar, "true"))
+
+	dir, err := ioutil.TempDir("", "bzztest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	conf, account := getTestAccount(t, dir)
+	node := &testNode{Dir: dir}
+	flags := []string{
+		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
+		"--ens-api", "",
+		"--datadir", dir,
+		"--ipcpath", conf.IPCPath,
+	}
+
+	//node.Cmd = runSwarm(t,flags...)
+	//node.Cmd.cmd.Env = envVars
+	//the above assignment does not work, so we need a custom Cmd here in order to pass envVars:
+	cmd := &exec.Cmd{
+		Path:   reexec.Self(),
+		Args:   append([]string{"swarm-test"}, flags...),
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+	cmd.Env = envVars
+	//stdout, err := cmd.StdoutPipe()
+	//if err != nil {
+	//	t.Fatal(err)
+	//}
+	//stdout = bufio.NewReader(stdout)
+	var stdin io.WriteCloser
+	if stdin, err = cmd.StdinPipe(); err != nil {
+		t.Fatal(err)
+	}
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	//cmd.InputLine(testPassphrase)
+	io.WriteString(stdin, testPassphrase+"\n")
+	defer func() {
+		if t.Failed() {
+			node.Shutdown()
+			cmd.Process.Kill()
+		}
+	}()
+	// wait for the node to start
+	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
+		node.Client, err = rpc.Dial(conf.IPCEndpoint())
+		if err == nil {
+			break
+		}
+	}
+
+	if node.Client == nil {
+		t.Fatal(err)
+	}
+
+	// load info
+	var info swarm.Info
+	if err := node.Client.Call(&info, "bzz_info"); err != nil {
+		t.Fatal(err)
+	}
+
+	if info.Port != httpPort {
+		t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
+	}
+
+	if info.NetworkId != 999 {
+		t.Fatalf("Expected network ID to be %d, got %d", 999, info.NetworkId)
+	}
+
+	if info.Cors != "*" {
+		t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors)
+	}
+
+	if info.SyncEnabled != true {
+		t.Fatal("Expected Sync to be enabled, but is false")
+	}
+
+	node.Shutdown()
+	cmd.Process.Kill()
+}
+
+func TestCmdLineOverridesFile(t *testing.T) {
+
+	// assign ports
+	httpPort, err := assignTCPPort()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//create a config file
+	//first, create a default conf
+	defaultConf := api.NewDefaultConfig()
+	//change some values in order to test if they have been loaded
+	defaultConf.SyncEnabled = false
+	defaultConf.NetworkId = 54
+	defaultConf.Port = "8588"
+	defaultConf.StoreParams.DbCapacity = 9000000
+	defaultConf.ChunkerParams.Branches = 64
+	defaultConf.HiveParams.CallInterval = 6000000000
+	defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
+	defaultConf.SyncParams.KeyBufferSize = 512
+	//create a TOML file
+	out, err := tomlSettings.Marshal(&defaultConf)
+	if err != nil {
+		t.Fatalf("Error creating TOML file in TestFileOverride: %v", err)
+	}
+	//write file
+	f, err := ioutil.TempFile("", "testconfig.toml")
+	if err != nil {
+		t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
+	}
+	//write file
+	_, err = f.WriteString(string(out))
+	if err != nil {
+		t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
+	}
+	f.Sync()
+
+	dir, err := ioutil.TempDir("", "bzztest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	conf, account := getTestAccount(t, dir)
+	node := &testNode{Dir: dir}
+
+	expectNetworkId := uint64(77)
+
+	flags := []string{
+		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "77",
+		fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort,
+		fmt.Sprintf("--%s", SwarmSyncEnabledFlag.Name),
+		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
+		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
+		"--ens-api", "",
+		"--datadir", dir,
+		"--ipcpath", conf.IPCPath,
+	}
+	node.Cmd = runSwarm(t, flags...)
+	node.Cmd.InputLine(testPassphrase)
+	defer func() {
+		if t.Failed() {
+			node.Shutdown()
+		}
+	}()
+	// wait for the node to start
+	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
+		node.Client, err = rpc.Dial(conf.IPCEndpoint())
+		if err == nil {
+			break
+		}
+	}
+	if node.Client == nil {
+		t.Fatal(err)
+	}
+
+	// load info
+	var info swarm.Info
+	if err := node.Client.Call(&info, "bzz_info"); err != nil {
+		t.Fatal(err)
+	}
+
+	if info.Port != httpPort {
+		t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
+	}
+
+	if info.NetworkId != expectNetworkId {
+		t.Fatalf("Expected network ID to be %d, got %d", expectNetworkId, info.NetworkId)
+	}
+
+	if info.SyncEnabled != true {
+		t.Fatal("Expected Sync to be enabled, but is false")
+	}
+
+	if info.StoreParams.DbCapacity != 9000000 {
+		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
+	}
+
+	if info.ChunkerParams.Branches != 64 {
+		t.Fatalf("Expected chunker params branches to be %d, got %d", 64, info.ChunkerParams.Branches)
+	}
+
+	if info.HiveParams.CallInterval != 6000000000 {
+		t.Fatalf("Expected HiveParams CallInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.CallInterval))
+	}
+
+	if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second {
+		t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval)
+	}
+
+	if info.SyncParams.KeyBufferSize != 512 {
+		t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
+	}
+
+	node.Shutdown()
+}

cmd/swarm/main.go

@@ -48,6 +48,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
+
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -66,49 +67,58 @@ var (
 
 var (
 	ChequebookAddrFlag = cli.StringFlag{
-		Name:  "chequebook",
+		Name:   "chequebook",
-		Usage: "chequebook contract address",
+		Usage:  "chequebook contract address",
+		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
 	}
 	SwarmAccountFlag = cli.StringFlag{
-		Name:  "bzzaccount",
+		Name:   "bzzaccount",
-		Usage: "Swarm account key file",
+		Usage:  "Swarm account key file",
+		EnvVar: SWARM_ENV_ACCOUNT,
 	}
 	SwarmListenAddrFlag = cli.StringFlag{
-		Name:  "httpaddr",
+		Name:   "httpaddr",
-		Usage: "Swarm HTTP API listening interface",
+		Usage:  "Swarm HTTP API listening interface",
+		EnvVar: SWARM_ENV_LISTEN_ADDR,
 	}
 	SwarmPortFlag = cli.StringFlag{
-		Name:  "bzzport",
+		Name:   "bzzport",
-		Usage: "Swarm local http api port",
+		Usage:  "Swarm local http api port",
+		EnvVar: SWARM_ENV_PORT,
 	}
 	SwarmNetworkIdFlag = cli.IntFlag{
-		Name:  "bzznetworkid",
+		Name:   "bzznetworkid",
-		Usage: "Network identifier (integer, default 3=swarm testnet)",
+		Usage:  "Network identifier (integer, default 3=swarm testnet)",
+		EnvVar: SWARM_ENV_NETWORK_ID,
 	}
 	SwarmConfigPathFlag = cli.StringFlag{
 		Name:  "bzzconfig",
-		Usage: "Swarm config file path (datadir/bzz)",
+		Usage: "DEPRECATED: please use --config path/to/TOML-file",
 	}
 	SwarmSwapEnabledFlag = cli.BoolFlag{
-		Name:  "swap",
+		Name:   "swap",
-		Usage: "Swarm SWAP enabled (default false)",
+		Usage:  "Swarm SWAP enabled (default false)",
+		EnvVar: SWARM_ENV_SWAP_ENABLE,
 	}
 	SwarmSwapAPIFlag = cli.StringFlag{
-		Name:  "swap-api",
+		Name:   "swap-api",
-		Usage: "URL of the Ethereum API provider to use to settle SWAP payments",
+		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
+		EnvVar: SWARM_ENV_SWAP_API,
 	}
 	SwarmSyncEnabledFlag = cli.BoolTFlag{
-		Name:  "sync",
+		Name:   "sync",
-		Usage: "Swarm Syncing enabled (default true)",
+		Usage:  "Swarm Syncing enabled (default true)",
+		EnvVar: SWARM_ENV_SYNC_ENABLE,
 	}
 	EnsAPIFlag = cli.StringFlag{
-		Name:  "ens-api",
+		Name:   "ens-api",
-		Usage: "URL of the Ethereum API provider to use for ENS record lookups",
+		Usage:  "URL of the Ethereum API provider to use for ENS record lookups",
-		Value: node.DefaultIPCEndpoint("geth"),
+		EnvVar: SWARM_ENV_ENS_API,
 	}
 	EnsAddrFlag = cli.StringFlag{
-		Name:  "ens-addr",
+		Name:   "ens-addr",
-		Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)",
+		Usage:  "ENS contract address (default is detected as testnet or mainnet using --ens-api)",
+		EnvVar: SWARM_ENV_ENS_ADDR,
 	}
 	SwarmApiFlag = cli.StringFlag{
 		Name:  "bzzapi",
@@ -136,8 +146,9 @@ var (
 		Usage: "force mime type",
 	}
 	CorsStringFlag = cli.StringFlag{
-		Name:  "corsdomain",
+		Name:   "corsdomain",
-		Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
+		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
+		EnvVar: SWARM_ENV_CORS,
 	}
 
 	// the following flags are deprecated and should be removed in the future
@@ -147,6 +158,12 @@ var (
 	}
 )
 
+//declare a few constant error messages, useful for later error check comparisons in test
+var (
+	SWARM_ERR_NO_BZZACCOUNT   = "bzzaccount option is required but not set; check your config file, command line or environment variables"
+	SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set"
+)
+
 var defaultNodeConfig = node.DefaultConfig
 
 // This init function sets defaults so cmd/swarm can run alongside geth.
@@ -302,6 +319,8 @@ Remove corrupt entries from a local chunk database.
 DEPRECATED: use 'swarm db clean'.
 `,
 		},
+		// See config.go
+		DumpConfigCommand,
 	}
 	sort.Sort(cli.CommandsByName(app.Commands))
 
@@ -325,6 +344,7 @@ DEPRECATED: use 'swarm db clean'.
 		CorsStringFlag,
 		EnsAPIFlag,
 		EnsAddrFlag,
+		SwarmTomlConfigPathFlag,
 		SwarmConfigPathFlag,
 		SwarmSwapEnabledFlag,
 		SwarmSwapAPIFlag,
@@ -377,19 +397,32 @@ func version(ctx *cli.Context) error {
 }
 
 func bzzd(ctx *cli.Context) error {
-	// exit if the deprecated --ethapi flag is set
+	//build a valid bzzapi.Config from all available sources:
-	if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" {
+	//default config, file config, command line and env vars
-		utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.")
+	bzzconfig, err := buildConfig(ctx)
+	if err != nil {
+		utils.Fatalf("unable to configure swarm: %v", err)
 	}
 
 	cfg := defaultNodeConfig
+	//geth only supports --datadir via command line
+	//in order to be consistent within swarm, if we pass --datadir via environment variable
+	//or via config file, we get the same directory for geth and swarm
+	if _, err := os.Stat(bzzconfig.Path); err == nil {
+		cfg.DataDir = bzzconfig.Path
+	}
+	//setup the ethereum node
 	utils.SetNodeConfig(ctx, &cfg)
 	stack, err := node.New(&cfg)
 	if err != nil {
 		utils.Fatalf("can't create node: %v", err)
 	}
-
+	//a few steps need to be done after the config phase is completed,
-	registerBzzService(ctx, stack)
+	//due to overriding behavior
+	initSwarmNode(bzzconfig, stack, ctx)
+	//register BZZ as node.Service in the ethereum node
+	registerBzzService(bzzconfig, ctx, stack)
+	//start the node
 	utils.StartNode(stack)
 
 	go func() {
@@ -401,13 +434,12 @@ func bzzd(ctx *cli.Context) error {
 		stack.Stop()
 	}()
 
-	networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name)
 	// Add bootnodes as initial peers.
-	if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
+	if bzzconfig.BootNodes != "" {
-		bootnodes := strings.Split(ctx.GlobalString(utils.BootnodesFlag.Name), ",")
+		bootnodes := strings.Split(bzzconfig.BootNodes, ",")
 		injectBootnodes(stack.Server(), bootnodes)
 	} else {
-		if networkId == 3 {
+		if bzzconfig.NetworkId == 3 {
 			injectBootnodes(stack.Server(), testbetBootNodes)
 		}
 	}
@@ -448,61 +480,31 @@ func detectEnsAddr(client *rpc.Client) (common.Address, error) {
 	}
 }
 
-func registerBzzService(ctx *cli.Context, stack *node.Node) {
+func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.Node) {
-	prvkey := getAccount(ctx, stack)
-
-	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
-	bzzdir := ctx.GlobalString(SwarmConfigPathFlag.Name)
-	if bzzdir == "" {
-		bzzdir = stack.InstanceDir()
-	}
-
-	bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name))
-	if err != nil {
-		utils.Fatalf("unable to configure swarm: %v", err)
-	}
-	bzzport := ctx.GlobalString(SwarmPortFlag.Name)
-	if len(bzzport) > 0 {
-		bzzconfig.Port = bzzport
-	}
-	if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" {
-		bzzconfig.ListenAddr = bzzaddr
-	}
-	swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name)
-	syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name)
-
-	swapapi := ctx.GlobalString(SwarmSwapAPIFlag.Name)
-	if swapEnabled && swapapi == "" {
-		utils.Fatalf("SWAP is enabled but --swap-api is not set")
-	}
-
-	ensapi := ctx.GlobalString(EnsAPIFlag.Name)
-	ensAddr := ctx.GlobalString(EnsAddrFlag.Name)
-
-	cors := ctx.GlobalString(CorsStringFlag.Name)
 
+	//define the swarm service boot function
 	boot := func(ctx *node.ServiceContext) (node.Service, error) {
 		var swapClient *ethclient.Client
-		if swapapi != "" {
+		var err error
-			log.Info("connecting to SWAP API", "url", swapapi)
+		if bzzconfig.SwapApi != "" {
-			swapClient, err = ethclient.Dial(swapapi)
+			log.Info("connecting to SWAP API", "url", bzzconfig.SwapApi)
+			swapClient, err = ethclient.Dial(bzzconfig.SwapApi)
 			if err != nil {
-				return nil, fmt.Errorf("error connecting to SWAP API %s: %s", swapapi, err)
+				return nil, fmt.Errorf("error connecting to SWAP API %s: %s", bzzconfig.SwapApi, err)
 			}
 		}
 
 		var ensClient *ethclient.Client
-		if ensapi != "" {
+		if bzzconfig.EnsApi != "" {
-			log.Info("connecting to ENS API", "url", ensapi)
+			log.Info("connecting to ENS API", "url", bzzconfig.EnsApi)
-			client, err := rpc.Dial(ensapi)
+			client, err := rpc.Dial(bzzconfig.EnsApi)
 			if err != nil {
-				return nil, fmt.Errorf("error connecting to ENS API %s: %s", ensapi, err)
+				return nil, fmt.Errorf("error connecting to ENS API %s: %s", bzzconfig.EnsApi, err)
 			}
 			ensClient = ethclient.NewClient(client)
 
-			if ensAddr != "" {
+			//no ENS root address set yet
-				bzzconfig.EnsRoot = common.HexToAddress(ensAddr)
+			if bzzconfig.EnsRoot == (common.Address{}) {
-			} else {
 				ensAddr, err := detectEnsAddr(client)
 				if err == nil {
 					bzzconfig.EnsRoot = ensAddr
@@ -512,21 +514,21 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 			}
 		}
 
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, swapEnabled, syncEnabled, cors)
+		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors)
 	}
+	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {
 		utils.Fatalf("Failed to register the Swarm service: %v", err)
 	}
 }
 
-func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
+func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
-	keyid := ctx.GlobalString(SwarmAccountFlag.Name)
+	//an account is mandatory
-
+	if bzzaccount == "" {
-	if keyid == "" {
+		utils.Fatalf(SWARM_ERR_NO_BZZACCOUNT)
-		utils.Fatalf("Option %q is required", SwarmAccountFlag.Name)
 	}
 	// Try to load the arg as a hex key file.
-	if key, err := crypto.LoadECDSA(keyid); err == nil {
+	if key, err := crypto.LoadECDSA(bzzaccount); err == nil {
 		log.Info("Swarm account key loaded", "address", crypto.PubkeyToAddress(key.PublicKey))
 		return key
 	}
@@ -534,7 +536,7 @@ func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	am := stack.AccountManager()
 	ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
-	return decryptStoreAccount(ks, keyid, utils.MakePasswordList(ctx))
+	return decryptStoreAccount(ks, bzzaccount, utils.MakePasswordList(ctx))
 }
 
 func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey {
@@ -552,7 +554,7 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []stri
 		utils.Fatalf("Can't find swarm account key %s", account)
 	}
 	if err != nil {
-		utils.Fatalf("Can't find swarm account key: %v", err)
+		utils.Fatalf("Can't find swarm account key: %v - Is the provided bzzaccount(%s) from the right datadir/Path?", err, account)
 	}
 	keyjson, err := ioutil.ReadFile(a.URL.Path)
 	if err != nil {

cmd/swarm/run_test.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/internal/cmdtest"
 	"github.com/ethereum/go-ethereum/node"
@@ -156,9 +157,9 @@ type testNode struct {
 
 const testPassphrase = "swarm-test-passphrase"
 
-func newTestNode(t *testing.T, dir string) *testNode {
+func getTestAccount(t *testing.T, dir string) (conf *node.Config, account accounts.Account) {
 	// create key
-	conf := &node.Config{
+	conf = &node.Config{
 		DataDir: dir,
 		IPCPath: "bzzd.ipc",
 		NoUSB:   true,
@@ -167,18 +168,24 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	if err != nil {
 		t.Fatal(err)
 	}
-	account, err := n.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore).NewAccount(testPassphrase)
+	account, err = n.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore).NewAccount(testPassphrase)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	node := &testNode{Dir: dir}
-
 	// use a unique IPCPath when running tests on Windows
 	if runtime.GOOS == "windows" {
 		conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", account.Address.String())
 	}
 
+	return conf, account
+}
+
+func newTestNode(t *testing.T, dir string) *testNode {
+
+	conf, account := getTestAccount(t, dir)
+	node := &testNode{Dir: dir}
+
 	// assign ports
 	httpPort, err := assignTCPPort()
 	if err != nil {

cmd/utils/flags.go

@@ -746,6 +746,12 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error
 	if err != nil || index < 0 {
 		return accounts.Account{}, fmt.Errorf("invalid account address or index %q", account)
 	}
+	log.Warn("-------------------------------------------------------------------")
+	log.Warn("Referring to accounts by order in the keystore folder is dangerous!")
+	log.Warn("This functionality is deprecated and will be removed in the future!")
+	log.Warn("Please use explicit addresses! (can search via `geth account list`)")
+	log.Warn("-------------------------------------------------------------------")
+
 	accs := ks.Accounts()
 	if len(accs) <= index {
 		return accounts.Account{}, fmt.Errorf("index %d higher than number of accounts %d", index, len(accs))
@@ -762,15 +768,6 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) {
 			Fatalf("Option %q: %v", EtherbaseFlag.Name, err)
 		}
 		cfg.Etherbase = account.Address
-		return
-	}
-	accounts := ks.Accounts()
-	if (cfg.Etherbase == common.Address{}) {
-		if len(accounts) > 0 {
-			cfg.Etherbase = accounts[0].Address
-		} else {
-			log.Warn("No etherbase set and no accounts found as default")
-		}
 	}
 }
 

console/console.go

@@ -192,6 +192,7 @@ func (c *Console) init(preload []string) error {
 	if obj := admin.Object(); obj != nil { // make sure the admin api is enabled over the interface
 		obj.Set("sleepBlocks", bridge.SleepBlocks)
 		obj.Set("sleep", bridge.Sleep)
+		obj.Set("clearHistory", c.clearHistory)
 	}
 	// Preload any JavaScript files before starting the console
 	for _, path := range preload {
@@ -216,6 +217,16 @@ func (c *Console) init(preload []string) error {
 	return nil
 }
 
+func (c *Console) clearHistory() {
+	c.history = nil
+	c.prompter.ClearHistory()
+	if err := os.Remove(c.histPath); err != nil {
+		fmt.Fprintln(c.printer, "can't delete history file:", err)
+	} else {
+		fmt.Fprintln(c.printer, "history file deleted")
+	}
+}
+
 // consoleOutput is an override for the console.log and console.error methods to
 // stream the output into the configured output stream instead of stdout.
 func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value {

console/console_test.go

@@ -68,6 +68,7 @@ func (p *hookedPrompter) PromptConfirm(prompt string) (bool, error) {
 }
 func (p *hookedPrompter) SetHistory(history []string)              {}
 func (p *hookedPrompter) AppendHistory(command string)             {}
+func (p *hookedPrompter) ClearHistory()                            {}
 func (p *hookedPrompter) SetWordCompleter(completer WordCompleter) {}
 
 // tester is a console test environment for the console tests to operate on.

console/prompter.go

@@ -51,6 +51,9 @@ type UserPrompter interface {
 	// if and only if the prompt to append was a valid command.
 	AppendHistory(command string)
 
+	// ClearHistory clears the entire history
+	ClearHistory()
+
 	// SetWordCompleter sets the completion function that the prompter will call to
 	// fetch completion candidates when the user presses tab.
 	SetWordCompleter(completer WordCompleter)
@@ -158,6 +161,11 @@ func (p *terminalPrompter) AppendHistory(command string) {
 	p.State.AppendHistory(command)
 }
 
+// ClearHistory clears the entire history
+func (p *terminalPrompter) ClearHistory() {
+	p.State.ClearHistory()
+}
+
 // SetWordCompleter sets the completion function that the prompter will call to
 // fetch completion candidates when the user presses tab.
 func (p *terminalPrompter) SetWordCompleter(completer WordCompleter) {

core/types/transaction.go

@@ -137,7 +137,7 @@ func isProtectedV(V *big.Int) bool {
 	return true
 }
 
-// DecodeRLP implements rlp.Encoder
+// EncodeRLP implements rlp.Encoder
 func (tx *Transaction) EncodeRLP(w io.Writer) error {
 	return rlp.Encode(w, &tx.data)
 }

crypto/crypto.go

@@ -98,6 +98,9 @@ func toECDSA(d []byte, strict bool) (*ecdsa.PrivateKey, error) {
 	}
 	priv.D = new(big.Int).SetBytes(d)
 	priv.PublicKey.X, priv.PublicKey.Y = priv.PublicKey.Curve.ScalarBaseMult(d)
+	if priv.PublicKey.X == nil {
+		return nil, errors.New("invalid private key")
+	}
 	return priv, nil
 }
 

crypto/crypto_test.go

@@ -40,6 +40,15 @@ func TestKeccak256Hash(t *testing.T) {
 	checkhash(t, "Sha3-256-array", func(in []byte) []byte { h := Keccak256Hash(in); return h[:] }, msg, exp)
 }
 
+func TestToECDSAErrors(t *testing.T) {
+	if _, err := HexToECDSA("0000000000000000000000000000000000000000000000000000000000000000"); err == nil {
+		t.Fatal("HexToECDSA should've returned error")
+	}
+	if _, err := HexToECDSA("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); err == nil {
+		t.Fatal("HexToECDSA should've returned error")
+	}
+}
+
 func BenchmarkSha3(b *testing.B) {
 	a := []byte("hello world")
 	for i := 0; i < b.N; i++ {

eth/api.go

@@ -452,7 +452,12 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, logConfig *vm.LogConf
 	}
 	statedb, err := blockchain.StateAt(blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1).Root())
 	if err != nil {
-		return false, structLogger.StructLogs(), err
+		switch err.(type) {
+		case *trie.MissingNodeError:
+			return false, structLogger.StructLogs(), fmt.Errorf("required historical state unavailable")
+		default:
+			return false, structLogger.StructLogs(), err
+		}
 	}
 
 	receipts, _, usedGas, err := processor.Process(block, statedb, config)
@@ -518,7 +523,12 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common.
 	}
 	msg, context, statedb, err := api.computeTxEnv(blockHash, int(txIndex))
 	if err != nil {
-		return nil, err
+		switch err.(type) {
+		case *trie.MissingNodeError:
+			return nil, fmt.Errorf("required historical state unavailable")
+		default:
+			return nil, err
+		}
 	}
 
 	// Run the transaction with tracing enabled.

eth/backend.go

@@ -310,10 +310,17 @@ func (s *Ethereum) Etherbase() (eb common.Address, err error) {
 	}
 	if wallets := s.AccountManager().Wallets(); len(wallets) > 0 {
 		if accounts := wallets[0].Accounts(); len(accounts) > 0 {
-			return accounts[0].Address, nil
+			etherbase := accounts[0].Address
+
+			s.lock.Lock()
+			s.etherbase = etherbase
+			s.lock.Unlock()
+
+			log.Info("Etherbase automatically configured", "address", etherbase)
+			return etherbase, nil
 		}
 	}
-	return common.Address{}, fmt.Errorf("etherbase address must be explicitly specified")
+	return common.Address{}, fmt.Errorf("etherbase must be explicitly specified")
 }
 
 // set in js console via admin interface or wrapper from cli flags

mobile/big.go

@@ -62,6 +62,16 @@ func (bi *BigInt) SetInt64(x int64) {
 	bi.bigint.SetInt64(x)
 }
 
+// Sign returns:
+//
+//	-1 if x <  0
+//	 0 if x == 0
+//	+1 if x >  0
+//
+func (bi *BigInt) Sign() int {
+	return bi.bigint.Sign()
+}
+
 // SetString sets the big int to x.
 //
 // The string prefix determines the actual conversion base. A prefix of "0x" or

rlp/raw.go

@@ -98,7 +98,7 @@ func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) {
 		tagsize = 1
 		contentsize = uint64(b - 0x80)
 		// Reject strings that should've been single bytes.
-		if contentsize == 1 && buf[1] < 128 {
+		if contentsize == 1 && len(buf) > 1 && buf[1] < 128 {
 			return 0, 0, 0, ErrCanonSize
 		}
 	case b < 0xC0:

rlp/raw_test.go

@@ -96,6 +96,7 @@ func TestSplit(t *testing.T) {
 		{input: "F90055", err: ErrCanonSize, rest: "F90055"},
 		{input: "FA0002FFFF", err: ErrCanonSize, rest: "FA0002FFFF"},
 
+		{input: "81", err: ErrValueTooLarge, rest: "81"},
 		{input: "8501010101", err: ErrValueTooLarge, rest: "8501010101"},
 		{input: "C60607080902", err: ErrValueTooLarge, rest: "C60607080902"},
 

swarm/api/config.go

@@ -18,15 +18,15 @@ package api
 
 import (
 	"crypto/ecdsa"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/services/swap"
 	"github.com/ethereum/go-ethereum/swarm/storage"
@@ -46,101 +46,68 @@ type Config struct {
 	*network.HiveParams
 	Swap *swap.SwapParams
 	*network.SyncParams
-	Path       string
+	Contract    common.Address
-	ListenAddr string
+	EnsRoot     common.Address
-	Port       string
+	EnsApi      string
-	PublicKey  string
+	Path        string
-	BzzKey     string
+	ListenAddr  string
-	EnsRoot    common.Address
+	Port        string
-	NetworkId  uint64
+	PublicKey   string
+	BzzKey      string
+	NetworkId   uint64
+	SwapEnabled bool
+	SyncEnabled bool
+	SwapApi     string
+	Cors        string
+	BzzAccount  string
+	BootNodes   string
 }
 
-// config is agnostic to where private key is coming from
+//create a default config with all parameters to set to defaults
-// so managing accounts is outside swarm and left to wrappers
+func NewDefaultConfig() (self *Config) {
-func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, networkId uint64) (self *Config, err error) {
-	address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address
-	dirpath := filepath.Join(path, "bzz-"+common.Bytes2Hex(address.Bytes()))
-	err = os.MkdirAll(dirpath, os.ModePerm)
-	if err != nil {
-		return
-	}
-	confpath := filepath.Join(dirpath, "config.json")
-	var data []byte
-	pubkey := crypto.FromECDSAPub(&prvKey.PublicKey)
-	pubkeyhex := common.ToHex(pubkey)
-	keyhex := crypto.Keccak256Hash(pubkey).Hex()
 
 	self = &Config{
-		SyncParams:    network.NewSyncParams(dirpath),
+		StoreParams:   storage.NewDefaultStoreParams(),
-		HiveParams:    network.NewHiveParams(dirpath),
 		ChunkerParams: storage.NewChunkerParams(),
-		StoreParams:   storage.NewStoreParams(dirpath),
+		HiveParams:    network.NewDefaultHiveParams(),
+		SyncParams:    network.NewDefaultSyncParams(),
+		Swap:          swap.NewDefaultSwapParams(),
 		ListenAddr:    DefaultHTTPListenAddr,
 		Port:          DefaultHTTPPort,
-		Path:          dirpath,
+		Path:          node.DefaultDataDir(),
-		Swap:          swap.DefaultSwapParams(contract, prvKey),
+		EnsApi:        node.DefaultIPCEndpoint("geth"),
-		PublicKey:     pubkeyhex,
-		BzzKey:        keyhex,
 		EnsRoot:       ens.TestNetAddress,
-		NetworkId:     networkId,
+		NetworkId:     network.NetworkId,
-	}
+		SwapEnabled:   false,
-	data, err = ioutil.ReadFile(confpath)
+		SyncEnabled:   true,
-
+		SwapApi:       "",
-	// if not set in function param, then set default for swarm network, will be overwritten by config file if present
+		BootNodes:     "",
-	if networkId == 0 {
-		self.NetworkId = network.NetworkId
-	}
-
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return
-		}
-
-		// file does not exist
-		// write out config file
-		err = self.Save()
-		if err != nil {
-			err = fmt.Errorf("error writing config: %v", err)
-		}
-		return
-	}
-
-	// file exists, deserialise
-	err = json.Unmarshal(data, self)
-	if err != nil {
-		return nil, fmt.Errorf("unable to parse config: %v", err)
-	}
-	// check public key
-	if pubkeyhex != self.PublicKey {
-		return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey)
-	}
-	if keyhex != self.BzzKey {
-		return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey)
-	}
-
-	// if set in function param, replace id set from config file
-	if networkId != 0 {
-		self.NetworkId = networkId
-	}
-
-	self.Swap.SetKey(prvKey)
-
-	if (self.EnsRoot == common.Address{}) {
-		self.EnsRoot = ens.TestNetAddress
 	}
 
 	return
 }
 
-func (self *Config) Save() error {
+//some config params need to be initialized after the complete
-	data, err := json.MarshalIndent(self, "", "    ")
+//config building phase is completed (e.g. due to overriding flags)
+func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
+
+	address := crypto.PubkeyToAddress(prvKey.PublicKey)
+	self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
+	err := os.MkdirAll(self.Path, os.ModePerm)
 	if err != nil {
-		return err
+		log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err))
+		return
 	}
-	err = os.MkdirAll(self.Path, os.ModePerm)
+
-	if err != nil {
+	pubkey := crypto.FromECDSAPub(&prvKey.PublicKey)
-		return err
+	pubkeyhex := common.ToHex(pubkey)
-	}
+	keyhex := crypto.Keccak256Hash(pubkey).Hex()
-	confpath := filepath.Join(self.Path, "config.json")
+
-	return ioutil.WriteFile(confpath, data, os.ModePerm)
+	self.PublicKey = pubkeyhex
+	self.BzzKey = keyhex
+
+	self.Swap.Init(self.Contract, prvKey)
+	self.SyncParams.Init(self.Path)
+	self.HiveParams.Init(self.Path)
+	self.StoreParams.Init(self.Path)
 }

swarm/api/config_test.go

@@ -17,109 +17,53 @@
 package api
 
 import (
-	"io/ioutil"
+	"reflect"
-	"os"
-	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-var (
+func TestConfig(t *testing.T) {
-	hexprvkey     = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c"
-	defaultConfig = `{
-    "ChunkDbPath": "` + filepath.Join("TMPDIR", "chunks") + `",
-    "DbCapacity": 5000000,
-    "CacheCapacity": 5000,
-    "Radius": 0,
-    "Branches": 128,
-    "Hash": "SHA3",
-    "CallInterval": 3000000000,
-    "KadDbPath": "` + filepath.Join("TMPDIR", "bzz-peers.json") + `",
-    "MaxProx": 8,
-    "ProxBinSize": 2,
-    "BucketSize": 4,
-    "PurgeInterval": 151200000000000,
-    "InitialRetryInterval": 42000000,
-    "MaxIdleInterval": 42000000000,
-    "ConnRetryExp": 2,
-    "Swap": {
-        "BuyAt": 20000000000,
-        "SellAt": 20000000000,
-        "PayAt": 100,
-        "DropAt": 10000,
-        "AutoCashInterval": 300000000000,
-        "AutoCashThreshold": 50000000000000,
-        "AutoDepositInterval": 300000000000,
-        "AutoDepositThreshold": 50000000000000,
-        "AutoDepositBuffer": 100000000000000,
-        "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3",
-        "Contract": "0x0000000000000000000000000000000000000000",
-        "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e"
-    },
-    "RequestDbPath": "` + filepath.Join("TMPDIR", "requests") + `",
-    "RequestDbBatchSize": 512,
-    "KeyBufferSize": 1024,
-    "SyncBatchSize": 128,
-    "SyncBufferSize": 128,
-    "SyncCacheSize": 1024,
-    "SyncPriorities": [
-        2,
-        1,
-        1,
-        0,
-        0
-    ],
-    "SyncModes": [
-        true,
-        true,
-        true,
-        true,
-        false
-    ],
-    "Path": "TMPDIR",
-    "ListenAddr": "127.0.0.1",
-    "Port": "8500",
-    "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3",
-    "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81",
-    "EnsRoot": "0x112234455c3a32fd11230c42e7bccd4a84e02010",
-    "NetworkId": 323
-}`
-)
 
-func TestConfigWriteRead(t *testing.T) {
+	var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c"
-	tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
 
 	prvkey, err := crypto.HexToECDSA(hexprvkey)
 	if err != nil {
 		t.Fatalf("failed to load private key: %v", err)
 	}
-	orig, err := NewConfig(tmp, common.Address{}, prvkey, 323)
+
-	if err != nil {
+	one := NewDefaultConfig()
-		t.Fatalf("expected no error, got %v", err)
+	two := NewDefaultConfig()
-	}
+
-	data, err := ioutil.ReadFile(filepath.Join(orig.Path, "config.json"))
+	if equal := reflect.DeepEqual(one, two); equal == false {
-	if err != nil {
+		t.Fatal("Two default configs are not equal")
-		t.Fatalf("default config file cannot be read: %v", err)
-	}
-	exp := strings.Replace(defaultConfig, "TMPDIR", orig.Path, -1)
-	exp = strings.Replace(exp, "\\", "\\\\", -1)
-	if string(data) != exp {
-		t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data))
 	}
 
-	conf, err := NewConfig(tmp, common.Address{}, prvkey, 323)
+	one.Init(prvkey)
-	if err != nil {
+
-		t.Fatalf("expected no error, got %v", err)
+	//the init function should set the following fields
+	if one.BzzKey == "" {
+		t.Fatal("Expected BzzKey to be set")
 	}
-	if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() {
+	if one.PublicKey == "" {
-		t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex())
+		t.Fatal("Expected PublicKey to be set")
 	}
 
+	//the Init function should append subdirs to the given path
+	if one.Swap.PayProfile.Beneficiary == (common.Address{}) {
+		t.Fatal("Failed to correctly initialize SwapParams")
+	}
+
+	if one.SyncParams.RequestDbPath == one.Path {
+		t.Fatal("Failed to correctly initialize SyncParams")
+	}
+
+	if one.HiveParams.KadDbPath == one.Path {
+		t.Fatal("Failed to correctly initialize HiveParams")
+	}
+
+	if one.StoreParams.ChunkDbPath == one.Path {
+		t.Fatal("Failed to correctly initialize StoreParams")
+	}
 }

swarm/network/hive.go

@@ -70,19 +70,25 @@ type HiveParams struct {
 	*kademlia.KadParams
 }
 
-func NewHiveParams(path string) *HiveParams {
+//create default params
-	kad := kademlia.NewKadParams()
+func NewDefaultHiveParams() *HiveParams {
+	kad := kademlia.NewDefaultKadParams()
 	// kad.BucketSize = bucketSize
 	// kad.MaxProx = maxProx
 	// kad.ProxBinSize = proxBinSize
 
 	return &HiveParams{
 		CallInterval: callInterval,
-		KadDbPath:    filepath.Join(path, "bzz-peers.json"),
 		KadParams:    kad,
 	}
 }
 
+//this can only finally be set after all config options (file, cmd line, env vars)
+//have been evaluated
+func (self *HiveParams) Init(path string) {
+	self.KadDbPath = filepath.Join(path, "bzz-peers.json")
+}
+
 func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive {
 	kad := kademlia.New(kademlia.Address(addr), params.KadParams)
 	return &Hive{

swarm/network/kademlia/kademlia.go

@@ -52,7 +52,7 @@ type KadParams struct {
 	ConnRetryExp         int
 }
 
-func NewKadParams() *KadParams {
+func NewDefaultKadParams() *KadParams {
 	return &KadParams{
 		MaxProx:              maxProx,
 		ProxBinSize:          proxBinSize,

swarm/network/kademlia/kademlia_test.go

@@ -63,7 +63,7 @@ func TestOn(t *testing.T) {
 	if !ok1 || !ok2 {
 		t.Errorf("oops")
 	}
-	kad := New(addr, NewKadParams())
+	kad := New(addr, NewDefaultKadParams())
 	err := kad.On(&testNode{addr: other}, nil)
 	_ = err
 }
@@ -72,7 +72,7 @@ func TestBootstrap(t *testing.T) {
 
 	test := func(test *bootstrapTest) bool {
 		// for any node kad.le, Target and N
-		params := NewKadParams()
+		params := NewDefaultKadParams()
 		params.MaxProx = test.MaxProx
 		params.BucketSize = test.BucketSize
 		params.ProxBinSize = test.BucketSize
@@ -127,7 +127,7 @@ func TestFindClosest(t *testing.T) {
 
 	test := func(test *FindClosestTest) bool {
 		// for any node kad.le, Target and N
-		params := NewKadParams()
+		params := NewDefaultKadParams()
 		params.MaxProx = 7
 		kad := New(test.Self, params)
 		var err error
@@ -198,7 +198,7 @@ var (
 func TestProxAdjust(t *testing.T) {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	self := gen(Address{}, r).(Address)
-	params := NewKadParams()
+	params := NewDefaultKadParams()
 	params.MaxProx = 7
 	kad := New(self, params)
 
@@ -232,7 +232,7 @@ func TestSaveLoad(t *testing.T) {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	addresses := gen([]Address{}, r).([]Address)
 	self := RandomAddress()
-	params := NewKadParams()
+	params := NewDefaultKadParams()
 	params.MaxProx = 7
 	kad := New(self, params)
 

swarm/network/syncer.go

@@ -131,9 +131,8 @@ type SyncParams struct {
 }
 
 // constructor with default values
-func NewSyncParams(bzzdir string) *SyncParams {
+func NewDefaultSyncParams() *SyncParams {
 	return &SyncParams{
-		RequestDbPath:      filepath.Join(bzzdir, "requests"),
 		RequestDbBatchSize: requestDbBatchSize,
 		KeyBufferSize:      keyBufferSize,
 		SyncBufferSize:     syncBufferSize,
@@ -144,6 +143,12 @@ func NewSyncParams(bzzdir string) *SyncParams {
 	}
 }
 
+//this can only finally be set after all config options (file, cmd line, env vars)
+//have been evaluated
+func (self *SyncParams) Init(path string) {
+	self.RequestDbPath = filepath.Join(path, "requests")
+}
+
 // syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding
 type syncer struct {
 	*SyncParams                     // sync parameters

swarm/services/swap/swap.go

@@ -80,17 +80,10 @@ type PayProfile struct {
 	lock        sync.RWMutex
 }
 
-func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapParams {
+//create params with default values
-	pubkey := &prvkey.PublicKey
+func NewDefaultSwapParams() *SwapParams {
 	return &SwapParams{
-		PayProfile: &PayProfile{
+		PayProfile: &PayProfile{},
-			PublicKey:   common.ToHex(crypto.FromECDSAPub(pubkey)),
-			Contract:    contract,
-			Beneficiary: crypto.PubkeyToAddress(*pubkey),
-			privateKey:  prvkey,
-			publicKey:   pubkey,
-			owner:       crypto.PubkeyToAddress(*pubkey),
-		},
 		Params: &swap.Params{
 			Profile: &swap.Profile{
 				BuyAt:  buyAt,
@@ -109,6 +102,21 @@ func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapP
 	}
 }
 
+//this can only finally be set after all config options (file, cmd line, env vars)
+//have been evaluated
+func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) {
+	pubkey := &prvkey.PublicKey
+
+	self.PayProfile = &PayProfile{
+		PublicKey:   common.ToHex(crypto.FromECDSAPub(pubkey)),
+		Contract:    contract,
+		Beneficiary: crypto.PubkeyToAddress(*pubkey),
+		privateKey:  prvkey,
+		publicKey:   pubkey,
+		owner:       crypto.PubkeyToAddress(*pubkey),
+	}
+}
+
 // swap constructor, parameters
 // * global chequebook, assume deployed service and
 // * the balance is at buffer.

swarm/storage/netstore.go

@@ -57,15 +57,21 @@ type StoreParams struct {
 	Radius        int
 }
 
-func NewStoreParams(path string) (self *StoreParams) {
+//create params with default values
+func NewDefaultStoreParams() (self *StoreParams) {
 	return &StoreParams{
-		ChunkDbPath:   filepath.Join(path, "chunks"),
 		DbCapacity:    defaultDbCapacity,
 		CacheCapacity: defaultCacheCapacity,
 		Radius:        defaultRadius,
 	}
 }
 
+//this can only finally be set after all config options (file, cmd line, env vars)
+//have been evaluated
+func (self *StoreParams) Init(path string) {
+	self.ChunkDbPath = filepath.Join(path, "chunks")
+}
+
 // netstore contructor, takes path argument that is used to initialise dbStore,
 // the persistent (disk) storage component of LocalStore
 // the second argument is the hive, the connection/logistics manager for the node

swarm/swarm.go

@@ -220,7 +220,7 @@ func (self *Swarm) Start(srv *p2p.Server) error {
 // stops all component services.
 func (self *Swarm) Stop() error {
 	self.dpa.Stop()
-	self.hive.Stop()
+	err := self.hive.Stop()
 	if ch := self.config.Swap.Chequebook(); ch != nil {
 		ch.Stop()
 		ch.Save()
@@ -230,7 +230,7 @@ func (self *Swarm) Stop() error {
 		self.lstore.DbStore.Close()
 	}
 	self.sfs.Stop()
-	return self.config.Save()
+	return err
 }
 
 // implements the node.Service interface
@@ -301,7 +301,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error {
 		return err
 	}
 	log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex()))
-	self.config.Save()
 	self.hive.DropAll()
 	return nil
 }
@@ -314,10 +313,9 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) {
 		return
 	}
 
-	config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId)
+	config := api.NewDefaultConfig()
-	if err != nil {
+	config.Path = datadir
-		return
+	config.Init(prvKey)
-	}
 	config.Port = port
 
 	dpa, err := storage.NewLocalDPA(datadir)

whisper/whisperv6/benchmarks_test.go

@@ -17,14 +17,16 @@
 package whisperv6
 
 import (
+	"crypto/sha256"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/crypto"
+	"golang.org/x/crypto/pbkdf2"
 )
 
 func BenchmarkDeriveKeyMaterial(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		deriveKeyMaterial([]byte("test"), 0)
+		pbkdf2.Key([]byte("test"), nil, 65356, aesKeyLength, sha256.New)
 	}
 }
 

whisper/whisperv6/doc.go

@@ -36,15 +36,15 @@ import (
 
 const (
 	EnvelopeVersion    = uint64(0)
-	ProtocolVersion    = uint64(5)
+	ProtocolVersion    = uint64(6)
-	ProtocolVersionStr = "5.0"
+	ProtocolVersionStr = "6.0"
 	ProtocolName       = "shh"
 
 	statusCode           = 0 // used by whisper protocol
 	messagesCode         = 1 // normal whisper message
 	p2pCode              = 2 // peer-to-peer message (to be consumed by the peer, but not forwarded any further)
 	p2pRequestCode       = 3 // peer-to-peer message, used by Dapp protocol
-	NumberOfMessageCodes = 64
+	NumberOfMessageCodes = 128
 
 	paddingMask   = byte(3)
 	signatureFlag = byte(4)
@@ -67,6 +67,8 @@ const (
 
 	DefaultTTL     = 50 // seconds
 	SynchAllowance = 10 // seconds
+
+	EnvelopeHeaderLength = 20
 )
 
 type unknownVersionError uint64

whisper/whisperv6/envelope.go

@@ -36,13 +36,11 @@ import (
 // Envelope represents a clear-text data packet to transmit through the Whisper
 // network. Its contents may or may not be encrypted and signed.
 type Envelope struct {
-	Version  []byte
+	Expiry uint32
-	Expiry   uint32
+	TTL    uint32
-	TTL      uint32
+	Topic  TopicType
-	Topic    TopicType
+	Data   []byte
-	AESNonce []byte
+	Nonce  uint64
-	Data     []byte
-	Nonce    uint64
 
 	pow  float64     // Message-specific PoW as described in the Whisper specification.
 	hash common.Hash // Cached hash of the envelope to avoid rehashing every time.
@@ -51,49 +49,29 @@ type Envelope struct {
 
 // size returns the size of envelope as it is sent (i.e. public fields only)
 func (e *Envelope) size() int {
-	return 20 + len(e.Version) + len(e.AESNonce) + len(e.Data)
+	return EnvelopeHeaderLength + len(e.Data)
 }
 
 // rlpWithoutNonce returns the RLP encoded envelope contents, except the nonce.
 func (e *Envelope) rlpWithoutNonce() []byte {
-	res, _ := rlp.EncodeToBytes([]interface{}{e.Version, e.Expiry, e.TTL, e.Topic, e.AESNonce, e.Data})
+	res, _ := rlp.EncodeToBytes([]interface{}{e.Expiry, e.TTL, e.Topic, e.Data})
 	return res
 }
 
 // NewEnvelope wraps a Whisper message with expiration and destination data
 // included into an envelope for network forwarding.
-func NewEnvelope(ttl uint32, topic TopicType, aesNonce []byte, msg *sentMessage) *Envelope {
+func NewEnvelope(ttl uint32, topic TopicType, msg *sentMessage) *Envelope {
 	env := Envelope{
-		Version:  make([]byte, 1),
+		Expiry: uint32(time.Now().Add(time.Second * time.Duration(ttl)).Unix()),
-		Expiry:   uint32(time.Now().Add(time.Second * time.Duration(ttl)).Unix()),
+		TTL:    ttl,
-		TTL:      ttl,
+		Topic:  topic,
-		Topic:    topic,
+		Data:   msg.Raw,
-		AESNonce: aesNonce,
+		Nonce:  0,
-		Data:     msg.Raw,
-		Nonce:    0,
-	}
-
-	if EnvelopeVersion < 256 {
-		env.Version[0] = byte(EnvelopeVersion)
-	} else {
-		panic("please increase the size of Envelope.Version before releasing this version")
 	}
 
 	return &env
 }
 
-func (e *Envelope) IsSymmetric() bool {
-	return len(e.AESNonce) > 0
-}
-
-func (e *Envelope) isAsymmetric() bool {
-	return !e.IsSymmetric()
-}
-
-func (e *Envelope) Ver() uint64 {
-	return bytesToUintLittleEndian(e.Version)
-}
-
 // Seal closes the envelope by spending the requested amount of time as a proof
 // of work on hashing the data.
 func (e *Envelope) Seal(options *MessageParams) error {
@@ -209,7 +187,7 @@ func (e *Envelope) OpenAsymmetric(key *ecdsa.PrivateKey) (*ReceivedMessage, erro
 // OpenSymmetric tries to decrypt an envelope, potentially encrypted with a particular key.
 func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) {
 	msg = &ReceivedMessage{Raw: e.Data}
-	err = msg.decryptSymmetric(key, e.AESNonce)
+	err = msg.decryptSymmetric(key)
 	if err != nil {
 		msg = nil
 	}
@@ -218,12 +196,18 @@ func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) {
 
 // Open tries to decrypt an envelope, and populates the message fields in case of success.
 func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) {
-	if e.isAsymmetric() {
+	// The API interface forbids filters doing both symmetric and
+	// asymmetric encryption.
+	if watcher.expectsAsymmetricEncryption() && watcher.expectsSymmetricEncryption() {
+		return nil
+	}
+
+	if watcher.expectsAsymmetricEncryption() {
 		msg, _ = e.OpenAsymmetric(watcher.KeyAsym)
 		if msg != nil {
 			msg.Dst = &watcher.KeyAsym.PublicKey
 		}
-	} else if e.IsSymmetric() {
+	} else if watcher.expectsSymmetricEncryption() {
 		msg, _ = e.OpenSymmetric(watcher.KeySym)
 		if msg != nil {
 			msg.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym)
@@ -240,7 +224,6 @@ func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) {
 		msg.TTL = e.TTL
 		msg.Sent = e.Expiry - e.TTL
 		msg.EnvelopeHash = e.Hash()
-		msg.EnvelopeVersion = e.Ver()
 	}
 	return msg
 }

whisper/whisperv6/envelope_test.go

@@ -0,0 +1,64 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains the tests associated with the Whisper protocol Envelope object.
+
+package whisperv6
+
+import (
+	mrand "math/rand"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+func TestEnvelopeOpenAcceptsOnlyOneKeyTypeInFilter(t *testing.T) {
+	symKey := make([]byte, aesKeyLength)
+	mrand.Read(symKey)
+
+	asymKey, err := crypto.GenerateKey()
+	if err != nil {
+		t.Fatalf("failed GenerateKey with seed %d: %s.", seed, err)
+	}
+
+	params := MessageParams{
+		PoW:      0.01,
+		WorkTime: 1,
+		TTL:      uint32(mrand.Intn(1024)),
+		Payload:  make([]byte, 50),
+		KeySym:   symKey,
+		Dst:      nil,
+	}
+
+	mrand.Read(params.Payload)
+
+	msg, err := NewSentMessage(&params)
+	if err != nil {
+		t.Fatalf("failed to create new message with seed %d: %s.", seed, err)
+	}
+
+	e, err := msg.Wrap(&params)
+	if err != nil {
+		t.Fatalf("Failed to Wrap the message in an envelope with seed %d: %s", seed, err)
+	}
+
+	f := Filter{KeySym: symKey, KeyAsym: asymKey}
+
+	decrypted := e.Open(&f)
+	if decrypted != nil {
+		t.Fatalf("Managed to decrypt a message with an invalid filter, seed %d", seed)
+	}
+}

whisper/whisperv6/filter.go

@@ -53,6 +53,10 @@ func NewFilters(w *Whisper) *Filters {
 }
 
 func (fs *Filters) Install(watcher *Filter) (string, error) {
+	if watcher.KeySym != nil && watcher.KeyAsym != nil {
+		return "", fmt.Errorf("filters must choose between symmetric and asymmetric keys")
+	}
+
 	if watcher.Messages == nil {
 		watcher.Messages = make(map[common.Hash]*ReceivedMessage)
 	}
@@ -175,6 +179,9 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) {
 	return all
 }
 
+// MatchMessage checks if the filter matches an already decrypted
+// message (i.e. a Message that has already been handled by
+// MatchEnvelope when checked by a previous filter)
 func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
 	if f.PoW > 0 && msg.PoW < f.PoW {
 		return false
@@ -188,17 +195,15 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
 	return false
 }
 
+// MatchEvelope checks if it's worth decrypting the message. If
+// it returns `true`, client code is expected to attempt decrypting
+// the message and subsequently call MatchMessage.
 func (f *Filter) MatchEnvelope(envelope *Envelope) bool {
 	if f.PoW > 0 && envelope.pow < f.PoW {
 		return false
 	}
 
-	if f.expectsAsymmetricEncryption() && envelope.isAsymmetric() {
+	return f.MatchTopic(envelope.Topic)
-		return f.MatchTopic(envelope.Topic)
-	} else if f.expectsSymmetricEncryption() && envelope.IsSymmetric() {
-		return f.MatchTopic(envelope.Topic)
-	}
-	return false
 }
 
 func (f *Filter) MatchTopic(topic TopicType) bool {

whisper/whisperv6/filter_test.go

@@ -229,6 +229,36 @@ func TestInstallIdenticalFilters(t *testing.T) {
 	}
 }
 
+func TestInstallFilterWithSymAndAsymKeys(t *testing.T) {
+	InitSingleTest()
+
+	w := New(&Config{})
+	filters := NewFilters(w)
+	filter1, _ := generateFilter(t, true)
+
+	asymKey, err := crypto.GenerateKey()
+	if err != nil {
+		t.Fatalf("Unable to create asymetric keys: %v", err)
+	}
+
+	// Copy the first filter since some of its fields
+	// are randomly gnerated.
+	filter := &Filter{
+		KeySym:   filter1.KeySym,
+		KeyAsym:  asymKey,
+		Topics:   filter1.Topics,
+		PoW:      filter1.PoW,
+		AllowP2P: filter1.AllowP2P,
+		Messages: make(map[common.Hash]*ReceivedMessage),
+	}
+
+	_, err = filters.Install(filter)
+
+	if err == nil {
+		t.Fatalf("Error detecting that a filter had both an asymmetric and symmetric key, with seed %d", seed)
+	}
+}
+
 func TestComparePubKey(t *testing.T) {
 	InitSingleTest()
 
@@ -312,12 +342,6 @@ func TestMatchEnvelope(t *testing.T) {
 		t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed)
 	}
 
-	// asymmetric + matching topic: mismatch
-	match = fasym.MatchEnvelope(env)
-	if match {
-		t.Fatalf("failed MatchEnvelope() asymmetric with seed %d.", seed)
-	}
-
 	// symmetric + matching topic + insufficient PoW: mismatch
 	fsym.PoW = env.PoW() + 1.0
 	match = fsym.MatchEnvelope(env)

whisper/whisperv6/message.go

@@ -61,6 +61,7 @@ type ReceivedMessage struct {
 	Payload   []byte
 	Padding   []byte
 	Signature []byte
+	Salt      []byte
 
 	PoW   float64          // Proof of work as described in the Whisper spec
 	Sent  uint32           // Time when the message was posted into the network
@@ -69,9 +70,8 @@ type ReceivedMessage struct {
 	Dst   *ecdsa.PublicKey // Message recipient (identity used to decode the message)
 	Topic TopicType
 
-	SymKeyHash      common.Hash // The Keccak256Hash of the key, associated with the Topic
+	SymKeyHash   common.Hash // The Keccak256Hash of the key, associated with the Topic
-	EnvelopeHash    common.Hash // Message envelope hash to act as a unique id
+	EnvelopeHash common.Hash // Message envelope hash to act as a unique id
-	EnvelopeVersion uint64
 }
 
 func isMessageSigned(flags byte) bool {
@@ -124,6 +124,10 @@ func (msg *sentMessage) appendPadding(params *MessageParams) error {
 	if params.Src != nil {
 		rawSize += signatureLength
 	}
+
+	if params.KeySym != nil {
+		rawSize += AESNonceLength
+	}
 	odd := rawSize % padSizeLimit
 
 	if len(params.Padding) != 0 {
@@ -196,31 +200,31 @@ func (msg *sentMessage) encryptAsymmetric(key *ecdsa.PublicKey) error {
 
 // encryptSymmetric encrypts a message with a topic key, using AES-GCM-256.
 // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize).
-func (msg *sentMessage) encryptSymmetric(key []byte) (nonce []byte, err error) {
+func (msg *sentMessage) encryptSymmetric(key []byte) (err error) {
 	if !validateSymmetricKey(key) {
-		return nil, errors.New("invalid key provided for symmetric encryption")
+		return errors.New("invalid key provided for symmetric encryption")
 	}
 
 	block, err := aes.NewCipher(key)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	aesgcm, err := cipher.NewGCM(block)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// never use more than 2^32 random nonces with a given key
-	nonce = make([]byte, aesgcm.NonceSize())
+	salt := make([]byte, aesgcm.NonceSize())
-	_, err = crand.Read(nonce)
+	_, err = crand.Read(salt)
 	if err != nil {
-		return nil, err
+		return err
-	} else if !validateSymmetricKey(nonce) {
+	} else if !validateSymmetricKey(salt) {
-		return nil, errors.New("crypto/rand failed to generate nonce")
+		return errors.New("crypto/rand failed to generate salt")
 	}
 
-	msg.Raw = aesgcm.Seal(nil, nonce, msg.Raw, nil)
+	msg.Raw = append(aesgcm.Seal(nil, salt, msg.Raw, nil), salt...)
-	return nonce, nil
+	return nil
 }
 
 // Wrap bundles the message into an Envelope to transmit over the network.
@@ -233,11 +237,10 @@ func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er
 			return nil, err
 		}
 	}
-	var nonce []byte
 	if options.Dst != nil {
 		err = msg.encryptAsymmetric(options.Dst)
 	} else if options.KeySym != nil {
-		nonce, err = msg.encryptSymmetric(options.KeySym)
+		err = msg.encryptSymmetric(options.KeySym)
 	} else {
 		err = errors.New("unable to encrypt the message: neither symmetric nor assymmetric key provided")
 	}
@@ -245,7 +248,7 @@ func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er
 		return nil, err
 	}
 
-	envelope = NewEnvelope(options.TTL, options.Topic, nonce, msg)
+	envelope = NewEnvelope(options.TTL, options.Topic, msg)
 	if err = envelope.Seal(options); err != nil {
 		return nil, err
 	}
@@ -254,7 +257,14 @@ func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er
 
 // decryptSymmetric decrypts a message with a topic key, using AES-GCM-256.
 // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize).
-func (msg *ReceivedMessage) decryptSymmetric(key []byte, nonce []byte) error {
+func (msg *ReceivedMessage) decryptSymmetric(key []byte) error {
+	// In v6, symmetric messages are expected to contain the 12-byte
+	// "salt" at the end of the payload.
+	if len(msg.Raw) < AESNonceLength {
+		return errors.New("missing salt or invalid payload in symmetric message")
+	}
+	salt := msg.Raw[len(msg.Raw)-AESNonceLength:]
+
 	block, err := aes.NewCipher(key)
 	if err != nil {
 		return err
@@ -263,15 +273,16 @@ func (msg *ReceivedMessage) decryptSymmetric(key []byte, nonce []byte) error {
 	if err != nil {
 		return err
 	}
-	if len(nonce) != aesgcm.NonceSize() {
+	if len(salt) != aesgcm.NonceSize() {
-		log.Error("decrypting the message", "AES nonce size", len(nonce))
+		log.Error("decrypting the message", "AES salt size", len(salt))
-		return errors.New("wrong AES nonce size")
+		return errors.New("wrong AES salt size")
 	}
-	decrypted, err := aesgcm.Open(nil, nonce, msg.Raw, nil)
+	decrypted, err := aesgcm.Open(nil, salt, msg.Raw[:len(msg.Raw)-AESNonceLength], nil)
 	if err != nil {
 		return err
 	}
 	msg.Raw = decrypted
+	msg.Salt = salt
 	return nil
 }
 

whisper/whisperv6/message_test.go

@@ -174,10 +174,8 @@ func TestMessageSeal(t *testing.T) {
 		t.Fatalf("failed to create new message with seed %d: %s.", seed, err)
 	}
 	params.TTL = 1
-	aesnonce := make([]byte, 12)
-	mrand.Read(aesnonce)
 
-	env := NewEnvelope(params.TTL, params.Topic, aesnonce, msg)
+	env := NewEnvelope(params.TTL, params.Topic, msg)
 	if err != nil {
 		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
 	}
@@ -242,7 +240,12 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) {
 		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
 	}
 
-	f := Filter{KeyAsym: key, KeySym: params.KeySym}
+	var f Filter
+	if symmetric {
+		f = Filter{KeySym: params.KeySym}
+	} else {
+		f = Filter{KeyAsym: key}
+	}
 	decrypted := env.Open(&f)
 	if decrypted == nil {
 		t.Fatalf("failed to open with seed %d.", seed)
@@ -413,3 +416,59 @@ func TestPadding(t *testing.T) {
 		singlePaddingTest(t, n)
 	}
 }
+
+func TestPaddingAppendedToSymMessages(t *testing.T) {
+	params := &MessageParams{
+		Payload: make([]byte, 246),
+		KeySym:  make([]byte, aesKeyLength),
+	}
+
+	// Simulate a message with a payload just under 256 so that
+	// payload + flag + aesnonce > 256. Check that the result
+	// is padded on the next 256 boundary.
+	msg := sentMessage{}
+	msg.Raw = make([]byte, len(params.Payload)+1+AESNonceLength)
+
+	err := msg.appendPadding(params)
+
+	if err != nil {
+		t.Fatalf("Error appending padding to message %v", err)
+		return
+	}
+
+	if len(msg.Raw) != 512 {
+		t.Errorf("Invalid size %d != 512", len(msg.Raw))
+	}
+}
+
+func TestPaddingAppendedToSymMessagesWithSignature(t *testing.T) {
+	params := &MessageParams{
+		Payload: make([]byte, 246),
+		KeySym:  make([]byte, aesKeyLength),
+	}
+
+	pSrc, err := crypto.GenerateKey()
+
+	if err != nil {
+		t.Fatalf("Error creating the signature key %v", err)
+		return
+	}
+	params.Src = pSrc
+
+	// Simulate a message with a payload just under 256 so that
+	// payload + flag + aesnonce > 256. Check that the result
+	// is padded on the next 256 boundary.
+	msg := sentMessage{}
+	msg.Raw = make([]byte, len(params.Payload)+1+AESNonceLength+signatureLength)
+
+	err = msg.appendPadding(params)
+
+	if err != nil {
+		t.Fatalf("Error appending padding to message %v", err)
+		return
+	}
+
+	if len(msg.Raw) != 512 {
+		t.Errorf("Invalid size %d != 512", len(msg.Raw))
+	}
+}

whisper/whisperv6/whisper.go

@@ -367,7 +367,9 @@ func (w *Whisper) AddSymKeyFromPassword(password string) (string, error) {
 		return "", fmt.Errorf("failed to generate unique ID")
 	}
 
-	derived, err := deriveKeyMaterial([]byte(password), EnvelopeVersion)
+	// kdf should run no less than 0.1 seconds on an average computer,
+	// because it's an once in a session experience
+	derived := pbkdf2.Key([]byte(password), nil, 65356, aesKeyLength, sha256.New)
 	if err != nil {
 		return "", err
 	}
@@ -587,17 +589,6 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) {
 		return false, fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash())
 	}
 
-	if len(envelope.Version) > 4 {
-		return false, fmt.Errorf("oversized version [%x]", envelope.Hash())
-	}
-
-	aesNonceSize := len(envelope.AESNonce)
-	if aesNonceSize != 0 && aesNonceSize != AESNonceLength {
-		// the standard AES GCM nonce size is 12 bytes,
-		// but constant gcmStandardNonceSize cannot be accessed (not exported)
-		return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash())
-	}
-
 	if envelope.PoW() < wh.MinPow() {
 		log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex())
 		return false, nil // drop envelope without error
@@ -635,16 +626,11 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) {
 
 // postEvent queues the message for further processing.
 func (w *Whisper) postEvent(envelope *Envelope, isP2P bool) {
-	// if the version of incoming message is higher than
+	if isP2P {
-	// currently supported version, we can not decrypt it,
+		w.p2pMsgQueue <- envelope
-	// and therefore just ignore this message
+	} else {
-	if envelope.Ver() <= EnvelopeVersion {
+		w.checkOverflow()
-		if isP2P {
+		w.messageQueue <- envelope
-			w.p2pMsgQueue <- envelope
-		} else {
-			w.checkOverflow()
-			w.messageQueue <- envelope
-		}
 	}
 }
 
@@ -830,19 +816,6 @@ func BytesToUintBigEndian(b []byte) (res uint64) {
 	return res
 }
 
-// deriveKeyMaterial derives symmetric key material from the key or password.
-// pbkdf2 is used for security, in case people use password instead of randomly generated keys.
-func deriveKeyMaterial(key []byte, version uint64) (derivedKey []byte, err error) {
-	if version == 0 {
-		// kdf should run no less than 0.1 seconds on average compute,
-		// because it's a once in a session experience
-		derivedKey := pbkdf2.Key(key, nil, 65356, aesKeyLength, sha256.New)
-		return derivedKey, nil
-	} else {
-		return nil, unknownVersionError(version)
-	}
-}
-
 // GenerateRandomID generates a random string, which is then returned to be used as a key id
 func GenerateRandomID() (id string, err error) {
 	buf := make([]byte, keyIdSize)

whisper/whisperv6/whisper_test.go

@@ -19,11 +19,13 @@ package whisperv6
 import (
 	"bytes"
 	"crypto/ecdsa"
+	"crypto/sha256"
 	mrand "math/rand"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"golang.org/x/crypto/pbkdf2"
 )
 
 func TestWhisperBasic(t *testing.T) {
@@ -79,14 +81,7 @@ func TestWhisperBasic(t *testing.T) {
 	}
 
 	var derived []byte
-	ver := uint64(0xDEADBEEF)
+	derived = pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New)
-	if _, err := deriveKeyMaterial(peerID, ver); err != unknownVersionError(ver) {
-		t.Fatalf("failed deriveKeyMaterial with param = %v: %s.", peerID, err)
-	}
-	derived, err = deriveKeyMaterial(peerID, 0)
-	if err != nil {
-		t.Fatalf("failed second deriveKeyMaterial with param = %v: %s.", peerID, err)
-	}
 	if !validateSymmetricKey(derived) {
 		t.Fatalf("failed validateSymmetricKey with param = %v.", derived)
 	}