diff --git a/.travis.yml b/.travis.yml index 1bd69da347..cc1b185fce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ matrix: sudo: required go: 1.7.x script: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse - sudo chmod 666 /dev/fuse - sudo chown root:$USER /etc/fuse.conf @@ -20,7 +19,6 @@ matrix: sudo: required go: 1.8.x script: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse - sudo chmod 666 /dev/fuse - sudo chown root:$USER /etc/fuse.conf @@ -33,7 +31,6 @@ matrix: sudo: required go: 1.9.x script: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse - sudo chmod 666 /dev/fuse - sudo chown root:$USER /etc/fuse.conf @@ -42,7 +39,6 @@ matrix: - os: osx go: 1.9.x - sudo: required script: - brew update - brew install caskroom/cask/brew-cask @@ -53,15 +49,12 @@ matrix: # This builder only tests code linters on latest version of Go - os: linux dist: trusty - sudo: required go: 1.9.x env: - lint + git: + submodules: false # avoid cloning ethereum/tests script: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - go run build/ci.go lint # This builder does the Ubuntu PPA and Linux Azure uploads @@ -72,6 +65,8 @@ matrix: env: - ubuntu-ppa - azure-linux + git: + submodules: false # avoid cloning ethereum/tests addons: apt: packages: @@ -104,12 +99,13 @@ matrix: # This builder does the Linux Azure MIPS xgo uploads - os: linux dist: trusty - sudo: required services: - docker go: 1.9.x env: - azure-linux-mips + git: + submodules: false # avoid cloning ethereum/tests script: - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done @@ -146,6 +142,8 @@ matrix: env: - azure-android - maven-android + git: + submodules: false # avoid cloning ethereum/tests before_install: - curl https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH @@ -169,6 +167,8 @@ matrix: - azure-osx - azure-ios - cocoapods-ios + git: + submodules: false # avoid cloning ethereum/tests script: - go run build/ci.go install - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds @@ -193,15 +193,11 @@ matrix: go: 1.9.x env: - azure-purge + git: + submodules: false # avoid cloning ethereum/tests script: - go run build/ci.go purge -store gethstore/builds -days 14 -install: - - go get golang.org/x/tools/cmd/cover -script: - - go run build/ci.go install - - go run build/ci.go test -coverage - notifications: webhooks: urls: diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go new file mode 100644 index 0000000000..ec81bc741b --- /dev/null +++ b/cmd/swarm/config.go @@ -0,0 +1,321 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "errors" + "fmt" + "io" + "os" + "reflect" + "strconv" + "unicode" + + cli "gopkg.in/urfave/cli.v1" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/naoina/toml" + + bzzapi "github.com/ethereum/go-ethereum/swarm/api" +) + +var ( + //flag definition for the dumpconfig command + DumpConfigCommand = cli.Command{ + Action: utils.MigrateFlags(dumpConfig), + Name: "dumpconfig", + Usage: "Show configuration values", + ArgsUsage: "", + Flags: app.Flags, + Category: "MISCELLANEOUS COMMANDS", + Description: `The dumpconfig command shows configuration values.`, + } + + //flag definition for the config file command + SwarmTomlConfigPathFlag = cli.StringFlag{ + Name: "config", + Usage: "TOML configuration file", + } +) + +//constants for environment variables +const ( + SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" + SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" + SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" + SWARM_ENV_PORT = "SWARM_PORT" + SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" + SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" + SWARM_ENV_SWAP_API = "SWARM_SWAP_API" + SWARM_ENV_SYNC_ENABLE = "SWARM_SYNC_ENABLE" + SWARM_ENV_ENS_API = "SWARM_ENS_API" + SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" + SWARM_ENV_CORS = "SWARM_CORS" + SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" + GETH_ENV_DATADIR = "GETH_DATADIR" +) + +// These settings ensure that TOML keys use the same names as Go struct fields. +var tomlSettings = toml.Config{ + NormFieldName: func(rt reflect.Type, key string) string { + return key + }, + FieldToKey: func(rt reflect.Type, field string) string { + return field + }, + MissingField: func(rt reflect.Type, field string) error { + link := "" + if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" { + link = fmt.Sprintf(", check github.com/ethereum/go-ethereum/swarm/api/config.go for available fields") + } + return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link) + }, +} + +//before booting the swarm node, build the configuration +func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) { + //check for deprecated flags + checkDeprecated(ctx) + //start by creating a default config + config = bzzapi.NewDefaultConfig() + //first load settings from config file (if provided) + config, err = configFileOverride(config, ctx) + //override settings provided by environment variables + config = envVarsOverride(config) + //override settings provided by command line + config = cmdLineOverride(config, ctx) + + return +} + +//finally, after the configuration build phase is finished, initialize +func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) { + //at this point, all vars should be set in the Config + //get the account for the provided swarm account + prvkey := getAccount(config.BzzAccount, ctx, stack) + //set the resolved config path (geth --datadir) + config.Path = stack.InstanceDir() + //finally, initialize the configuration + config.Init(prvkey) + //configuration phase completed here + log.Debug("Starting Swarm with the following parameters:") + //after having created the config, print it to screen + log.Debug(printConfig(config)) +} + +//override the current config with whatever is in the config file, if a config file has been provided +func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) { + var err error + + //only do something if the -config flag has been set + if ctx.GlobalIsSet(SwarmTomlConfigPathFlag.Name) { + var filepath string + if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" { + utils.Fatalf("Config file flag provided with invalid file path") + } + f, err := os.Open(filepath) + if err != nil { + return nil, err + } + defer f.Close() + + //decode the TOML file into a Config struct + //note that we are decoding into the existing defaultConfig; + //if an entry is not present in the file, the default entry is kept + err = tomlSettings.NewDecoder(f).Decode(&config) + // Add file name to errors that have a line number. + if _, ok := err.(*toml.LineError); ok { + err = errors.New(filepath + ", " + err.Error()) + } + } + return config, err +} + +//override the current config with whatever is provided through the command line +//most values are not allowed a zero value (empty string), if not otherwise noted +func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Config { + + if keyid := ctx.GlobalString(SwarmAccountFlag.Name); keyid != "" { + currentConfig.BzzAccount = keyid + } + + if chbookaddr := ctx.GlobalString(ChequebookAddrFlag.Name); chbookaddr != "" { + currentConfig.Contract = common.HexToAddress(chbookaddr) + } + + if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" { + if id, _ := strconv.Atoi(networkid); id != 0 { + currentConfig.NetworkId = uint64(id) + } + } + + if ctx.GlobalIsSet(utils.DataDirFlag.Name) { + if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" { + currentConfig.Path = datadir + } + } + + bzzport := ctx.GlobalString(SwarmPortFlag.Name) + if len(bzzport) > 0 { + currentConfig.Port = bzzport + } + + if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" { + currentConfig.ListenAddr = bzzaddr + } + + if ctx.GlobalIsSet(SwarmSwapEnabledFlag.Name) { + currentConfig.SwapEnabled = true + } + + if ctx.GlobalIsSet(SwarmSyncEnabledFlag.Name) { + currentConfig.SyncEnabled = true + } + + currentConfig.SwapApi = ctx.GlobalString(SwarmSwapAPIFlag.Name) + if currentConfig.SwapEnabled && currentConfig.SwapApi == "" { + utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) + } + + //EnsApi can be set to "", so can't check for empty string, as it is allowed! + if ctx.GlobalIsSet(EnsAPIFlag.Name) { + currentConfig.EnsApi = ctx.GlobalString(EnsAPIFlag.Name) + } + + if ensaddr := ctx.GlobalString(EnsAddrFlag.Name); ensaddr != "" { + currentConfig.EnsRoot = common.HexToAddress(ensaddr) + } + + if cors := ctx.GlobalString(CorsStringFlag.Name); cors != "" { + currentConfig.Cors = cors + } + + if ctx.GlobalIsSet(utils.BootnodesFlag.Name) { + currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name) + } + + return currentConfig + +} + +//override the current config with whatver is provided in environment variables +//most values are not allowed a zero value (empty string), if not otherwise noted +func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { + + if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" { + currentConfig.BzzAccount = keyid + } + + if chbookaddr := os.Getenv(SWARM_ENV_CHEQUEBOOK_ADDR); chbookaddr != "" { + currentConfig.Contract = common.HexToAddress(chbookaddr) + } + + if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" { + if id, _ := strconv.Atoi(networkid); id != 0 { + currentConfig.NetworkId = uint64(id) + } + } + + if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" { + currentConfig.Path = datadir + } + + bzzport := os.Getenv(SWARM_ENV_PORT) + if len(bzzport) > 0 { + currentConfig.Port = bzzport + } + + if bzzaddr := os.Getenv(SWARM_ENV_LISTEN_ADDR); bzzaddr != "" { + currentConfig.ListenAddr = bzzaddr + } + + if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" { + if swap, err := strconv.ParseBool(swapenable); err != nil { + currentConfig.SwapEnabled = swap + } + } + + if syncenable := os.Getenv(SWARM_ENV_SYNC_ENABLE); syncenable != "" { + if sync, err := strconv.ParseBool(syncenable); err != nil { + currentConfig.SyncEnabled = sync + } + } + + if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" { + currentConfig.SwapApi = swapapi + } + + if currentConfig.SwapEnabled && currentConfig.SwapApi == "" { + utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) + } + + //EnsApi can be set to "", so can't check for empty string, as it is allowed + if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists == true { + currentConfig.EnsApi = ensapi + } + + if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" { + currentConfig.EnsRoot = common.HexToAddress(ensaddr) + } + + if cors := os.Getenv(SWARM_ENV_CORS); cors != "" { + currentConfig.Cors = cors + } + + if bootnodes := os.Getenv(SWARM_ENV_BOOTNODES); bootnodes != "" { + currentConfig.BootNodes = bootnodes + } + + return currentConfig +} + +// dumpConfig is the dumpconfig command. +// writes a default config to STDOUT +func dumpConfig(ctx *cli.Context) error { + cfg, err := buildConfig(ctx) + if err != nil { + utils.Fatalf(fmt.Sprintf("Uh oh - dumpconfig triggered an error %v", err)) + } + comment := "" + out, err := tomlSettings.Marshal(&cfg) + if err != nil { + return err + } + io.WriteString(os.Stdout, comment) + os.Stdout.Write(out) + return nil +} + +//deprecated flags checked here +func checkDeprecated(ctx *cli.Context) { + // exit if the deprecated --ethapi flag is set + if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" { + utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.") + } +} + +//print a Config as string +func printConfig(config *bzzapi.Config) string { + out, err := tomlSettings.Marshal(&config) + if err != nil { + return (fmt.Sprintf("Something is not right with the configuration: %v", err)) + } + return string(out) +} diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go new file mode 100644 index 0000000000..7ffe2cfb97 --- /dev/null +++ b/cmd/swarm/config_test.go @@ -0,0 +1,459 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "testing" + "time" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm" + "github.com/ethereum/go-ethereum/swarm/api" + + "github.com/docker/docker/pkg/reexec" +) + +func TestDumpConfig(t *testing.T) { + swarm := runSwarm(t, "dumpconfig") + defaultConf := api.NewDefaultConfig() + out, err := tomlSettings.Marshal(&defaultConf) + if err != nil { + t.Fatal(err) + } + swarm.Expect(string(out)) + swarm.ExpectExit() +} + +func TestFailsSwapEnabledNoSwapApi(t *testing.T) { + flags := []string{ + fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42", + fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545", + fmt.Sprintf("--%s", SwarmSwapEnabledFlag.Name), + } + + swarm := runSwarm(t, flags...) + swarm.Expect("Fatal: " + SWARM_ERR_SWAP_SET_NO_API + "\n") + swarm.ExpectExit() +} + +func TestFailsNoBzzAccount(t *testing.T) { + flags := []string{ + fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42", + fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545", + } + + swarm := runSwarm(t, flags...) + swarm.Expect("Fatal: " + SWARM_ERR_NO_BZZACCOUNT + "\n") + swarm.ExpectExit() +} + +func TestCmdLineOverrides(t *testing.T) { + dir, err := ioutil.TempDir("", "bzztest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + conf, account := getTestAccount(t, dir) + node := &testNode{Dir: dir} + + // assign ports + httpPort, err := assignTCPPort() + if err != nil { + t.Fatal(err) + } + + flags := []string{ + fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42", + fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort, + fmt.Sprintf("--%s", SwarmSyncEnabledFlag.Name), + fmt.Sprintf("--%s", CorsStringFlag.Name), "*", + fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), + fmt.Sprintf("--%s", EnsAPIFlag.Name), "", + "--datadir", dir, + "--ipcpath", conf.IPCPath, + } + node.Cmd = runSwarm(t, flags...) + node.Cmd.InputLine(testPassphrase) + defer func() { + if t.Failed() { + node.Shutdown() + } + }() + // wait for the node to start + for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { + node.Client, err = rpc.Dial(conf.IPCEndpoint()) + if err == nil { + break + } + } + if node.Client == nil { + t.Fatal(err) + } + + // load info + var info swarm.Info + if err := node.Client.Call(&info, "bzz_info"); err != nil { + t.Fatal(err) + } + + if info.Port != httpPort { + t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port) + } + + if info.NetworkId != 42 { + t.Fatalf("Expected network ID to be %d, got %d", 42, info.NetworkId) + } + + if info.SyncEnabled != true { + t.Fatal("Expected Sync to be enabled, but is false") + } + + if info.Cors != "*" { + t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors) + } + + node.Shutdown() +} + +func TestFileOverrides(t *testing.T) { + + // assign ports + httpPort, err := assignTCPPort() + if err != nil { + t.Fatal(err) + } + + //create a config file + //first, create a default conf + defaultConf := api.NewDefaultConfig() + //change some values in order to test if they have been loaded + defaultConf.SyncEnabled = true + defaultConf.NetworkId = 54 + defaultConf.Port = httpPort + defaultConf.StoreParams.DbCapacity = 9000000 + defaultConf.ChunkerParams.Branches = 64 + defaultConf.HiveParams.CallInterval = 6000000000 + defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second + defaultConf.SyncParams.KeyBufferSize = 512 + //create a TOML string + out, err := tomlSettings.Marshal(&defaultConf) + if err != nil { + t.Fatalf("Error creating TOML file in TestFileOverride: %v", err) + } + //create file + f, err := ioutil.TempFile("", "testconfig.toml") + if err != nil { + t.Fatalf("Error writing TOML file in TestFileOverride: %v", err) + } + //write file + _, err = f.WriteString(string(out)) + if err != nil { + t.Fatalf("Error writing TOML file in TestFileOverride: %v", err) + } + f.Sync() + + dir, err := ioutil.TempDir("", "bzztest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + conf, account := getTestAccount(t, dir) + node := &testNode{Dir: dir} + + flags := []string{ + fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(), + fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), + "--ens-api", "", + "--ipcpath", conf.IPCPath, + "--datadir", dir, + } + node.Cmd = runSwarm(t, flags...) + node.Cmd.InputLine(testPassphrase) + defer func() { + if t.Failed() { + node.Shutdown() + } + }() + // wait for the node to start + for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { + node.Client, err = rpc.Dial(conf.IPCEndpoint()) + if err == nil { + break + } + } + if node.Client == nil { + t.Fatal(err) + } + + // load info + var info swarm.Info + if err := node.Client.Call(&info, "bzz_info"); err != nil { + t.Fatal(err) + } + + if info.Port != httpPort { + t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port) + } + + if info.NetworkId != 54 { + t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId) + } + + if info.SyncEnabled != true { + t.Fatal("Expected Sync to be enabled, but is false") + } + + if info.StoreParams.DbCapacity != 9000000 { + t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId) + } + + if info.ChunkerParams.Branches != 64 { + t.Fatalf("Expected chunker params branches to be %d, got %d", 64, info.ChunkerParams.Branches) + } + + if info.HiveParams.CallInterval != 6000000000 { + t.Fatalf("Expected HiveParams CallInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.CallInterval)) + } + + if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second { + t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval) + } + + if info.SyncParams.KeyBufferSize != 512 { + t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize) + } + + node.Shutdown() +} + +func TestEnvVars(t *testing.T) { + // assign ports + httpPort, err := assignTCPPort() + if err != nil { + t.Fatal(err) + } + + envVars := os.Environ() + envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmPortFlag.EnvVar, httpPort)) + envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmNetworkIdFlag.EnvVar, "999")) + envVars = append(envVars, fmt.Sprintf("%s=%s", CorsStringFlag.EnvVar, "*")) + envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmSyncEnabledFlag.EnvVar, "true")) + + dir, err := ioutil.TempDir("", "bzztest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + conf, account := getTestAccount(t, dir) + node := &testNode{Dir: dir} + flags := []string{ + fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), + "--ens-api", "", + "--datadir", dir, + "--ipcpath", conf.IPCPath, + } + + //node.Cmd = runSwarm(t,flags...) + //node.Cmd.cmd.Env = envVars + //the above assignment does not work, so we need a custom Cmd here in order to pass envVars: + cmd := &exec.Cmd{ + Path: reexec.Self(), + Args: append([]string{"swarm-test"}, flags...), + Stderr: os.Stderr, + Stdout: os.Stdout, + } + cmd.Env = envVars + //stdout, err := cmd.StdoutPipe() + //if err != nil { + // t.Fatal(err) + //} + //stdout = bufio.NewReader(stdout) + var stdin io.WriteCloser + if stdin, err = cmd.StdinPipe(); err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + //cmd.InputLine(testPassphrase) + io.WriteString(stdin, testPassphrase+"\n") + defer func() { + if t.Failed() { + node.Shutdown() + cmd.Process.Kill() + } + }() + // wait for the node to start + for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { + node.Client, err = rpc.Dial(conf.IPCEndpoint()) + if err == nil { + break + } + } + + if node.Client == nil { + t.Fatal(err) + } + + // load info + var info swarm.Info + if err := node.Client.Call(&info, "bzz_info"); err != nil { + t.Fatal(err) + } + + if info.Port != httpPort { + t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port) + } + + if info.NetworkId != 999 { + t.Fatalf("Expected network ID to be %d, got %d", 999, info.NetworkId) + } + + if info.Cors != "*" { + t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors) + } + + if info.SyncEnabled != true { + t.Fatal("Expected Sync to be enabled, but is false") + } + + node.Shutdown() + cmd.Process.Kill() +} + +func TestCmdLineOverridesFile(t *testing.T) { + + // assign ports + httpPort, err := assignTCPPort() + if err != nil { + t.Fatal(err) + } + + //create a config file + //first, create a default conf + defaultConf := api.NewDefaultConfig() + //change some values in order to test if they have been loaded + defaultConf.SyncEnabled = false + defaultConf.NetworkId = 54 + defaultConf.Port = "8588" + defaultConf.StoreParams.DbCapacity = 9000000 + defaultConf.ChunkerParams.Branches = 64 + defaultConf.HiveParams.CallInterval = 6000000000 + defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second + defaultConf.SyncParams.KeyBufferSize = 512 + //create a TOML file + out, err := tomlSettings.Marshal(&defaultConf) + if err != nil { + t.Fatalf("Error creating TOML file in TestFileOverride: %v", err) + } + //write file + f, err := ioutil.TempFile("", "testconfig.toml") + if err != nil { + t.Fatalf("Error writing TOML file in TestFileOverride: %v", err) + } + //write file + _, err = f.WriteString(string(out)) + if err != nil { + t.Fatalf("Error writing TOML file in TestFileOverride: %v", err) + } + f.Sync() + + dir, err := ioutil.TempDir("", "bzztest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + conf, account := getTestAccount(t, dir) + node := &testNode{Dir: dir} + + expectNetworkId := uint64(77) + + flags := []string{ + fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "77", + fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort, + fmt.Sprintf("--%s", SwarmSyncEnabledFlag.Name), + fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(), + fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), + "--ens-api", "", + "--datadir", dir, + "--ipcpath", conf.IPCPath, + } + node.Cmd = runSwarm(t, flags...) + node.Cmd.InputLine(testPassphrase) + defer func() { + if t.Failed() { + node.Shutdown() + } + }() + // wait for the node to start + for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { + node.Client, err = rpc.Dial(conf.IPCEndpoint()) + if err == nil { + break + } + } + if node.Client == nil { + t.Fatal(err) + } + + // load info + var info swarm.Info + if err := node.Client.Call(&info, "bzz_info"); err != nil { + t.Fatal(err) + } + + if info.Port != httpPort { + t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port) + } + + if info.NetworkId != expectNetworkId { + t.Fatalf("Expected network ID to be %d, got %d", expectNetworkId, info.NetworkId) + } + + if info.SyncEnabled != true { + t.Fatal("Expected Sync to be enabled, but is false") + } + + if info.StoreParams.DbCapacity != 9000000 { + t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId) + } + + if info.ChunkerParams.Branches != 64 { + t.Fatalf("Expected chunker params branches to be %d, got %d", 64, info.ChunkerParams.Branches) + } + + if info.HiveParams.CallInterval != 6000000000 { + t.Fatalf("Expected HiveParams CallInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.CallInterval)) + } + + if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second { + t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval) + } + + if info.SyncParams.KeyBufferSize != 512 { + t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize) + } + + node.Shutdown() +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 603fd9b941..77315a4265 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -48,6 +48,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" + "gopkg.in/urfave/cli.v1" ) @@ -66,49 +67,58 @@ var ( var ( ChequebookAddrFlag = cli.StringFlag{ - Name: "chequebook", - Usage: "chequebook contract address", + Name: "chequebook", + Usage: "chequebook contract address", + EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR, } SwarmAccountFlag = cli.StringFlag{ - Name: "bzzaccount", - Usage: "Swarm account key file", + Name: "bzzaccount", + Usage: "Swarm account key file", + EnvVar: SWARM_ENV_ACCOUNT, } SwarmListenAddrFlag = cli.StringFlag{ - Name: "httpaddr", - Usage: "Swarm HTTP API listening interface", + Name: "httpaddr", + Usage: "Swarm HTTP API listening interface", + EnvVar: SWARM_ENV_LISTEN_ADDR, } SwarmPortFlag = cli.StringFlag{ - Name: "bzzport", - Usage: "Swarm local http api port", + Name: "bzzport", + Usage: "Swarm local http api port", + EnvVar: SWARM_ENV_PORT, } SwarmNetworkIdFlag = cli.IntFlag{ - Name: "bzznetworkid", - Usage: "Network identifier (integer, default 3=swarm testnet)", + Name: "bzznetworkid", + Usage: "Network identifier (integer, default 3=swarm testnet)", + EnvVar: SWARM_ENV_NETWORK_ID, } SwarmConfigPathFlag = cli.StringFlag{ Name: "bzzconfig", - Usage: "Swarm config file path (datadir/bzz)", + Usage: "DEPRECATED: please use --config path/to/TOML-file", } SwarmSwapEnabledFlag = cli.BoolFlag{ - Name: "swap", - Usage: "Swarm SWAP enabled (default false)", + Name: "swap", + Usage: "Swarm SWAP enabled (default false)", + EnvVar: SWARM_ENV_SWAP_ENABLE, } SwarmSwapAPIFlag = cli.StringFlag{ - Name: "swap-api", - Usage: "URL of the Ethereum API provider to use to settle SWAP payments", + Name: "swap-api", + Usage: "URL of the Ethereum API provider to use to settle SWAP payments", + EnvVar: SWARM_ENV_SWAP_API, } SwarmSyncEnabledFlag = cli.BoolTFlag{ - Name: "sync", - Usage: "Swarm Syncing enabled (default true)", + Name: "sync", + Usage: "Swarm Syncing enabled (default true)", + EnvVar: SWARM_ENV_SYNC_ENABLE, } EnsAPIFlag = cli.StringFlag{ - Name: "ens-api", - Usage: "URL of the Ethereum API provider to use for ENS record lookups", - Value: node.DefaultIPCEndpoint("geth"), + Name: "ens-api", + Usage: "URL of the Ethereum API provider to use for ENS record lookups", + EnvVar: SWARM_ENV_ENS_API, } EnsAddrFlag = cli.StringFlag{ - Name: "ens-addr", - Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)", + Name: "ens-addr", + Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)", + EnvVar: SWARM_ENV_ENS_ADDR, } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", @@ -136,8 +146,9 @@ var ( Usage: "force mime type", } CorsStringFlag = cli.StringFlag{ - Name: "corsdomain", - Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", + Name: "corsdomain", + Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", + EnvVar: SWARM_ENV_CORS, } // the following flags are deprecated and should be removed in the future @@ -147,6 +158,12 @@ var ( } ) +//declare a few constant error messages, useful for later error check comparisons in test +var ( + SWARM_ERR_NO_BZZACCOUNT = "bzzaccount option is required but not set; check your config file, command line or environment variables" + SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set" +) + var defaultNodeConfig = node.DefaultConfig // This init function sets defaults so cmd/swarm can run alongside geth. @@ -302,6 +319,8 @@ Remove corrupt entries from a local chunk database. DEPRECATED: use 'swarm db clean'. `, }, + // See config.go + DumpConfigCommand, } sort.Sort(cli.CommandsByName(app.Commands)) @@ -325,6 +344,7 @@ DEPRECATED: use 'swarm db clean'. CorsStringFlag, EnsAPIFlag, EnsAddrFlag, + SwarmTomlConfigPathFlag, SwarmConfigPathFlag, SwarmSwapEnabledFlag, SwarmSwapAPIFlag, @@ -377,19 +397,32 @@ func version(ctx *cli.Context) error { } func bzzd(ctx *cli.Context) error { - // exit if the deprecated --ethapi flag is set - if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" { - utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.") + //build a valid bzzapi.Config from all available sources: + //default config, file config, command line and env vars + bzzconfig, err := buildConfig(ctx) + if err != nil { + utils.Fatalf("unable to configure swarm: %v", err) } cfg := defaultNodeConfig + //geth only supports --datadir via command line + //in order to be consistent within swarm, if we pass --datadir via environment variable + //or via config file, we get the same directory for geth and swarm + if _, err := os.Stat(bzzconfig.Path); err == nil { + cfg.DataDir = bzzconfig.Path + } + //setup the ethereum node utils.SetNodeConfig(ctx, &cfg) stack, err := node.New(&cfg) if err != nil { utils.Fatalf("can't create node: %v", err) } - - registerBzzService(ctx, stack) + //a few steps need to be done after the config phase is completed, + //due to overriding behavior + initSwarmNode(bzzconfig, stack, ctx) + //register BZZ as node.Service in the ethereum node + registerBzzService(bzzconfig, ctx, stack) + //start the node utils.StartNode(stack) go func() { @@ -401,13 +434,12 @@ func bzzd(ctx *cli.Context) error { stack.Stop() }() - networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name) // Add bootnodes as initial peers. - if ctx.GlobalIsSet(utils.BootnodesFlag.Name) { - bootnodes := strings.Split(ctx.GlobalString(utils.BootnodesFlag.Name), ",") + if bzzconfig.BootNodes != "" { + bootnodes := strings.Split(bzzconfig.BootNodes, ",") injectBootnodes(stack.Server(), bootnodes) } else { - if networkId == 3 { + if bzzconfig.NetworkId == 3 { injectBootnodes(stack.Server(), testbetBootNodes) } } @@ -448,61 +480,31 @@ func detectEnsAddr(client *rpc.Client) (common.Address, error) { } } -func registerBzzService(ctx *cli.Context, stack *node.Node) { - prvkey := getAccount(ctx, stack) - - chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name)) - bzzdir := ctx.GlobalString(SwarmConfigPathFlag.Name) - if bzzdir == "" { - bzzdir = stack.InstanceDir() - } - - bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name)) - if err != nil { - utils.Fatalf("unable to configure swarm: %v", err) - } - bzzport := ctx.GlobalString(SwarmPortFlag.Name) - if len(bzzport) > 0 { - bzzconfig.Port = bzzport - } - if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" { - bzzconfig.ListenAddr = bzzaddr - } - swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name) - syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name) - - swapapi := ctx.GlobalString(SwarmSwapAPIFlag.Name) - if swapEnabled && swapapi == "" { - utils.Fatalf("SWAP is enabled but --swap-api is not set") - } - - ensapi := ctx.GlobalString(EnsAPIFlag.Name) - ensAddr := ctx.GlobalString(EnsAddrFlag.Name) - - cors := ctx.GlobalString(CorsStringFlag.Name) +func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.Node) { + //define the swarm service boot function boot := func(ctx *node.ServiceContext) (node.Service, error) { var swapClient *ethclient.Client - if swapapi != "" { - log.Info("connecting to SWAP API", "url", swapapi) - swapClient, err = ethclient.Dial(swapapi) + var err error + if bzzconfig.SwapApi != "" { + log.Info("connecting to SWAP API", "url", bzzconfig.SwapApi) + swapClient, err = ethclient.Dial(bzzconfig.SwapApi) if err != nil { - return nil, fmt.Errorf("error connecting to SWAP API %s: %s", swapapi, err) + return nil, fmt.Errorf("error connecting to SWAP API %s: %s", bzzconfig.SwapApi, err) } } var ensClient *ethclient.Client - if ensapi != "" { - log.Info("connecting to ENS API", "url", ensapi) - client, err := rpc.Dial(ensapi) + if bzzconfig.EnsApi != "" { + log.Info("connecting to ENS API", "url", bzzconfig.EnsApi) + client, err := rpc.Dial(bzzconfig.EnsApi) if err != nil { - return nil, fmt.Errorf("error connecting to ENS API %s: %s", ensapi, err) + return nil, fmt.Errorf("error connecting to ENS API %s: %s", bzzconfig.EnsApi, err) } ensClient = ethclient.NewClient(client) - if ensAddr != "" { - bzzconfig.EnsRoot = common.HexToAddress(ensAddr) - } else { + //no ENS root address set yet + if bzzconfig.EnsRoot == (common.Address{}) { ensAddr, err := detectEnsAddr(client) if err == nil { bzzconfig.EnsRoot = ensAddr @@ -512,21 +514,21 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, swapEnabled, syncEnabled, cors) + return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors) } + //register within the ethereum node if err := stack.Register(boot); err != nil { utils.Fatalf("Failed to register the Swarm service: %v", err) } } -func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { - keyid := ctx.GlobalString(SwarmAccountFlag.Name) - - if keyid == "" { - utils.Fatalf("Option %q is required", SwarmAccountFlag.Name) +func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { + //an account is mandatory + if bzzaccount == "" { + utils.Fatalf(SWARM_ERR_NO_BZZACCOUNT) } // Try to load the arg as a hex key file. - if key, err := crypto.LoadECDSA(keyid); err == nil { + if key, err := crypto.LoadECDSA(bzzaccount); err == nil { log.Info("Swarm account key loaded", "address", crypto.PubkeyToAddress(key.PublicKey)) return key } @@ -534,7 +536,7 @@ func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { am := stack.AccountManager() ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - return decryptStoreAccount(ks, keyid, utils.MakePasswordList(ctx)) + return decryptStoreAccount(ks, bzzaccount, utils.MakePasswordList(ctx)) } func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey { @@ -552,7 +554,7 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []stri utils.Fatalf("Can't find swarm account key %s", account) } if err != nil { - utils.Fatalf("Can't find swarm account key: %v", err) + utils.Fatalf("Can't find swarm account key: %v - Is the provided bzzaccount(%s) from the right datadir/Path?", err, account) } keyjson, err := ioutil.ReadFile(a.URL.Path) if err != nil { diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index aaaf9e1e53..ed15028685 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/internal/cmdtest" "github.com/ethereum/go-ethereum/node" @@ -156,9 +157,9 @@ type testNode struct { const testPassphrase = "swarm-test-passphrase" -func newTestNode(t *testing.T, dir string) *testNode { +func getTestAccount(t *testing.T, dir string) (conf *node.Config, account accounts.Account) { // create key - conf := &node.Config{ + conf = &node.Config{ DataDir: dir, IPCPath: "bzzd.ipc", NoUSB: true, @@ -167,18 +168,24 @@ func newTestNode(t *testing.T, dir string) *testNode { if err != nil { t.Fatal(err) } - account, err := n.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore).NewAccount(testPassphrase) + account, err = n.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore).NewAccount(testPassphrase) if err != nil { t.Fatal(err) } - node := &testNode{Dir: dir} - // use a unique IPCPath when running tests on Windows if runtime.GOOS == "windows" { conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", account.Address.String()) } + return conf, account +} + +func newTestNode(t *testing.T, dir string) *testNode { + + conf, account := getTestAccount(t, dir) + node := &testNode{Dir: dir} + // assign ports httpPort, err := assignTCPPort() if err != nil { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index aa999eaae7..30edf199c9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -746,6 +746,12 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error if err != nil || index < 0 { return accounts.Account{}, fmt.Errorf("invalid account address or index %q", account) } + log.Warn("-------------------------------------------------------------------") + log.Warn("Referring to accounts by order in the keystore folder is dangerous!") + log.Warn("This functionality is deprecated and will be removed in the future!") + log.Warn("Please use explicit addresses! (can search via `geth account list`)") + log.Warn("-------------------------------------------------------------------") + accs := ks.Accounts() if len(accs) <= index { return accounts.Account{}, fmt.Errorf("index %d higher than number of accounts %d", index, len(accs)) @@ -762,15 +768,6 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) { Fatalf("Option %q: %v", EtherbaseFlag.Name, err) } cfg.Etherbase = account.Address - return - } - accounts := ks.Accounts() - if (cfg.Etherbase == common.Address{}) { - if len(accounts) > 0 { - cfg.Etherbase = accounts[0].Address - } else { - log.Warn("No etherbase set and no accounts found as default") - } } } diff --git a/console/console.go b/console/console.go index c42c9bfbb3..1ecbfd0b0f 100644 --- a/console/console.go +++ b/console/console.go @@ -192,6 +192,7 @@ func (c *Console) init(preload []string) error { if obj := admin.Object(); obj != nil { // make sure the admin api is enabled over the interface obj.Set("sleepBlocks", bridge.SleepBlocks) obj.Set("sleep", bridge.Sleep) + obj.Set("clearHistory", c.clearHistory) } // Preload any JavaScript files before starting the console for _, path := range preload { @@ -216,6 +217,16 @@ func (c *Console) init(preload []string) error { return nil } +func (c *Console) clearHistory() { + c.history = nil + c.prompter.ClearHistory() + if err := os.Remove(c.histPath); err != nil { + fmt.Fprintln(c.printer, "can't delete history file:", err) + } else { + fmt.Fprintln(c.printer, "history file deleted") + } +} + // consoleOutput is an override for the console.log and console.error methods to // stream the output into the configured output stream instead of stdout. func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value { diff --git a/console/console_test.go b/console/console_test.go index d296807854..05aec2cc0c 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -68,6 +68,7 @@ func (p *hookedPrompter) PromptConfirm(prompt string) (bool, error) { } func (p *hookedPrompter) SetHistory(history []string) {} func (p *hookedPrompter) AppendHistory(command string) {} +func (p *hookedPrompter) ClearHistory() {} func (p *hookedPrompter) SetWordCompleter(completer WordCompleter) {} // tester is a console test environment for the console tests to operate on. diff --git a/console/prompter.go b/console/prompter.go index 6acbfb0e25..ea03694d41 100644 --- a/console/prompter.go +++ b/console/prompter.go @@ -51,6 +51,9 @@ type UserPrompter interface { // if and only if the prompt to append was a valid command. AppendHistory(command string) + // ClearHistory clears the entire history + ClearHistory() + // SetWordCompleter sets the completion function that the prompter will call to // fetch completion candidates when the user presses tab. SetWordCompleter(completer WordCompleter) @@ -158,6 +161,11 @@ func (p *terminalPrompter) AppendHistory(command string) { p.State.AppendHistory(command) } +// ClearHistory clears the entire history +func (p *terminalPrompter) ClearHistory() { + p.State.ClearHistory() +} + // SetWordCompleter sets the completion function that the prompter will call to // fetch completion candidates when the user presses tab. func (p *terminalPrompter) SetWordCompleter(completer WordCompleter) { diff --git a/core/types/transaction.go b/core/types/transaction.go index a46521236c..7e2933bb1a 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -137,7 +137,7 @@ func isProtectedV(V *big.Int) bool { return true } -// DecodeRLP implements rlp.Encoder +// EncodeRLP implements rlp.Encoder func (tx *Transaction) EncodeRLP(w io.Writer) error { return rlp.Encode(w, &tx.data) } diff --git a/crypto/crypto.go b/crypto/crypto.go index 8161769d3d..3a98bfb503 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -98,6 +98,9 @@ func toECDSA(d []byte, strict bool) (*ecdsa.PrivateKey, error) { } priv.D = new(big.Int).SetBytes(d) priv.PublicKey.X, priv.PublicKey.Y = priv.PublicKey.Curve.ScalarBaseMult(d) + if priv.PublicKey.X == nil { + return nil, errors.New("invalid private key") + } return priv, nil } diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index b4c441e5f7..8350354623 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -40,6 +40,15 @@ func TestKeccak256Hash(t *testing.T) { checkhash(t, "Sha3-256-array", func(in []byte) []byte { h := Keccak256Hash(in); return h[:] }, msg, exp) } +func TestToECDSAErrors(t *testing.T) { + if _, err := HexToECDSA("0000000000000000000000000000000000000000000000000000000000000000"); err == nil { + t.Fatal("HexToECDSA should've returned error") + } + if _, err := HexToECDSA("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); err == nil { + t.Fatal("HexToECDSA should've returned error") + } +} + func BenchmarkSha3(b *testing.B) { a := []byte("hello world") for i := 0; i < b.N; i++ { diff --git a/eth/api.go b/eth/api.go index a907c36491..c748f75de4 100644 --- a/eth/api.go +++ b/eth/api.go @@ -452,7 +452,12 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, logConfig *vm.LogConf } statedb, err := blockchain.StateAt(blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1).Root()) if err != nil { - return false, structLogger.StructLogs(), err + switch err.(type) { + case *trie.MissingNodeError: + return false, structLogger.StructLogs(), fmt.Errorf("required historical state unavailable") + default: + return false, structLogger.StructLogs(), err + } } receipts, _, usedGas, err := processor.Process(block, statedb, config) @@ -518,7 +523,12 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common. } msg, context, statedb, err := api.computeTxEnv(blockHash, int(txIndex)) if err != nil { - return nil, err + switch err.(type) { + case *trie.MissingNodeError: + return nil, fmt.Errorf("required historical state unavailable") + default: + return nil, err + } } // Run the transaction with tracing enabled. diff --git a/eth/backend.go b/eth/backend.go index e7f0f57dd4..c39974a2c0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -310,10 +310,17 @@ func (s *Ethereum) Etherbase() (eb common.Address, err error) { } if wallets := s.AccountManager().Wallets(); len(wallets) > 0 { if accounts := wallets[0].Accounts(); len(accounts) > 0 { - return accounts[0].Address, nil + etherbase := accounts[0].Address + + s.lock.Lock() + s.etherbase = etherbase + s.lock.Unlock() + + log.Info("Etherbase automatically configured", "address", etherbase) + return etherbase, nil } } - return common.Address{}, fmt.Errorf("etherbase address must be explicitly specified") + return common.Address{}, fmt.Errorf("etherbase must be explicitly specified") } // set in js console via admin interface or wrapper from cli flags diff --git a/mobile/big.go b/mobile/big.go index 564fbad47c..dd7b158786 100644 --- a/mobile/big.go +++ b/mobile/big.go @@ -62,6 +62,16 @@ func (bi *BigInt) SetInt64(x int64) { bi.bigint.SetInt64(x) } +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (bi *BigInt) Sign() int { + return bi.bigint.Sign() +} + // SetString sets the big int to x. // // The string prefix determines the actual conversion base. A prefix of "0x" or diff --git a/rlp/raw.go b/rlp/raw.go index 6bf1c1df8e..2b3f328f66 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -98,7 +98,7 @@ func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { tagsize = 1 contentsize = uint64(b - 0x80) // Reject strings that should've been single bytes. - if contentsize == 1 && buf[1] < 128 { + if contentsize == 1 && len(buf) > 1 && buf[1] < 128 { return 0, 0, 0, ErrCanonSize } case b < 0xC0: diff --git a/rlp/raw_test.go b/rlp/raw_test.go index bac09d8d4e..2aad042100 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -96,6 +96,7 @@ func TestSplit(t *testing.T) { {input: "F90055", err: ErrCanonSize, rest: "F90055"}, {input: "FA0002FFFF", err: ErrCanonSize, rest: "FA0002FFFF"}, + {input: "81", err: ErrValueTooLarge, rest: "81"}, {input: "8501010101", err: ErrValueTooLarge, rest: "8501010101"}, {input: "C60607080902", err: ErrValueTooLarge, rest: "C60607080902"}, diff --git a/swarm/api/config.go b/swarm/api/config.go index d8d25b1c82..140c938ae0 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -18,15 +18,15 @@ package api import ( "crypto/ecdsa" - "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/services/swap" "github.com/ethereum/go-ethereum/swarm/storage" @@ -46,101 +46,68 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams *network.SyncParams - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - EnsRoot common.Address - NetworkId uint64 + Contract common.Address + EnsRoot common.Address + EnsApi string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + SwapApi string + Cors string + BzzAccount string + BootNodes string } -// config is agnostic to where private key is coming from -// so managing accounts is outside swarm and left to wrappers -func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, networkId uint64) (self *Config, err error) { - address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address - dirpath := filepath.Join(path, "bzz-"+common.Bytes2Hex(address.Bytes())) - err = os.MkdirAll(dirpath, os.ModePerm) - if err != nil { - return - } - confpath := filepath.Join(dirpath, "config.json") - var data []byte - pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) - pubkeyhex := common.ToHex(pubkey) - keyhex := crypto.Keccak256Hash(pubkey).Hex() +//create a default config with all parameters to set to defaults +func NewDefaultConfig() (self *Config) { self = &Config{ - SyncParams: network.NewSyncParams(dirpath), - HiveParams: network.NewHiveParams(dirpath), + StoreParams: storage.NewDefaultStoreParams(), ChunkerParams: storage.NewChunkerParams(), - StoreParams: storage.NewStoreParams(dirpath), + HiveParams: network.NewDefaultHiveParams(), + SyncParams: network.NewDefaultSyncParams(), + Swap: swap.NewDefaultSwapParams(), ListenAddr: DefaultHTTPListenAddr, Port: DefaultHTTPPort, - Path: dirpath, - Swap: swap.DefaultSwapParams(contract, prvKey), - PublicKey: pubkeyhex, - BzzKey: keyhex, + Path: node.DefaultDataDir(), + EnsApi: node.DefaultIPCEndpoint("geth"), EnsRoot: ens.TestNetAddress, - NetworkId: networkId, - } - data, err = ioutil.ReadFile(confpath) - - // if not set in function param, then set default for swarm network, will be overwritten by config file if present - if networkId == 0 { - self.NetworkId = network.NetworkId - } - - if err != nil { - if !os.IsNotExist(err) { - return - } - - // file does not exist - // write out config file - err = self.Save() - if err != nil { - err = fmt.Errorf("error writing config: %v", err) - } - return - } - - // file exists, deserialise - err = json.Unmarshal(data, self) - if err != nil { - return nil, fmt.Errorf("unable to parse config: %v", err) - } - // check public key - if pubkeyhex != self.PublicKey { - return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey) - } - if keyhex != self.BzzKey { - return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) - } - - // if set in function param, replace id set from config file - if networkId != 0 { - self.NetworkId = networkId - } - - self.Swap.SetKey(prvKey) - - if (self.EnsRoot == common.Address{}) { - self.EnsRoot = ens.TestNetAddress + NetworkId: network.NetworkId, + SwapEnabled: false, + SyncEnabled: true, + SwapApi: "", + BootNodes: "", } return } -func (self *Config) Save() error { - data, err := json.MarshalIndent(self, "", " ") +//some config params need to be initialized after the complete +//config building phase is completed (e.g. due to overriding flags) +func (self *Config) Init(prvKey *ecdsa.PrivateKey) { + + address := crypto.PubkeyToAddress(prvKey.PublicKey) + self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes())) + err := os.MkdirAll(self.Path, os.ModePerm) if err != nil { - return err + log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err)) + return } - err = os.MkdirAll(self.Path, os.ModePerm) - if err != nil { - return err - } - confpath := filepath.Join(self.Path, "config.json") - return ioutil.WriteFile(confpath, data, os.ModePerm) + + pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) + pubkeyhex := common.ToHex(pubkey) + keyhex := crypto.Keccak256Hash(pubkey).Hex() + + self.PublicKey = pubkeyhex + self.BzzKey = keyhex + + self.Swap.Init(self.Contract, prvKey) + self.SyncParams.Init(self.Path) + self.HiveParams.Init(self.Path) + self.StoreParams.Init(self.Path) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 85d056270e..2e03a610e1 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -17,109 +17,53 @@ package api import ( - "io/ioutil" - "os" - "path/filepath" - "strings" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) -var ( - hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" - defaultConfig = `{ - "ChunkDbPath": "` + filepath.Join("TMPDIR", "chunks") + `", - "DbCapacity": 5000000, - "CacheCapacity": 5000, - "Radius": 0, - "Branches": 128, - "Hash": "SHA3", - "CallInterval": 3000000000, - "KadDbPath": "` + filepath.Join("TMPDIR", "bzz-peers.json") + `", - "MaxProx": 8, - "ProxBinSize": 2, - "BucketSize": 4, - "PurgeInterval": 151200000000000, - "InitialRetryInterval": 42000000, - "MaxIdleInterval": 42000000000, - "ConnRetryExp": 2, - "Swap": { - "BuyAt": 20000000000, - "SellAt": 20000000000, - "PayAt": 100, - "DropAt": 10000, - "AutoCashInterval": 300000000000, - "AutoCashThreshold": 50000000000000, - "AutoDepositInterval": 300000000000, - "AutoDepositThreshold": 50000000000000, - "AutoDepositBuffer": 100000000000000, - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "Contract": "0x0000000000000000000000000000000000000000", - "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e" - }, - "RequestDbPath": "` + filepath.Join("TMPDIR", "requests") + `", - "RequestDbBatchSize": 512, - "KeyBufferSize": 1024, - "SyncBatchSize": 128, - "SyncBufferSize": 128, - "SyncCacheSize": 1024, - "SyncPriorities": [ - 2, - 1, - 1, - 0, - 0 - ], - "SyncModes": [ - true, - true, - true, - true, - false - ], - "Path": "TMPDIR", - "ListenAddr": "127.0.0.1", - "Port": "8500", - "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", - "EnsRoot": "0x112234455c3a32fd11230c42e7bccd4a84e02010", - "NetworkId": 323 -}` -) +func TestConfig(t *testing.T) { -func TestConfigWriteRead(t *testing.T) { - tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) + var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" prvkey, err := crypto.HexToECDSA(hexprvkey) if err != nil { t.Fatalf("failed to load private key: %v", err) } - orig, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - data, err := ioutil.ReadFile(filepath.Join(orig.Path, "config.json")) - if err != nil { - t.Fatalf("default config file cannot be read: %v", err) - } - exp := strings.Replace(defaultConfig, "TMPDIR", orig.Path, -1) - exp = strings.Replace(exp, "\\", "\\\\", -1) - if string(data) != exp { - t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data)) + + one := NewDefaultConfig() + two := NewDefaultConfig() + + if equal := reflect.DeepEqual(one, two); equal == false { + t.Fatal("Two default configs are not equal") } - conf, err := NewConfig(tmp, common.Address{}, prvkey, 323) - if err != nil { - t.Fatalf("expected no error, got %v", err) + one.Init(prvkey) + + //the init function should set the following fields + if one.BzzKey == "" { + t.Fatal("Expected BzzKey to be set") } - if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() { - t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex()) + if one.PublicKey == "" { + t.Fatal("Expected PublicKey to be set") } + //the Init function should append subdirs to the given path + if one.Swap.PayProfile.Beneficiary == (common.Address{}) { + t.Fatal("Failed to correctly initialize SwapParams") + } + + if one.SyncParams.RequestDbPath == one.Path { + t.Fatal("Failed to correctly initialize SyncParams") + } + + if one.HiveParams.KadDbPath == one.Path { + t.Fatal("Failed to correctly initialize HiveParams") + } + + if one.StoreParams.ChunkDbPath == one.Path { + t.Fatal("Failed to correctly initialize StoreParams") + } } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index d37b7e4006..2504a46100 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -70,19 +70,25 @@ type HiveParams struct { *kademlia.KadParams } -func NewHiveParams(path string) *HiveParams { - kad := kademlia.NewKadParams() +//create default params +func NewDefaultHiveParams() *HiveParams { + kad := kademlia.NewDefaultKadParams() // kad.BucketSize = bucketSize // kad.MaxProx = maxProx // kad.ProxBinSize = proxBinSize return &HiveParams{ CallInterval: callInterval, - KadDbPath: filepath.Join(path, "bzz-peers.json"), KadParams: kad, } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *HiveParams) Init(path string) { + self.KadDbPath = filepath.Join(path, "bzz-peers.json") +} + func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive { kad := kademlia.New(kademlia.Address(addr), params.KadParams) return &Hive{ diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index bf976a3e1d..0abc42a19c 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -52,7 +52,7 @@ type KadParams struct { ConnRetryExp int } -func NewKadParams() *KadParams { +func NewDefaultKadParams() *KadParams { return &KadParams{ MaxProx: maxProx, ProxBinSize: proxBinSize, diff --git a/swarm/network/kademlia/kademlia_test.go b/swarm/network/kademlia/kademlia_test.go index 417ccecae6..88858908a4 100644 --- a/swarm/network/kademlia/kademlia_test.go +++ b/swarm/network/kademlia/kademlia_test.go @@ -63,7 +63,7 @@ func TestOn(t *testing.T) { if !ok1 || !ok2 { t.Errorf("oops") } - kad := New(addr, NewKadParams()) + kad := New(addr, NewDefaultKadParams()) err := kad.On(&testNode{addr: other}, nil) _ = err } @@ -72,7 +72,7 @@ func TestBootstrap(t *testing.T) { test := func(test *bootstrapTest) bool { // for any node kad.le, Target and N - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = test.MaxProx params.BucketSize = test.BucketSize params.ProxBinSize = test.BucketSize @@ -127,7 +127,7 @@ func TestFindClosest(t *testing.T) { test := func(test *FindClosestTest) bool { // for any node kad.le, Target and N - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(test.Self, params) var err error @@ -198,7 +198,7 @@ var ( func TestProxAdjust(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) self := gen(Address{}, r).(Address) - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(self, params) @@ -232,7 +232,7 @@ func TestSaveLoad(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) addresses := gen([]Address{}, r).([]Address) self := RandomAddress() - params := NewKadParams() + params := NewDefaultKadParams() params.MaxProx = 7 kad := New(self, params) diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index d76af022c1..6d729fcb9e 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -131,9 +131,8 @@ type SyncParams struct { } // constructor with default values -func NewSyncParams(bzzdir string) *SyncParams { +func NewDefaultSyncParams() *SyncParams { return &SyncParams{ - RequestDbPath: filepath.Join(bzzdir, "requests"), RequestDbBatchSize: requestDbBatchSize, KeyBufferSize: keyBufferSize, SyncBufferSize: syncBufferSize, @@ -144,6 +143,12 @@ func NewSyncParams(bzzdir string) *SyncParams { } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *SyncParams) Init(path string) { + self.RequestDbPath = filepath.Join(path, "requests") +} + // syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding type syncer struct { *SyncParams // sync parameters diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index 093892e8d9..1f9b22b904 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -80,17 +80,10 @@ type PayProfile struct { lock sync.RWMutex } -func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapParams { - pubkey := &prvkey.PublicKey +//create params with default values +func NewDefaultSwapParams() *SwapParams { return &SwapParams{ - PayProfile: &PayProfile{ - PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)), - Contract: contract, - Beneficiary: crypto.PubkeyToAddress(*pubkey), - privateKey: prvkey, - publicKey: pubkey, - owner: crypto.PubkeyToAddress(*pubkey), - }, + PayProfile: &PayProfile{}, Params: &swap.Params{ Profile: &swap.Profile{ BuyAt: buyAt, @@ -109,6 +102,21 @@ func DefaultSwapParams(contract common.Address, prvkey *ecdsa.PrivateKey) *SwapP } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) { + pubkey := &prvkey.PublicKey + + self.PayProfile = &PayProfile{ + PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)), + Contract: contract, + Beneficiary: crypto.PubkeyToAddress(*pubkey), + privateKey: prvkey, + publicKey: pubkey, + owner: crypto.PubkeyToAddress(*pubkey), + } +} + // swap constructor, parameters // * global chequebook, assume deployed service and // * the balance is at buffer. diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 7b0612edc5..5d4f17deb1 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -57,15 +57,21 @@ type StoreParams struct { Radius int } -func NewStoreParams(path string) (self *StoreParams) { +//create params with default values +func NewDefaultStoreParams() (self *StoreParams) { return &StoreParams{ - ChunkDbPath: filepath.Join(path, "chunks"), DbCapacity: defaultDbCapacity, CacheCapacity: defaultCacheCapacity, Radius: defaultRadius, } } +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *StoreParams) Init(path string) { + self.ChunkDbPath = filepath.Join(path, "chunks") +} + // netstore contructor, takes path argument that is used to initialise dbStore, // the persistent (disk) storage component of LocalStore // the second argument is the hive, the connection/logistics manager for the node diff --git a/swarm/swarm.go b/swarm/swarm.go index 9db15325ae..3be3660b58 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -220,7 +220,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { // stops all component services. func (self *Swarm) Stop() error { self.dpa.Stop() - self.hive.Stop() + err := self.hive.Stop() if ch := self.config.Swap.Chequebook(); ch != nil { ch.Stop() ch.Save() @@ -230,7 +230,7 @@ func (self *Swarm) Stop() error { self.lstore.DbStore.Close() } self.sfs.Stop() - return self.config.Save() + return err } // implements the node.Service interface @@ -301,7 +301,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error { return err } log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex())) - self.config.Save() self.hive.DropAll() return nil } @@ -314,10 +313,9 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { return } - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) - if err != nil { - return - } + config := api.NewDefaultConfig() + config.Path = datadir + config.Init(prvKey) config.Port = port dpa, err := storage.NewLocalDPA(datadir) diff --git a/whisper/whisperv6/benchmarks_test.go b/whisper/whisperv6/benchmarks_test.go index 9f413e7b0a..0473179da5 100644 --- a/whisper/whisperv6/benchmarks_test.go +++ b/whisper/whisperv6/benchmarks_test.go @@ -17,14 +17,16 @@ package whisperv6 import ( + "crypto/sha256" "testing" "github.com/ethereum/go-ethereum/crypto" + "golang.org/x/crypto/pbkdf2" ) func BenchmarkDeriveKeyMaterial(b *testing.B) { for i := 0; i < b.N; i++ { - deriveKeyMaterial([]byte("test"), 0) + pbkdf2.Key([]byte("test"), nil, 65356, aesKeyLength, sha256.New) } } diff --git a/whisper/whisperv6/doc.go b/whisper/whisperv6/doc.go index e64dd2f42d..64925ba48b 100644 --- a/whisper/whisperv6/doc.go +++ b/whisper/whisperv6/doc.go @@ -36,15 +36,15 @@ import ( const ( EnvelopeVersion = uint64(0) - ProtocolVersion = uint64(5) - ProtocolVersionStr = "5.0" + ProtocolVersion = uint64(6) + ProtocolVersionStr = "6.0" ProtocolName = "shh" statusCode = 0 // used by whisper protocol messagesCode = 1 // normal whisper message p2pCode = 2 // peer-to-peer message (to be consumed by the peer, but not forwarded any further) p2pRequestCode = 3 // peer-to-peer message, used by Dapp protocol - NumberOfMessageCodes = 64 + NumberOfMessageCodes = 128 paddingMask = byte(3) signatureFlag = byte(4) @@ -67,6 +67,8 @@ const ( DefaultTTL = 50 // seconds SynchAllowance = 10 // seconds + + EnvelopeHeaderLength = 20 ) type unknownVersionError uint64 diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index c93ab84a28..676df669bf 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -36,13 +36,11 @@ import ( // Envelope represents a clear-text data packet to transmit through the Whisper // network. Its contents may or may not be encrypted and signed. type Envelope struct { - Version []byte - Expiry uint32 - TTL uint32 - Topic TopicType - AESNonce []byte - Data []byte - Nonce uint64 + Expiry uint32 + TTL uint32 + Topic TopicType + Data []byte + Nonce uint64 pow float64 // Message-specific PoW as described in the Whisper specification. hash common.Hash // Cached hash of the envelope to avoid rehashing every time. @@ -51,49 +49,29 @@ type Envelope struct { // size returns the size of envelope as it is sent (i.e. public fields only) func (e *Envelope) size() int { - return 20 + len(e.Version) + len(e.AESNonce) + len(e.Data) + return EnvelopeHeaderLength + len(e.Data) } // rlpWithoutNonce returns the RLP encoded envelope contents, except the nonce. func (e *Envelope) rlpWithoutNonce() []byte { - res, _ := rlp.EncodeToBytes([]interface{}{e.Version, e.Expiry, e.TTL, e.Topic, e.AESNonce, e.Data}) + res, _ := rlp.EncodeToBytes([]interface{}{e.Expiry, e.TTL, e.Topic, e.Data}) return res } // NewEnvelope wraps a Whisper message with expiration and destination data // included into an envelope for network forwarding. -func NewEnvelope(ttl uint32, topic TopicType, aesNonce []byte, msg *sentMessage) *Envelope { +func NewEnvelope(ttl uint32, topic TopicType, msg *sentMessage) *Envelope { env := Envelope{ - Version: make([]byte, 1), - Expiry: uint32(time.Now().Add(time.Second * time.Duration(ttl)).Unix()), - TTL: ttl, - Topic: topic, - AESNonce: aesNonce, - Data: msg.Raw, - Nonce: 0, - } - - if EnvelopeVersion < 256 { - env.Version[0] = byte(EnvelopeVersion) - } else { - panic("please increase the size of Envelope.Version before releasing this version") + Expiry: uint32(time.Now().Add(time.Second * time.Duration(ttl)).Unix()), + TTL: ttl, + Topic: topic, + Data: msg.Raw, + Nonce: 0, } return &env } -func (e *Envelope) IsSymmetric() bool { - return len(e.AESNonce) > 0 -} - -func (e *Envelope) isAsymmetric() bool { - return !e.IsSymmetric() -} - -func (e *Envelope) Ver() uint64 { - return bytesToUintLittleEndian(e.Version) -} - // Seal closes the envelope by spending the requested amount of time as a proof // of work on hashing the data. func (e *Envelope) Seal(options *MessageParams) error { @@ -209,7 +187,7 @@ func (e *Envelope) OpenAsymmetric(key *ecdsa.PrivateKey) (*ReceivedMessage, erro // OpenSymmetric tries to decrypt an envelope, potentially encrypted with a particular key. func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) { msg = &ReceivedMessage{Raw: e.Data} - err = msg.decryptSymmetric(key, e.AESNonce) + err = msg.decryptSymmetric(key) if err != nil { msg = nil } @@ -218,12 +196,18 @@ func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) { // Open tries to decrypt an envelope, and populates the message fields in case of success. func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) { - if e.isAsymmetric() { + // The API interface forbids filters doing both symmetric and + // asymmetric encryption. + if watcher.expectsAsymmetricEncryption() && watcher.expectsSymmetricEncryption() { + return nil + } + + if watcher.expectsAsymmetricEncryption() { msg, _ = e.OpenAsymmetric(watcher.KeyAsym) if msg != nil { msg.Dst = &watcher.KeyAsym.PublicKey } - } else if e.IsSymmetric() { + } else if watcher.expectsSymmetricEncryption() { msg, _ = e.OpenSymmetric(watcher.KeySym) if msg != nil { msg.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym) @@ -240,7 +224,6 @@ func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) { msg.TTL = e.TTL msg.Sent = e.Expiry - e.TTL msg.EnvelopeHash = e.Hash() - msg.EnvelopeVersion = e.Ver() } return msg } diff --git a/whisper/whisperv6/envelope_test.go b/whisper/whisperv6/envelope_test.go new file mode 100644 index 0000000000..410b250a3f --- /dev/null +++ b/whisper/whisperv6/envelope_test.go @@ -0,0 +1,64 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains the tests associated with the Whisper protocol Envelope object. + +package whisperv6 + +import ( + mrand "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/crypto" +) + +func TestEnvelopeOpenAcceptsOnlyOneKeyTypeInFilter(t *testing.T) { + symKey := make([]byte, aesKeyLength) + mrand.Read(symKey) + + asymKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("failed GenerateKey with seed %d: %s.", seed, err) + } + + params := MessageParams{ + PoW: 0.01, + WorkTime: 1, + TTL: uint32(mrand.Intn(1024)), + Payload: make([]byte, 50), + KeySym: symKey, + Dst: nil, + } + + mrand.Read(params.Payload) + + msg, err := NewSentMessage(¶ms) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + + e, err := msg.Wrap(¶ms) + if err != nil { + t.Fatalf("Failed to Wrap the message in an envelope with seed %d: %s", seed, err) + } + + f := Filter{KeySym: symKey, KeyAsym: asymKey} + + decrypted := e.Open(&f) + if decrypted != nil { + t.Fatalf("Managed to decrypt a message with an invalid filter, seed %d", seed) + } +} diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index 5cb371b7d6..2f52dd6b98 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -53,6 +53,10 @@ func NewFilters(w *Whisper) *Filters { } func (fs *Filters) Install(watcher *Filter) (string, error) { + if watcher.KeySym != nil && watcher.KeyAsym != nil { + return "", fmt.Errorf("filters must choose between symmetric and asymmetric keys") + } + if watcher.Messages == nil { watcher.Messages = make(map[common.Hash]*ReceivedMessage) } @@ -175,6 +179,9 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) { return all } +// MatchMessage checks if the filter matches an already decrypted +// message (i.e. a Message that has already been handled by +// MatchEnvelope when checked by a previous filter) func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { if f.PoW > 0 && msg.PoW < f.PoW { return false @@ -188,17 +195,15 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { return false } +// MatchEvelope checks if it's worth decrypting the message. If +// it returns `true`, client code is expected to attempt decrypting +// the message and subsequently call MatchMessage. func (f *Filter) MatchEnvelope(envelope *Envelope) bool { if f.PoW > 0 && envelope.pow < f.PoW { return false } - if f.expectsAsymmetricEncryption() && envelope.isAsymmetric() { - return f.MatchTopic(envelope.Topic) - } else if f.expectsSymmetricEncryption() && envelope.IsSymmetric() { - return f.MatchTopic(envelope.Topic) - } - return false + return f.MatchTopic(envelope.Topic) } func (f *Filter) MatchTopic(topic TopicType) bool { diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 58d90d60c2..dd0de0f6ec 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -229,6 +229,36 @@ func TestInstallIdenticalFilters(t *testing.T) { } } +func TestInstallFilterWithSymAndAsymKeys(t *testing.T) { + InitSingleTest() + + w := New(&Config{}) + filters := NewFilters(w) + filter1, _ := generateFilter(t, true) + + asymKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("Unable to create asymetric keys: %v", err) + } + + // Copy the first filter since some of its fields + // are randomly gnerated. + filter := &Filter{ + KeySym: filter1.KeySym, + KeyAsym: asymKey, + Topics: filter1.Topics, + PoW: filter1.PoW, + AllowP2P: filter1.AllowP2P, + Messages: make(map[common.Hash]*ReceivedMessage), + } + + _, err = filters.Install(filter) + + if err == nil { + t.Fatalf("Error detecting that a filter had both an asymmetric and symmetric key, with seed %d", seed) + } +} + func TestComparePubKey(t *testing.T) { InitSingleTest() @@ -312,12 +342,6 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed) } - // asymmetric + matching topic: mismatch - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope() asymmetric with seed %d.", seed) - } - // symmetric + matching topic + insufficient PoW: mismatch fsym.PoW = env.PoW() + 1.0 match = fsym.MatchEnvelope(env) diff --git a/whisper/whisperv6/message.go b/whisper/whisperv6/message.go index 0815f07a2d..f8df50336e 100644 --- a/whisper/whisperv6/message.go +++ b/whisper/whisperv6/message.go @@ -61,6 +61,7 @@ type ReceivedMessage struct { Payload []byte Padding []byte Signature []byte + Salt []byte PoW float64 // Proof of work as described in the Whisper spec Sent uint32 // Time when the message was posted into the network @@ -69,9 +70,8 @@ type ReceivedMessage struct { Dst *ecdsa.PublicKey // Message recipient (identity used to decode the message) Topic TopicType - SymKeyHash common.Hash // The Keccak256Hash of the key, associated with the Topic - EnvelopeHash common.Hash // Message envelope hash to act as a unique id - EnvelopeVersion uint64 + SymKeyHash common.Hash // The Keccak256Hash of the key, associated with the Topic + EnvelopeHash common.Hash // Message envelope hash to act as a unique id } func isMessageSigned(flags byte) bool { @@ -124,6 +124,10 @@ func (msg *sentMessage) appendPadding(params *MessageParams) error { if params.Src != nil { rawSize += signatureLength } + + if params.KeySym != nil { + rawSize += AESNonceLength + } odd := rawSize % padSizeLimit if len(params.Padding) != 0 { @@ -196,31 +200,31 @@ func (msg *sentMessage) encryptAsymmetric(key *ecdsa.PublicKey) error { // encryptSymmetric encrypts a message with a topic key, using AES-GCM-256. // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize). -func (msg *sentMessage) encryptSymmetric(key []byte) (nonce []byte, err error) { +func (msg *sentMessage) encryptSymmetric(key []byte) (err error) { if !validateSymmetricKey(key) { - return nil, errors.New("invalid key provided for symmetric encryption") + return errors.New("invalid key provided for symmetric encryption") } block, err := aes.NewCipher(key) if err != nil { - return nil, err + return err } aesgcm, err := cipher.NewGCM(block) if err != nil { - return nil, err + return err } // never use more than 2^32 random nonces with a given key - nonce = make([]byte, aesgcm.NonceSize()) - _, err = crand.Read(nonce) + salt := make([]byte, aesgcm.NonceSize()) + _, err = crand.Read(salt) if err != nil { - return nil, err - } else if !validateSymmetricKey(nonce) { - return nil, errors.New("crypto/rand failed to generate nonce") + return err + } else if !validateSymmetricKey(salt) { + return errors.New("crypto/rand failed to generate salt") } - msg.Raw = aesgcm.Seal(nil, nonce, msg.Raw, nil) - return nonce, nil + msg.Raw = append(aesgcm.Seal(nil, salt, msg.Raw, nil), salt...) + return nil } // Wrap bundles the message into an Envelope to transmit over the network. @@ -233,11 +237,10 @@ func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er return nil, err } } - var nonce []byte if options.Dst != nil { err = msg.encryptAsymmetric(options.Dst) } else if options.KeySym != nil { - nonce, err = msg.encryptSymmetric(options.KeySym) + err = msg.encryptSymmetric(options.KeySym) } else { err = errors.New("unable to encrypt the message: neither symmetric nor assymmetric key provided") } @@ -245,7 +248,7 @@ func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er return nil, err } - envelope = NewEnvelope(options.TTL, options.Topic, nonce, msg) + envelope = NewEnvelope(options.TTL, options.Topic, msg) if err = envelope.Seal(options); err != nil { return nil, err } @@ -254,7 +257,14 @@ func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er // decryptSymmetric decrypts a message with a topic key, using AES-GCM-256. // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize). -func (msg *ReceivedMessage) decryptSymmetric(key []byte, nonce []byte) error { +func (msg *ReceivedMessage) decryptSymmetric(key []byte) error { + // In v6, symmetric messages are expected to contain the 12-byte + // "salt" at the end of the payload. + if len(msg.Raw) < AESNonceLength { + return errors.New("missing salt or invalid payload in symmetric message") + } + salt := msg.Raw[len(msg.Raw)-AESNonceLength:] + block, err := aes.NewCipher(key) if err != nil { return err @@ -263,15 +273,16 @@ func (msg *ReceivedMessage) decryptSymmetric(key []byte, nonce []byte) error { if err != nil { return err } - if len(nonce) != aesgcm.NonceSize() { - log.Error("decrypting the message", "AES nonce size", len(nonce)) - return errors.New("wrong AES nonce size") + if len(salt) != aesgcm.NonceSize() { + log.Error("decrypting the message", "AES salt size", len(salt)) + return errors.New("wrong AES salt size") } - decrypted, err := aesgcm.Open(nil, nonce, msg.Raw, nil) + decrypted, err := aesgcm.Open(nil, salt, msg.Raw[:len(msg.Raw)-AESNonceLength], nil) if err != nil { return err } msg.Raw = decrypted + msg.Salt = salt return nil } diff --git a/whisper/whisperv6/message_test.go b/whisper/whisperv6/message_test.go index 912b90f145..c90bcc01ed 100644 --- a/whisper/whisperv6/message_test.go +++ b/whisper/whisperv6/message_test.go @@ -174,10 +174,8 @@ func TestMessageSeal(t *testing.T) { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } params.TTL = 1 - aesnonce := make([]byte, 12) - mrand.Read(aesnonce) - env := NewEnvelope(params.TTL, params.Topic, aesnonce, msg) + env := NewEnvelope(params.TTL, params.Topic, msg) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } @@ -242,7 +240,12 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - f := Filter{KeyAsym: key, KeySym: params.KeySym} + var f Filter + if symmetric { + f = Filter{KeySym: params.KeySym} + } else { + f = Filter{KeyAsym: key} + } decrypted := env.Open(&f) if decrypted == nil { t.Fatalf("failed to open with seed %d.", seed) @@ -413,3 +416,59 @@ func TestPadding(t *testing.T) { singlePaddingTest(t, n) } } + +func TestPaddingAppendedToSymMessages(t *testing.T) { + params := &MessageParams{ + Payload: make([]byte, 246), + KeySym: make([]byte, aesKeyLength), + } + + // Simulate a message with a payload just under 256 so that + // payload + flag + aesnonce > 256. Check that the result + // is padded on the next 256 boundary. + msg := sentMessage{} + msg.Raw = make([]byte, len(params.Payload)+1+AESNonceLength) + + err := msg.appendPadding(params) + + if err != nil { + t.Fatalf("Error appending padding to message %v", err) + return + } + + if len(msg.Raw) != 512 { + t.Errorf("Invalid size %d != 512", len(msg.Raw)) + } +} + +func TestPaddingAppendedToSymMessagesWithSignature(t *testing.T) { + params := &MessageParams{ + Payload: make([]byte, 246), + KeySym: make([]byte, aesKeyLength), + } + + pSrc, err := crypto.GenerateKey() + + if err != nil { + t.Fatalf("Error creating the signature key %v", err) + return + } + params.Src = pSrc + + // Simulate a message with a payload just under 256 so that + // payload + flag + aesnonce > 256. Check that the result + // is padded on the next 256 boundary. + msg := sentMessage{} + msg.Raw = make([]byte, len(params.Payload)+1+AESNonceLength+signatureLength) + + err = msg.appendPadding(params) + + if err != nil { + t.Fatalf("Error appending padding to message %v", err) + return + } + + if len(msg.Raw) != 512 { + t.Errorf("Invalid size %d != 512", len(msg.Raw)) + } +} diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index e2b884f3d4..d09baab3fd 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -367,7 +367,9 @@ func (w *Whisper) AddSymKeyFromPassword(password string) (string, error) { return "", fmt.Errorf("failed to generate unique ID") } - derived, err := deriveKeyMaterial([]byte(password), EnvelopeVersion) + // kdf should run no less than 0.1 seconds on an average computer, + // because it's an once in a session experience + derived := pbkdf2.Key([]byte(password), nil, 65356, aesKeyLength, sha256.New) if err != nil { return "", err } @@ -587,17 +589,6 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { return false, fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash()) } - if len(envelope.Version) > 4 { - return false, fmt.Errorf("oversized version [%x]", envelope.Hash()) - } - - aesNonceSize := len(envelope.AESNonce) - if aesNonceSize != 0 && aesNonceSize != AESNonceLength { - // the standard AES GCM nonce size is 12 bytes, - // but constant gcmStandardNonceSize cannot be accessed (not exported) - return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash()) - } - if envelope.PoW() < wh.MinPow() { log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex()) return false, nil // drop envelope without error @@ -635,16 +626,11 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { // postEvent queues the message for further processing. func (w *Whisper) postEvent(envelope *Envelope, isP2P bool) { - // if the version of incoming message is higher than - // currently supported version, we can not decrypt it, - // and therefore just ignore this message - if envelope.Ver() <= EnvelopeVersion { - if isP2P { - w.p2pMsgQueue <- envelope - } else { - w.checkOverflow() - w.messageQueue <- envelope - } + if isP2P { + w.p2pMsgQueue <- envelope + } else { + w.checkOverflow() + w.messageQueue <- envelope } } @@ -830,19 +816,6 @@ func BytesToUintBigEndian(b []byte) (res uint64) { return res } -// deriveKeyMaterial derives symmetric key material from the key or password. -// pbkdf2 is used for security, in case people use password instead of randomly generated keys. -func deriveKeyMaterial(key []byte, version uint64) (derivedKey []byte, err error) { - if version == 0 { - // kdf should run no less than 0.1 seconds on average compute, - // because it's a once in a session experience - derivedKey := pbkdf2.Key(key, nil, 65356, aesKeyLength, sha256.New) - return derivedKey, nil - } else { - return nil, unknownVersionError(version) - } -} - // GenerateRandomID generates a random string, which is then returned to be used as a key id func GenerateRandomID() (id string, err error) { buf := make([]byte, keyIdSize) diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index c7cea40146..aeb54b3ab8 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -19,11 +19,13 @@ package whisperv6 import ( "bytes" "crypto/ecdsa" + "crypto/sha256" mrand "math/rand" "testing" "time" "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/pbkdf2" ) func TestWhisperBasic(t *testing.T) { @@ -79,14 +81,7 @@ func TestWhisperBasic(t *testing.T) { } var derived []byte - ver := uint64(0xDEADBEEF) - if _, err := deriveKeyMaterial(peerID, ver); err != unknownVersionError(ver) { - t.Fatalf("failed deriveKeyMaterial with param = %v: %s.", peerID, err) - } - derived, err = deriveKeyMaterial(peerID, 0) - if err != nil { - t.Fatalf("failed second deriveKeyMaterial with param = %v: %s.", peerID, err) - } + derived = pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New) if !validateSymmetricKey(derived) { t.Fatalf("failed validateSymmetricKey with param = %v.", derived) }