From 057af8c5c842714feff675faeab089e497ec1739 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 1 Dec 2017 10:32:14 +0100 Subject: [PATCH 001/110] swarm: add CLI --ens-endpoint flag (#14386) Implement a CLI flag that can be repeated to allow multiple ENS resolvers for different TLDs. --- cmd/swarm/main.go | 103 ++++++++++++++---------------- cmd/swarm/main_test.go | 141 +++++++++++++++++++++++++++++++++++++++++ swarm/api/api.go | 64 +++++++++++++++++++ swarm/api/api_test.go | 117 ++++++++++++++++++++++++++++++++++ swarm/swarm.go | 97 +++++++++++++++++++++++++--- 5 files changed, 459 insertions(+), 63 deletions(-) create mode 100644 cmd/swarm/main_test.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 603fd9b941..98a0352dca 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -17,11 +17,9 @@ package main import ( - "context" "crypto/ecdsa" "fmt" "io/ioutil" - "math/big" "os" "os/signal" "runtime" @@ -29,14 +27,13 @@ import ( "strconv" "strings" "syscall" - "time" + "unicode" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" - "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -45,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" "gopkg.in/urfave/cli.v1" @@ -101,6 +97,10 @@ var ( Name: "sync", Usage: "Swarm Syncing enabled (default true)", } + EnsEndpointsFlag = cli.StringSliceFlag{ + Name: "ens-endpoint", + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", + } EnsAPIFlag = cli.StringFlag{ Name: "ens-api", Usage: "URL of the Ethereum API provider to use for ENS record lookups", @@ -323,6 +323,7 @@ DEPRECATED: use 'swarm db clean'. utils.PasswordFileFlag, // bzzd-specific flags CorsStringFlag, + EnsEndpointsFlag, EnsAPIFlag, EnsAddrFlag, SwarmConfigPathFlag, @@ -416,38 +417,6 @@ func bzzd(ctx *cli.Context) error { return nil } -// detectEnsAddr determines the ENS contract address by getting both the -// version and genesis hash using the client and matching them to either -// mainnet or testnet addresses -func detectEnsAddr(client *rpc.Client) (common.Address, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - var version string - if err := client.CallContext(ctx, &version, "net_version"); err != nil { - return common.Address{}, err - } - - block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) - if err != nil { - return common.Address{}, err - } - - switch { - - case version == "1" && block.Hash() == params.MainnetGenesisHash: - log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) - return ens.MainNetAddress, nil - - case version == "3" && block.Hash() == params.TestnetGenesisHash: - log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) - return ens.TestNetAddress, nil - - default: - return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) - } -} - func registerBzzService(ctx *cli.Context, stack *node.Node) { prvkey := getAccount(ctx, stack) @@ -476,6 +445,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { utils.Fatalf("SWAP is enabled but --swap-api is not set") } + ensEndpoints := ctx.GlobalStringSlice(EnsEndpointsFlag.Name) ensapi := ctx.GlobalString(EnsAPIFlag.Name) ensAddr := ctx.GlobalString(EnsAddrFlag.Name) @@ -491,34 +461,59 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } - var ensClient *ethclient.Client + ensClientConfigs := []swarm.ENSClientConfig{} if ensapi != "" { - log.Info("connecting to ENS API", "url", ensapi) - client, err := rpc.Dial(ensapi) - if err != nil { - return nil, fmt.Errorf("error connecting to ENS API %s: %s", ensapi, err) - } - ensClient = ethclient.NewClient(client) - - if ensAddr != "" { - bzzconfig.EnsRoot = common.HexToAddress(ensAddr) - } else { - ensAddr, err := detectEnsAddr(client) - if err == nil { - bzzconfig.EnsRoot = ensAddr - } else { - log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", bzzconfig.EnsRoot), "err", err) + ensClientConfigs = append(ensClientConfigs, swarm.ENSClientConfig{ + Endpoint: ensapi, + ContractAddress: ensAddr, + }) + } + if ensEndpoints != nil { + for _, s := range ensEndpoints { + if s != "" { + ensClientConfigs = append(ensClientConfigs, parseFlagEnsEndpoint(s)) } } } + if len(ensClientConfigs) == 0 { + log.Warn("No ENS, please specify non-empty --ens-api or --ens-endpoint to use domain name resolution") + } - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, swapEnabled, syncEnabled, cors) + return swarm.NewSwarm(ctx, swapClient, ensClientConfigs, bzzconfig, swapEnabled, syncEnabled, cors) } if err := stack.Register(boot); err != nil { utils.Fatalf("Failed to register the Swarm service: %v", err) } } +func parseFlagEnsEndpoint(s string) swarm.ENSClientConfig { + isAllLetterString := func(s string) bool { + for _, r := range s { + if !unicode.IsLetter(r) { + return false + } + } + return true + } + endpoint := s + var addr, tld string + if i := strings.Index(endpoint, ":"); i > 0 { + if isAllLetterString(endpoint[:i]) && len(endpoint) > i+2 && endpoint[i+1:i+3] != "//" { + tld = endpoint[:i] + endpoint = endpoint[i+1:] + } + } + if i := strings.Index(endpoint, "@"); i > 0 { + addr = endpoint[:i] + endpoint = endpoint[i+1:] + } + return swarm.ENSClientConfig{ + Endpoint: endpoint, + ContractAddress: addr, + TLD: tld, + } +} + func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { keyid := ctx.GlobalString(SwarmAccountFlag.Name) diff --git a/cmd/swarm/main_test.go b/cmd/swarm/main_test.go new file mode 100644 index 0000000000..054034434b --- /dev/null +++ b/cmd/swarm/main_test.go @@ -0,0 +1,141 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "testing" + + "github.com/ethereum/go-ethereum/swarm" +) + +func TestParseFlagEnsEndpoint(t *testing.T) { + for _, x := range []struct { + description string + value string + config swarm.ENSClientConfig + }{ + { + description: "IPC endpoint", + value: "/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + }, + }, + { + description: "HTTP endpoint", + value: "http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + }, + }, + { + description: "WS endpoint", + value: "ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + }, + }, + { + description: "IPC Endpoint and TLD", + value: "test:/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + TLD: "test", + }, + }, + { + description: "HTTP endpoint and TLD", + value: "test:http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + TLD: "test", + }, + }, + { + description: "WS endpoint and TLD", + value: "test:ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + TLD: "test", + }, + }, + { + description: "IPC Endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + }, + }, + { + description: "HTTP endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + }, + }, + { + description: "WS endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + }, + }, + { + description: "IPC Endpoint, TLD and contract address", + value: "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + TLD: "test", + }, + }, + { + description: "HTTP endpoint, TLD and contract address", + value: "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + TLD: "eth", + }, + }, + { + description: "WS endpoint, TLD and contract address", + value: "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + TLD: "eth", + }, + }, + } { + t.Run(x.description, func(t *testing.T) { + config := parseFlagEnsEndpoint(x.value) + if config.Endpoint != x.config.Endpoint { + t.Errorf("expected Endpoint %q, got %q", x.config.Endpoint, config.Endpoint) + } + if config.ContractAddress != x.config.ContractAddress { + t.Errorf("expected ContractAddress %q, got %q", x.config.ContractAddress, config.ContractAddress) + } + if config.TLD != x.config.TLD { + t.Errorf("expected TLD %q, got %q", x.config.TLD, config.TLD) + } + }) + } +} diff --git a/swarm/api/api.go b/swarm/api/api.go index 79de29a1cf..d0c3754e00 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,6 +17,7 @@ package api import ( + "errors" "fmt" "io" "net/http" @@ -40,6 +41,69 @@ type Resolver interface { Resolve(string) (common.Hash, error) } +// errNoResolver is returned by MultiResolver.Resolve if no resolver +// can be found for the address. +var errNoResolver = errors.New("no resolver") + +// MultiResolver is used to resolve URL addresses based on their TLDs. +// Each TLD can have multiple resolvers, and the resoluton from the +// first one in the sequence will be returned. +type MultiResolver struct { + resolvers map[string][]Resolver +} + +// MultiResolverOption sets options for MultiResolver and is used as +// arguments for its constructor. +type MultiResolverOption func(*MultiResolver) + +// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers +// for a specific TLD. If TLD is an empty string, the resolver will be added +// to the list of default resolver, the ones that will be used for resolution +// of addresses which do not have their TLD resolver specified. +func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { + return func(m *MultiResolver) { + if _, ok := m.resolvers[tld]; !ok { + m.resolvers[tld] = []Resolver{} + } + m.resolvers[tld] = append(m.resolvers[tld], r) + } +} + +// NewMultiResolver creates a new instance of MultiResolver. +func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { + m = &MultiResolver{ + resolvers: map[string][]Resolver{}, + } + for _, o := range opts { + o(m) + } + return m +} + +// Resolve resolves address by choosing a Resolver by TLD. +// If there are more default Resolvers, or for a specific TLD, +// the Hash from the the first one which does not return error +// will be returned. +func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { + rs, _ := m.resolvers[""] + if i := strings.LastIndex(addr, "."); i >= 0 { + rstld, ok := m.resolvers[addr[i+1:]] + if ok { + rs = rstld + } + } + if rs == nil { + return h, errNoResolver + } + for _, r := range rs { + h, err = r.Resolve(addr) + if err == nil { + return + } + } + return +} + /* Api implements webserver/file system related content storage and retrieval on top of the dpa diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index f9caed27f5..36cbdb83c5 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -237,3 +237,120 @@ func TestAPIResolve(t *testing.T) { }) } } + +func TestMultiResolver(t *testing.T) { + doesntResolve := newTestResolver("") + + ethAddr := "swarm.eth" + ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222" + ethResolve := newTestResolver(ethHash) + + testAddr := "swarm.test" + testHash := "0x1111111111111111111111111111111111111111111111111111111111111111" + testResolve := newTestResolver(testHash) + + tests := []struct { + desc string + r Resolver + addr string + result string + err error + }{ + { + desc: "No resolvers, returns error", + r: NewMultiResolver(), + err: errNoResolver, + }, + { + desc: "One default resolver, returns resolved address", + r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, first doesn't resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Three TLD resolvers, third resolves, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One TLD resolver doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One defautl and one TLD resolver, all doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + err: errors.New(`DNS name not found: "swarm.eth"`), + }, + { + desc: "Two TLD resolvers, both resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + MultiResolverOptionWithResolver(testResolve, "test"), + ), + addr: testAddr, + result: testHash, + }, + } + for _, x := range tests { + t.Run(x.desc, func(t *testing.T) { + res, err := x.r.Resolve(x.addr) + if err == nil { + if x.err != nil { + t.Fatalf("expected error %q, got result %q", x.err, res.Hex()) + } + if res.Hex() != x.result { + t.Fatalf("expected result %q, got %q", x.result, res.Hex()) + } + } else { + if x.err == nil { + t.Fatalf("expected no error, got %q", err) + } + if err.Error() != x.err.Error() { + t.Fatalf("expected error %q, got %q", x.err, err) + } + } + }) + } +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 9db15325ae..9b6abc5b96 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -21,7 +21,9 @@ import ( "context" "crypto/ecdsa" "fmt" + "math/big" "net" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -33,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" @@ -76,7 +79,7 @@ func (self *Swarm) API() *SwarmAPI { // creates a new swarm service instance // implements node.Service -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClientConfigs []ENSClientConfig, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") } @@ -133,18 +136,22 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) - // set up high level api - transactOpts := bind.NewKeyedTransactor(self.privateKey) - - if ensClient == nil { - log.Warn("No ENS, please specify non-empty --ens-api to use domain name resolution") - } else { - self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, ensClient) + if len(ensClientConfigs) == 1 { + self.dns, err = newEnsClient(ensClientConfigs[0].Endpoint, ensClientConfigs[0].ContractAddress, config) if err != nil { return nil, err } + } else if len(ensClientConfigs) > 1 { + opts := []api.MultiResolverOption{} + for _, c := range ensClientConfigs { + r, err := newEnsClient(c.Endpoint, c.ContractAddress, config) + if err != nil { + return nil, err + } + opts = append(opts, api.MultiResolverOptionWithResolver(r, c.TLD)) + } + self.dns = api.NewMultiResolver(opts...) } - log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) self.api = api.NewApi(self.dpa, self.dns) // Manifests for Smart Hosting @@ -156,6 +163,78 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e return self, nil } +// ENSClientConfig holds information to construct ENS resolver. If TLD is non-empty, +// the resolver will be used only for that TLD. If ContractAddress is empty, +// it will be discovered by the client, or used one from the configuration. +type ENSClientConfig struct { + Endpoint string + ContractAddress string + TLD string +} + +// newEnsClient creates a new ENS client for that is a consumer of +// a ENS API on a specific endpoint. It is used as a helper function +// for creating multiple resolvers in NewSwarm function. +func newEnsClient(endpoint, addr string, config *api.Config) (*ens.ENS, error) { + log.Info("connecting to ENS API", "url", endpoint) + client, err := rpc.Dial(endpoint) + if err != nil { + return nil, fmt.Errorf("error connecting to ENS API %s: %s", endpoint, err) + } + ensClient := ethclient.NewClient(client) + + ensRoot := config.EnsRoot + if addr != "" { + ensRoot = common.HexToAddress(addr) + } else { + a, err := detectEnsAddr(client) + if err == nil { + ensRoot = a + } else { + log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", ensRoot), "err", err) + } + } + transactOpts := bind.NewKeyedTransactor(config.Swap.PrivateKey()) + dns, err := ens.NewENS(transactOpts, ensRoot, ensClient) + if err != nil { + return nil, err + } + log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar %v @ address %v", endpoint, ensRoot.Hex())) + return dns, err +} + +// detectEnsAddr determines the ENS contract address by getting both the +// version and genesis hash using the client and matching them to either +// mainnet or testnet addresses +func detectEnsAddr(client *rpc.Client) (common.Address, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var version string + if err := client.CallContext(ctx, &version, "net_version"); err != nil { + return common.Address{}, err + } + + block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) + if err != nil { + return common.Address{}, err + } + + switch { + + case version == "1" && block.Hash() == params.MainnetGenesisHash: + log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) + return ens.MainNetAddress, nil + + case version == "3" && block.Hash() == params.TestnetGenesisHash: + log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) + return ens.TestNetAddress, nil + + default: + return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) + } +} + /* Start is called when the stack is started * starts the network kademlia hive peer management From 7898e0d585ac16ca80ddef3ef6933cc6d12ba576 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 12:44:24 +0100 Subject: [PATCH 002/110] swarm: multiple --ens-api flags Allow multiple --ens-api flags to be specified with value format [tld:][contract-addr@]url. Backward compatibility with only one --ens-api flag and --ens-addr flag is preserved and conflict cases are handled: - multiple --ens-api with --ens-addr returns an error - single --ens-api with contract address and --ens-addr with different contract address returns an error Previously implemented --ens-endpoint is removed. Its functionality is replaced with multiple --ens-api flags. --- cmd/swarm/main.go | 48 ++++++++++++++++++++++++++---------------- cmd/swarm/main_test.go | 4 ++-- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 98a0352dca..70cc42425b 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -97,14 +97,9 @@ var ( Name: "sync", Usage: "Swarm Syncing enabled (default true)", } - EnsEndpointsFlag = cli.StringSliceFlag{ - Name: "ens-endpoint", - Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", - } - EnsAPIFlag = cli.StringFlag{ + EnsAPIFlag = cli.StringSliceFlag{ Name: "ens-api", - Usage: "URL of the Ethereum API provider to use for ENS record lookups", - Value: node.DefaultIPCEndpoint("geth"), + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", } EnsAddrFlag = cli.StringFlag{ Name: "ens-addr", @@ -323,7 +318,6 @@ DEPRECATED: use 'swarm db clean'. utils.PasswordFileFlag, // bzzd-specific flags CorsStringFlag, - EnsEndpointsFlag, EnsAPIFlag, EnsAddrFlag, SwarmConfigPathFlag, @@ -445,8 +439,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { utils.Fatalf("SWAP is enabled but --swap-api is not set") } - ensEndpoints := ctx.GlobalStringSlice(EnsEndpointsFlag.Name) - ensapi := ctx.GlobalString(EnsAPIFlag.Name) + ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) ensAddr := ctx.GlobalString(EnsAddrFlag.Name) cors := ctx.GlobalString(CorsStringFlag.Name) @@ -462,21 +455,40 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } ensClientConfigs := []swarm.ENSClientConfig{} - if ensapi != "" { + switch len(ensAPIs) { + case 0: ensClientConfigs = append(ensClientConfigs, swarm.ENSClientConfig{ - Endpoint: ensapi, + Endpoint: node.DefaultIPCEndpoint("geth"), ContractAddress: ensAddr, }) - } - if ensEndpoints != nil { - for _, s := range ensEndpoints { + case 1: + // Check if only one --ens-api is specified in order to use --ens-addr value + // to preserve the backward compatibility with single --ens-api flag. + // Multiple + c := parseFlagEnsAPI(ensAPIs[0]) + if ensAddr != "" { + // If contract address is specified in both cases, check for conflict. + if c.ContractAddress != "" && ensAddr != c.ContractAddress { + utils.Fatalf("--ens-addr flag in conflict with --ens-api flag contract address") + } + c.ContractAddress = ensAddr + } + ensClientConfigs = append(ensClientConfigs, c) + default: + // Backward compatibility with single --ens-api flag and --ens-addr is preserved. + // Check for case where multiple --ens-api flags are set with --ens-addr where + // the specified contract address is not clear to which api belongs. + if ensAddr != "" { + utils.Fatalf("--ens-addr flag can not be used with multiple --ens-api flags") + } + for _, s := range ensAPIs { if s != "" { - ensClientConfigs = append(ensClientConfigs, parseFlagEnsEndpoint(s)) + ensClientConfigs = append(ensClientConfigs, parseFlagEnsAPI(s)) } } } if len(ensClientConfigs) == 0 { - log.Warn("No ENS, please specify non-empty --ens-api or --ens-endpoint to use domain name resolution") + log.Warn("No ENS, please specify non-empty --ens-api to use domain name resolution") } return swarm.NewSwarm(ctx, swapClient, ensClientConfigs, bzzconfig, swapEnabled, syncEnabled, cors) @@ -486,7 +498,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } -func parseFlagEnsEndpoint(s string) swarm.ENSClientConfig { +func parseFlagEnsAPI(s string) swarm.ENSClientConfig { isAllLetterString := func(s string) bool { for _, r := range s { if !unicode.IsLetter(r) { diff --git a/cmd/swarm/main_test.go b/cmd/swarm/main_test.go index 054034434b..f815f3387d 100644 --- a/cmd/swarm/main_test.go +++ b/cmd/swarm/main_test.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/swarm" ) -func TestParseFlagEnsEndpoint(t *testing.T) { +func TestParseFlagEnsAPI(t *testing.T) { for _, x := range []struct { description string value string @@ -126,7 +126,7 @@ func TestParseFlagEnsEndpoint(t *testing.T) { }, } { t.Run(x.description, func(t *testing.T) { - config := parseFlagEnsEndpoint(x.value) + config := parseFlagEnsAPI(x.value) if config.Endpoint != x.config.Endpoint { t.Errorf("expected Endpoint %q, got %q", x.config.Endpoint, config.Endpoint) } From b33a051a487bed2a010b62a6f2dc531157341013 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:20:29 +0100 Subject: [PATCH 003/110] swarm: add comment for parseFlagEnsAPI and fix a mistake in comment in code --- cmd/swarm/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 70cc42425b..d9744c011d 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -464,7 +464,6 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { case 1: // Check if only one --ens-api is specified in order to use --ens-addr value // to preserve the backward compatibility with single --ens-api flag. - // Multiple c := parseFlagEnsAPI(ensAPIs[0]) if ensAddr != "" { // If contract address is specified in both cases, check for conflict. @@ -498,6 +497,9 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } +// parseFlagEnsAPI parses EnsAPIFlag according to format +// [tld:][contract-addr@]url and returns ENSClientConfig structure +// with endpoint, contract address and TLD. func parseFlagEnsAPI(s string) swarm.ENSClientConfig { isAllLetterString := func(s string) bool { for _, r := range s { From 15ad6f27da1bbcb4219f0786d3d08d52c27a4385 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:28:11 +0100 Subject: [PATCH 004/110] swarm: check if "--ens-api ''" is specified in order to disable ENS --- cmd/swarm/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index d9744c011d..a937a91191 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -462,6 +462,10 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { ContractAddress: ensAddr, }) case 1: + // Check if "--ens-api ''" is specified in order to disable ENS. + if ensAPIs[0] == "" { + break + } // Check if only one --ens-api is specified in order to use --ens-addr value // to preserve the backward compatibility with single --ens-api flag. c := parseFlagEnsAPI(ensAPIs[0]) From 34edbc8868b84e15d7102a1164c7af8ef4873882 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:29:37 +0100 Subject: [PATCH 005/110] swarm/api: remove unneeded assignment in MultiResolverOptionWithResolver --- swarm/api/api.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index d0c3754e00..3682ab3092 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -62,9 +62,6 @@ type MultiResolverOption func(*MultiResolver) // of addresses which do not have their TLD resolver specified. func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { return func(m *MultiResolver) { - if _, ok := m.resolvers[tld]; !ok { - m.resolvers[tld] = []Resolver{} - } m.resolvers[tld] = append(m.resolvers[tld], r) } } From a758b5cf7ac3111c053bc1275ca849f7aae080e1 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:31:00 +0100 Subject: [PATCH 006/110] swarm/api: initialize map with make to comply with the convention --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 3682ab3092..d6e9d37327 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -69,7 +69,7 @@ func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption // NewMultiResolver creates a new instance of MultiResolver. func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { m = &MultiResolver{ - resolvers: map[string][]Resolver{}, + resolvers: make(map[string][]Resolver), } for _, o := range opts { o(m) From 3732c15faa83e60024ef46b3def0c105c6912c47 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:32:33 +0100 Subject: [PATCH 007/110] swarm/api: remove unneeded blank assignement --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index d6e9d37327..8bd3866434 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -82,7 +82,7 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // the Hash from the the first one which does not return error // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { - rs, _ := m.resolvers[""] + rs := m.resolvers[""] if i := strings.LastIndex(addr, "."); i >= 0 { rstld, ok := m.resolvers[addr[i+1:]] if ok { From e451b65faef95b159c38a607f716866e7bd3310f Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:41:21 +0100 Subject: [PATCH 008/110] swarm: deprecate --ens-addr CLI flag with a warning message --- cmd/swarm/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index a937a91191..48cf032a24 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -101,10 +101,6 @@ var ( Name: "ens-api", Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", } - EnsAddrFlag = cli.StringFlag{ - Name: "ens-addr", - Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)", - } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", Usage: "Swarm HTTP endpoint", @@ -140,6 +136,10 @@ var ( Name: "ethapi", Usage: "DEPRECATED: please use --ens-api and --swap-api", } + DeprecatedEnsAddrFlag = cli.StringFlag{ + Name: "ens-addr", + Usage: "DEPRECATED: ENS contract address, please use --ens-api with contract address according to its format", + } ) var defaultNodeConfig = node.DefaultConfig @@ -319,7 +319,6 @@ DEPRECATED: use 'swarm db clean'. // bzzd-specific flags CorsStringFlag, EnsAPIFlag, - EnsAddrFlag, SwarmConfigPathFlag, SwarmSwapEnabledFlag, SwarmSwapAPIFlag, @@ -338,6 +337,7 @@ DEPRECATED: use 'swarm db clean'. SwarmUploadMimeType, //deprecated flags DeprecatedEthAPIFlag, + DeprecatedEnsAddrFlag, } app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { @@ -440,7 +440,11 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) - ensAddr := ctx.GlobalString(EnsAddrFlag.Name) + ensAddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name) + + if ensAddr != "" { + log.Warn("--ens-addr is no longer a valid command line flag, please use --ens-api to specify contract address.") + } cors := ctx.GlobalString(CorsStringFlag.Name) From 1dc19de5da64962a98a37bbc7b93a3895d2eb6e6 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 4 Dec 2017 22:56:52 +0100 Subject: [PATCH 009/110] swarm/api: use path.Ext instead strings.LastIndex in MultiResolver.Resolve --- swarm/api/api.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 8bd3866434..2d9db022e4 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "net/http" + "path" "regexp" "strings" "sync" @@ -83,8 +84,8 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs := m.resolvers[""] - if i := strings.LastIndex(addr, "."); i >= 0 { - rstld, ok := m.resolvers[addr[i+1:]] + if ext := path.Ext(addr); ext != "" { + rstld, ok := m.resolvers[ext[1:]] if ok { rs = rstld } From c0a4d9e1e64a09a19484c8c12e24505d9bacbd57 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 18 Dec 2017 16:22:39 +0100 Subject: [PATCH 010/110] cmd/swarm, swarm: disable ENS API by default Specifying ENS API CLI flag, env variable or configuration field is required for ENS resolving. Backward compatibility is preserved with --ens-api="" CLI flag value. --- cmd/swarm/config.go | 23 +++++------------------ swarm/api/config.go | 2 -- swarm/swarm.go | 23 +++++++---------------- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 9931b12a21..c66a2a4fe5 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -195,17 +195,13 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) } - //EnsAPIs can be set to "", so can't check for empty string, as it is allowed! if ctx.GlobalIsSet(EnsAPIFlag.Name) { ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) - // Disable ENS resolver if --ens-api="" is specified + // preserve backward compatibility to disable ENS with --ens-api="" if len(ensAPIs) == 1 && ensAPIs[0] == "" { - currentConfig.EnsDisabled = true - currentConfig.EnsAPIs = nil - } else { - currentConfig.EnsDisabled = false - currentConfig.EnsAPIs = ensAPIs + ensAPIs = nil } + currentConfig.EnsAPIs = ensAPIs } if ensaddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name); ensaddr != "" { @@ -275,17 +271,8 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) } - //EnsAPIs can be set to "", so can't check for empty string, as it is allowed - if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists { - ensAPIs := strings.Split(ensapi, ",") - // Disable ENS resolver if SWARM_ENS_API="" is specified - if len(ensAPIs) == 0 { - currentConfig.EnsDisabled = true - currentConfig.EnsAPIs = nil - } else { - currentConfig.EnsDisabled = false - currentConfig.EnsAPIs = ensAPIs - } + if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" { + currentConfig.EnsAPIs = strings.Split(ensapi, ",") } if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" { diff --git a/swarm/api/config.go b/swarm/api/config.go index 3f1a2a0742..6b224140a4 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -48,7 +48,6 @@ type Config struct { *network.SyncParams Contract common.Address EnsRoot common.Address - EnsDisabled bool EnsAPIs []string Path string ListenAddr string @@ -78,7 +77,6 @@ func NewDefaultConfig() (self *Config) { Path: node.DefaultDataDir(), EnsAPIs: nil, EnsRoot: ens.TestNetAddress, - EnsDisabled: false, NetworkId: network.NetworkId, SwapEnabled: false, SyncEnabled: true, diff --git a/swarm/swarm.go b/swarm/swarm.go index ae6805dfda..3c77d6eab6 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -138,26 +138,17 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) - if !config.EnsDisabled { - if len(config.EnsAPIs) == 0 { - // ENS is enabled and has no specific configuration, - // use defaults - self.dns, err = newEnsClient(node.DefaultIPCEndpoint("geth"), config.EnsRoot, config) + if len(config.EnsAPIs) > 0 { + opts := []api.MultiResolverOption{} + for _, c := range config.EnsAPIs { + tld, endpoint, addr := parseEnsAPIAddress(c) + r, err := newEnsClient(endpoint, addr, config) if err != nil { return nil, err } - } else { - opts := []api.MultiResolverOption{} - for _, c := range config.EnsAPIs { - tld, endpoint, addr := parseEnsAPIAddress(c) - r, err := newEnsClient(endpoint, addr, config) - if err != nil { - return nil, err - } - opts = append(opts, api.MultiResolverOptionWithResolver(r, tld)) - } - self.dns = api.NewMultiResolver(opts...) + opts = append(opts, api.MultiResolverOptionWithResolver(r, tld)) } + self.dns = api.NewMultiResolver(opts...) } self.api = api.NewApi(self.dpa, self.dns) From 0d6a735a72340130acd6b7e536dad5d8bee40d84 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 18 Dec 2017 23:07:48 +0100 Subject: [PATCH 011/110] swarm/api: implement NoResolverError with information about TLD MultiResolver needs to provide information about TLD that has no resolver configured for. --- swarm/api/api.go | 26 ++++++++++++++++++++------ swarm/api/api_test.go | 10 +++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 2d9db022e4..c143d8a3f9 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,7 +17,6 @@ package api import ( - "errors" "fmt" "io" "net/http" @@ -42,9 +41,22 @@ type Resolver interface { Resolve(string) (common.Hash, error) } -// errNoResolver is returned by MultiResolver.Resolve if no resolver +// NoResolverError is returned by MultiResolver.Resolve if no resolver // can be found for the address. -var errNoResolver = errors.New("no resolver") +type NoResolverError struct { + TLD string +} + +func NewNoResolverError(tld string) *NoResolverError { + return &NoResolverError{TLD: tld} +} + +func (e *NoResolverError) Error() string { + if e.TLD == "" { + return "no ENS resolver" + } + return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD) +} // MultiResolver is used to resolve URL addresses based on their TLDs. // Each TLD can have multiple resolvers, and the resoluton from the @@ -84,14 +96,16 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs := m.resolvers[""] - if ext := path.Ext(addr); ext != "" { - rstld, ok := m.resolvers[ext[1:]] + tld := path.Ext(addr) + if tld != "" { + tld = tld[1:] + rstld, ok := m.resolvers[tld] if ok { rs = rstld } } if rs == nil { - return h, errNoResolver + return h, NewNoResolverError(tld) } for _, r := range rs { h, err = r.Resolve(addr) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 36cbdb83c5..dbe7c1dac4 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -259,7 +259,7 @@ func TestMultiResolver(t *testing.T) { { desc: "No resolvers, returns error", r: NewMultiResolver(), - err: errNoResolver, + err: NewNoResolverError(""), }, { desc: "One default resolver, returns resolved address", @@ -332,6 +332,14 @@ func TestMultiResolver(t *testing.T) { addr: testAddr, result: testHash, }, + { + desc: "One TLD resolver, no default resolver, returns error for different TLD", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: testAddr, + err: NewNoResolverError("test"), + }, } for _, x := range tests { t.Run(x.desc, func(t *testing.T) { From dd5ae4fd8e28d1ce618668d213e81781ef59f067 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 19 Dec 2017 11:47:26 +0100 Subject: [PATCH 012/110] cmd/swarm: add validation for EnsAPIs configuration parameter --- cmd/swarm/config.go | 35 +++++++++++++++ cmd/swarm/config_test.go | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index c66a2a4fe5..566b00f488 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -102,6 +102,8 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) { config = envVarsOverride(config) //override settings provided by command line config = cmdLineOverride(config, ctx) + //validate configuration parameters + err = validateConfig(config) return } @@ -319,6 +321,39 @@ func checkDeprecated(ctx *cli.Context) { } } +//validate configuration parameters +func validateConfig(cfg *bzzapi.Config) (err error) { + for _, ensAPI := range cfg.EnsAPIs { + if ensAPI != "" { + if err := validateEnsAPIs(ensAPI); err != nil { + return fmt.Errorf("invalid format [tld:][contract-addr@]url for ENS API endpoint configuration %q: %v", ensAPI, err) + } + } + } + return nil +} + +//validate EnsAPIs configuration parameter +func validateEnsAPIs(s string) (err error) { + // missing contract address + if strings.HasPrefix(s, "@") { + return errors.New("missing contract address") + } + // missing url + if strings.HasSuffix(s, "@") { + return errors.New("missing url") + } + // missing tld + if strings.HasPrefix(s, ":") { + return errors.New("missing tld") + } + // missing url + if strings.HasSuffix(s, ":") { + return errors.New("missing url") + } + return nil +} + //print a Config as string func printConfig(config *bzzapi.Config) string { out, err := tomlSettings.Marshal(&config) diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 166980d148..9bf584f50c 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -457,3 +457,98 @@ func TestCmdLineOverridesFile(t *testing.T) { node.Shutdown() } + +func TestValidateConfig(t *testing.T) { + for _, c := range []struct { + cfg *api.Config + err string + }{ + { + cfg: &api.Config{EnsAPIs: []string{ + "/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:12344", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"eth:\": missing url", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"314159265dD8dbb310642f98f50C066173C1259b@\": missing url", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + ":314159265dD8dbb310642f98f50C066173C1259", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \":314159265dD8dbb310642f98f50C066173C1259\": missing tld", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "@/data/testnet/geth.ipc", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"@/data/testnet/geth.ipc\": missing contract address", + }, + } { + err := validateConfig(c.cfg) + if c.err != "" && err.Error() != c.err { + t.Errorf("expected error %q, got %q", c.err, err) + } + if c.err == "" && err != nil { + t.Errorf("unexpected error %q", err) + } + } +} From 820cf09c98706f71d4b02b6c25e62db15830f3fb Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 19 Dec 2017 23:05:23 +0100 Subject: [PATCH 013/110] cmd/swarm: return error early in buildConfig function --- cmd/swarm/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 566b00f488..a3c03c00d5 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -98,6 +98,9 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) { config = bzzapi.NewDefaultConfig() //first load settings from config file (if provided) config, err = configFileOverride(config, ctx) + if err != nil { + return nil, err + } //override settings provided by environment variables config = envVarsOverride(config) //override settings provided by command line From e7cd627d93d43ff52d13c66b336e12b96d9bfdef Mon Sep 17 00:00:00 2001 From: croath Date: Fri, 29 Dec 2017 19:46:40 +0800 Subject: [PATCH 014/110] accounts/abi: fix for one output interface crashing --- accounts/abi/argument.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index ad17fbf2b1..65f79c9d4c 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -169,6 +169,16 @@ func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { if err != nil { return err } + + // if we reach this part, there is only one output member from the contract event. + // for mobile, the result type is always a slice. + if reflect.Slice == value.Kind() && value.Len() >= 1 { + //check if it's not a byte slice + if reflect.TypeOf([]byte{}) != value.Type() { + value = value.Index(0).Elem() + } + } + return set(value, reflect.ValueOf(marshalledValue), arg) } From 88e67c552e5da4d114d80be3ff471f4f2ed030f5 Mon Sep 17 00:00:00 2001 From: croath Date: Sun, 31 Dec 2017 19:12:55 +0800 Subject: [PATCH 015/110] accounts/abi: add a test case for unpacking mobile interfaces --- accounts/abi/unpack_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 3910950732..a7901896ab 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -287,6 +287,42 @@ func TestUnpack(t *testing.T) { } } +var unpackMobileTests = []unpackTest{ + { + def: `[{"type": "uint256"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000001", + want: big.NewInt(1), + }, +} + +func TestUnpackMobileOnly(t *testing.T) { + for i, test := range unpackMobileTests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) + abi, err := JSON(strings.NewReader(def)) + if err != nil { + t.Fatalf("invalid ABI definition %s: %v", def, err) + } + encb, err := hex.DecodeString(test.enc) + if err != nil { + t.Fatalf("invalid hex: %s" + test.enc) + } + outptr := reflect.New(reflect.TypeOf(test.want)) + results := make([]interface{}, 1) + copy(results, []interface{}{outptr.Interface()}) + err = abi.Unpack(&results, "method", encb) + if err := test.checkError(err); err != nil { + t.Errorf("test %d (%v) failed: %v", i, test.def, err) + return + } + out := outptr.Elem().Interface() + if !reflect.DeepEqual(test.want, out) { + t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out) + } + }) + } +} + type methodMultiOutput struct { Int *big.Int String string From 407339085fd6fffd15032572bc6ceb472504a451 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 2 Jan 2018 23:30:09 +0100 Subject: [PATCH 016/110] p2p/protocols, p2p/testing: protocol abstraction and testing --- p2p/protocols/protocol.go | 311 ++++++++++++++++++++++++++ p2p/protocols/protocol_test.go | 389 +++++++++++++++++++++++++++++++++ p2p/testing/peerpool.go | 67 ++++++ p2p/testing/protocolsession.go | 206 +++++++++++++++++ p2p/testing/protocoltester.go | 200 +++++++++++++++++ 5 files changed, 1173 insertions(+) create mode 100644 p2p/protocols/protocol.go create mode 100644 p2p/protocols/protocol_test.go create mode 100644 p2p/testing/peerpool.go create mode 100644 p2p/testing/protocolsession.go create mode 100644 p2p/testing/protocoltester.go diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go new file mode 100644 index 0000000000..9914c99587 --- /dev/null +++ b/p2p/protocols/protocol.go @@ -0,0 +1,311 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Package protocols is an extension to p2p. It offers a user friendly simple way to define +devp2p subprotocols by abstracting away code standardly shared by protocols. + +* automate assigments of code indexes to messages +* automate RLP decoding/encoding based on reflecting +* provide the forever loop to read incoming messages +* standardise error handling related to communication +* standardised handshake negotiation +* TODO: automatic generation of wire protocol specification for peers + +*/ +package protocols + +import ( + "context" + "fmt" + "reflect" + "sync" + + "github.com/ethereum/go-ethereum/p2p" +) + +// error codes used by this protocol scheme +const ( + ErrMsgTooLong = iota + ErrDecode + ErrWrite + ErrInvalidMsgCode + ErrInvalidMsgType + ErrHandshake + ErrNoHandler + ErrHandler +) + +// error description strings associated with the codes +var errorToString = map[int]string{ + ErrMsgTooLong: "Message too long", + ErrDecode: "Invalid message (RLP error)", + ErrWrite: "Error sending message", + ErrInvalidMsgCode: "Invalid message code", + ErrInvalidMsgType: "Invalid message type", + ErrHandshake: "Handshake error", + ErrNoHandler: "No handler registered error", + ErrHandler: "Message handler error", +} + +/* +Error implements the standard go error interface. +Use: + + errorf(code, format, params ...interface{}) + +Prints as: + + :
+ +where description is given by code in errorToString +and details is fmt.Sprintf(format, params...) + +exported field Code can be checked +*/ +type Error struct { + Code int + message string + format string + params []interface{} +} + +func (e Error) Error() (message string) { + if len(e.message) == 0 { + name, ok := errorToString[e.Code] + if !ok { + panic("invalid message code") + } + e.message = name + if e.format != "" { + e.message += ": " + fmt.Sprintf(e.format, e.params...) + } + } + return e.message +} + +func errorf(code int, format string, params ...interface{}) *Error { + return &Error{ + Code: code, + format: format, + params: params, + } +} + +// Spec is a protocol specification including its name and version as well as +// the types of messages which are exchanged +type Spec struct { + // Name is the name of the protocol, often a three-letter word + Name string + + // Version is the version number of the protocol + Version uint + + // MaxMsgSize is the maximum accepted length of the message payload + MaxMsgSize uint32 + + // Messages is a list of message data types which this protocol uses, with + // each message type being sent with its array index as the code (so + // [&foo{}, &bar{}, &baz{}] would send foo, bar and baz with codes + // 0, 1 and 2 respectively) + // each message must have a single unique data type + Messages []interface{} + + initOnce sync.Once + codes map[reflect.Type]uint64 + types map[uint64]reflect.Type +} + +func (s *Spec) init() { + s.initOnce.Do(func() { + s.codes = make(map[reflect.Type]uint64, len(s.Messages)) + s.types = make(map[uint64]reflect.Type, len(s.Messages)) + for i, msg := range s.Messages { + code := uint64(i) + typ := reflect.TypeOf(msg) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + s.codes[typ] = code + s.types[code] = typ + } + }) +} + +// Length returns the number of message types in the protocol +func (s *Spec) Length() uint64 { + return uint64(len(s.Messages)) +} + +// GetCode returns the message code of a type, and boolean second argument is +// false if the message type is not found +func (s *Spec) GetCode(msg interface{}) (uint64, bool) { + s.init() + typ := reflect.TypeOf(msg) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + code, ok := s.codes[typ] + return code, ok +} + +// NewMsg construct a new message type given the code +func (s *Spec) NewMsg(code uint64) (interface{}, bool) { + s.init() + typ, ok := s.types[code] + if !ok { + return nil, false + } + return reflect.New(typ).Interface(), true +} + +// Peer represents a remote peer or protocol instance that is running on a peer connection with +// a remote peer +type Peer struct { + *p2p.Peer // the p2p.Peer object representing the remote + rw p2p.MsgReadWriter // p2p.MsgReadWriter to send messages to and read messages from + spec *Spec +} + +// NewPeer constructs a new peer +// this constructor is called by the p2p.Protocol#Run function +// the first two arguments are the arguments passed to p2p.Protocol.Run function +// the third argument is the Spec describing the protocol +func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer { + return &Peer{ + Peer: p, + rw: rw, + spec: spec, + } +} + +// Run starts the forever loop that handles incoming messages +// called within the p2p.Protocol#Run function +// the handler argument is a function which is called for each message received +// from the remote peer, a returned error causes the loop to exit +// resulting in disconnection +func (p *Peer) Run(handler func(msg interface{}) error) error { + for { + if err := p.handleIncoming(handler); err != nil { + return err + } + } +} + +// Drop disconnects a peer. +// TODO: may need to implement protocol drop only? don't want to kick off the peer +// if they are useful for other protocols +func (p *Peer) Drop(err error) { + p.Disconnect(p2p.DiscSubprotocolError) +} + +// Send takes a message, encodes it in RLP, finds the right message code and sends the +// message off to the peer +// this low level call will be wrapped by libraries providing routed or broadcast sends +// but often just used to forward and push messages to directly connected peers +func (p *Peer) Send(msg interface{}) error { + code, found := p.spec.GetCode(msg) + if !found { + return errorf(ErrInvalidMsgType, "%v", code) + } + return p2p.Send(p.rw, code, msg) +} + +// handleIncoming(code) +// is called each cycle of the main forever loop that dispatches incoming messages +// if this returns an error the loop returns and the peer is disconnected with the error +// this generic handler +// * checks message size, +// * checks for out-of-range message codes, +// * handles decoding with reflection, +// * call handlers as callbacks +func (p *Peer) handleIncoming(handle func(msg interface{}) error) error { + msg, err := p.rw.ReadMsg() + if err != nil { + return err + } + // make sure that the payload has been fully consumed + defer msg.Discard() + + if msg.Size > p.spec.MaxMsgSize { + return errorf(ErrMsgTooLong, "%v > %v", msg.Size, p.spec.MaxMsgSize) + } + + val, ok := p.spec.NewMsg(msg.Code) + if !ok { + return errorf(ErrInvalidMsgCode, "%v", msg.Code) + } + if err := msg.Decode(val); err != nil { + return errorf(ErrDecode, "<= %v: %v", msg, err) + } + + // call the registered handler callbacks + // a registered callback take the decoded message as argument as an interface + // which the handler is supposed to cast to the appropriate type + // it is entirely safe not to check the cast in the handler since the handler is + // chosen based on the proper type in the first place + if err := handle(val); err != nil { + return errorf(ErrHandler, "(msg code %v): %v", msg.Code, err) + } + return nil +} + +// Handshake negotiates a handshake on the peer connection +// * arguments +// * context +// * the local handshake to be sent to the remote peer +// * funcion to be called on the remote handshake (can be nil) +// * expects a remote handshake back of the same type +// * the dialing peer needs to send the handshake first and then waits for remote +// * the listening peer waits for the remote handshake and then sends it +// returns the remote handshake and an error +func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interface{}) error) (rhs interface{}, err error) { + if _, ok := p.spec.GetCode(hs); !ok { + return nil, errorf(ErrHandshake, "unknown handshake message type: %T", hs) + } + errc := make(chan error, 2) + handle := func(msg interface{}) error { + rhs = msg + if verify != nil { + return verify(rhs) + } + return nil + } + send := func() { errc <- p.Send(hs) } + receive := func() { errc <- p.handleIncoming(handle) } + + go func() { + if p.Inbound() { + receive() + send() + } else { + send() + receive() + } + }() + + for i := 0; i < 2; i++ { + select { + case err = <-errc: + case <-ctx.Done(): + err = ctx.Err() + } + if err != nil { + return nil, errorf(ErrHandshake, err.Error()) + } + } + return rhs, nil +} diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go new file mode 100644 index 0000000000..b041a650a1 --- /dev/null +++ b/p2p/protocols/protocol_test.go @@ -0,0 +1,389 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" +) + +// handshake message type +type hs0 struct { + C uint +} + +// message to kill/drop the peer with nodeID +type kill struct { + C discover.NodeID +} + +// message to drop connection +type drop struct { +} + +/// protoHandshake represents module-independent aspects of the protocol and is +// the first message peers send and receive as part the initial exchange +type protoHandshake struct { + Version uint // local and remote peer should have identical version + NetworkID string // local and remote peer should have identical network id +} + +// checkProtoHandshake verifies local and remote protoHandshakes match +func checkProtoHandshake(testVersion uint, testNetworkID string) func(interface{}) error { + return func(rhs interface{}) error { + remote := rhs.(*protoHandshake) + if remote.NetworkID != testNetworkID { + return fmt.Errorf("%s (!= %s)", remote.NetworkID, testNetworkID) + } + + if remote.Version != testVersion { + return fmt.Errorf("%d (!= %d)", remote.Version, testVersion) + } + return nil + } +} + +// newProtocol sets up a protocol +// the run function here demonstrates a typical protocol using peerPool, handshake +// and messages registered to handlers +func newProtocol(pp *p2ptest.TestPeerPool) func(*p2p.Peer, p2p.MsgReadWriter) error { + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + protoHandshake{}, + hs0{}, + kill{}, + drop{}, + }, + } + return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := NewPeer(p, rw, spec) + + // initiate one-off protohandshake and check validity + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + phs := &protoHandshake{42, "420"} + hsCheck := checkProtoHandshake(phs.Version, phs.NetworkID) + _, err := peer.Handshake(ctx, phs, hsCheck) + if err != nil { + return err + } + + lhs := &hs0{42} + // module handshake demonstrating a simple repeatable exchange of same-type message + hs, err := peer.Handshake(ctx, lhs, nil) + if err != nil { + return err + } + + if rmhs := hs.(*hs0); rmhs.C > lhs.C { + return fmt.Errorf("handshake mismatch remote %v > local %v", rmhs.C, lhs.C) + } + + handle := func(msg interface{}) error { + switch msg := msg.(type) { + + case *protoHandshake: + return errors.New("duplicate handshake") + + case *hs0: + rhs := msg + if rhs.C > lhs.C { + return fmt.Errorf("handshake mismatch remote %v > local %v", rhs.C, lhs.C) + } + lhs.C += rhs.C + return peer.Send(lhs) + + case *kill: + // demonstrates use of peerPool, killing another peer connection as a response to a message + id := msg.C + pp.Get(id).Drop(errors.New("killed")) + return nil + + case *drop: + // for testing we can trigger self induced disconnect upon receiving drop message + return errors.New("dropped") + + default: + return fmt.Errorf("unknown message type: %T", msg) + } + } + + pp.Add(peer) + defer pp.Remove(peer) + return peer.Run(handle) + } +} + +func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTester { + conf := adapters.RandomNodeConfig() + return p2ptest.NewProtocolTester(t, conf.ID, 2, newProtocol(pp)) +} + +func protoHandshakeExchange(id discover.NodeID, proto *protoHandshake) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: id, + }, + }, + }, + p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 0, + Msg: proto, + Peer: id, + }, + }, + }, + } +} + +func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + // TODO: make this more than one handshake + id := s.IDs[0] + if err := s.TestExchanges(protoHandshakeExchange(id, proto)...); err != nil { + t.Fatal(err) + } + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } +} + +func TestProtoHandshakeVersionMismatch(t *testing.T) { + runProtoHandshake(t, &protoHandshake{41, "420"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 41 (!= 42)").Error())) +} + +func TestProtoHandshakeNetworkIDMismatch(t *testing.T) { + runProtoHandshake(t, &protoHandshake{42, "421"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 421 (!= 420)").Error())) +} + +func TestProtoHandshakeSuccess(t *testing.T) { + runProtoHandshake(t, &protoHandshake{42, "420"}) +} + +func moduleHandshakeExchange(id discover.NodeID, resp uint) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &hs0{42}, + Peer: id, + }, + }, + }, + p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 1, + Msg: &hs0{resp}, + Peer: id, + }, + }, + }, + } +} + +func runModuleHandshake(t *testing.T, resp uint, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + id := s.IDs[0] + if err := s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...); err != nil { + t.Fatal(err) + } + if err := s.TestExchanges(moduleHandshakeExchange(id, resp)...); err != nil { + t.Fatal(err) + } + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } +} + +func TestModuleHandshakeError(t *testing.T) { + runModuleHandshake(t, 43, fmt.Errorf("handshake mismatch remote 43 > local 42")) +} + +func TestModuleHandshakeSuccess(t *testing.T) { + runModuleHandshake(t, 42) +} + +// testing complex interactions over multiple peers, relaying, dropping +func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + p2ptest.Exchange{ + Label: "primary handshake", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: a, + }, + p2ptest.Expect{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: b, + }, + }, + }, + p2ptest.Exchange{ + Label: "module handshake", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: a, + }, + p2ptest.Trigger{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: b, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &hs0{42}, + Peer: a, + }, + p2ptest.Expect{ + Code: 1, + Msg: &hs0{42}, + Peer: b, + }, + }, + }, + + p2ptest.Exchange{Label: "alternative module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: a}, + p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: b}}}, + p2ptest.Exchange{Label: "repeated module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{1}, Peer: a}}}, + p2ptest.Exchange{Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{p2ptest.Expect{Code: 1, Msg: &hs0{43}, Peer: a}}}} +} + +func runMultiplePeers(t *testing.T, peer int, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + + if err := s.TestExchanges(testMultiPeerSetup(s.IDs[0], s.IDs[1])...); err != nil { + t.Fatal(err) + } + // after some exchanges of messages, we can test state changes + // here this is simply demonstrated by the peerPool + // after the handshake negotiations peers must be added to the pool + // time.Sleep(1) + tick := time.NewTicker(10 * time.Millisecond) + timeout := time.NewTimer(1 * time.Second) +WAIT: + for { + select { + case <-tick.C: + if pp.Has(s.IDs[0]) { + break WAIT + } + case <-timeout.C: + t.Fatal("timeout") + } + } + if !pp.Has(s.IDs[1]) { + t.Fatalf("missing peer test-1: %v (%v)", pp, s.IDs) + } + + // peer 0 sends kill request for peer with index + err := s.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 2, + Msg: &kill{s.IDs[peer]}, + Peer: s.IDs[0], + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + // the peer not killed sends a drop request + err = s.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 3, + Msg: &drop{}, + Peer: s.IDs[(peer+1)%2], + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + // check the actual discconnect errors on the individual peers + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } + // test if disconnected peers have been removed from peerPool + if pp.Has(s.IDs[peer]) { + t.Fatalf("peer test-%v not dropped: %v (%v)", peer, pp, s.IDs) + } + +} + +func TestMultiplePeersDropSelf(t *testing.T) { + runMultiplePeers(t, 0, + fmt.Errorf("subprotocol error"), + fmt.Errorf("Message handler error: (msg code 3): dropped"), + ) +} + +func TestMultiplePeersDropOther(t *testing.T) { + runMultiplePeers(t, 1, + fmt.Errorf("Message handler error: (msg code 3): dropped"), + fmt.Errorf("subprotocol error"), + ) +} diff --git a/p2p/testing/peerpool.go b/p2p/testing/peerpool.go new file mode 100644 index 0000000000..45c6e61425 --- /dev/null +++ b/p2p/testing/peerpool.go @@ -0,0 +1,67 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testing + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" +) + +type TestPeer interface { + ID() discover.NodeID + Drop(error) +} + +// TestPeerPool is an example peerPool to demonstrate registration of peer connections +type TestPeerPool struct { + lock sync.Mutex + peers map[discover.NodeID]TestPeer +} + +func NewTestPeerPool() *TestPeerPool { + return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)} +} + +func (self *TestPeerPool) Add(p TestPeer) { + self.lock.Lock() + defer self.lock.Unlock() + log.Trace(fmt.Sprintf("pp add peer %v", p.ID())) + self.peers[p.ID()] = p + +} + +func (self *TestPeerPool) Remove(p TestPeer) { + self.lock.Lock() + defer self.lock.Unlock() + delete(self.peers, p.ID()) +} + +func (self *TestPeerPool) Has(id discover.NodeID) bool { + self.lock.Lock() + defer self.lock.Unlock() + _, ok := self.peers[id] + return ok +} + +func (self *TestPeerPool) Get(id discover.NodeID) TestPeer { + self.lock.Lock() + defer self.lock.Unlock() + return self.peers[id] +} diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go new file mode 100644 index 0000000000..a779aeebbe --- /dev/null +++ b/p2p/testing/protocolsession.go @@ -0,0 +1,206 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testing + +import ( + "errors" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +// ProtocolSession is a quasi simulation of a pivot node running +// a service and a number of dummy peers that can send (trigger) or +// receive (expect) messages +type ProtocolSession struct { + Server *p2p.Server + IDs []discover.NodeID + adapter *adapters.SimAdapter + events chan *p2p.PeerEvent +} + +// Exchange is the basic units of protocol tests +// the triggers and expects in the arrays are run immediately and asynchronously +// thus one cannot have multiple expects for the SAME peer with DIFFERENT message types +// because it's unpredictable which expect will receive which message +// (with expect #1 and #2, messages might be sent #2 and #1, and both expects will complain about wrong message code) +// an exchange is defined on a session +type Exchange struct { + Label string + Triggers []Trigger + Expects []Expect +} + +// Trigger is part of the exchange, incoming message for the pivot node +// sent by a peer +type Trigger struct { + Msg interface{} // type of message to be sent + Code uint64 // code of message is given + Peer discover.NodeID // the peer to send the message to + Timeout time.Duration // timeout duration for the sending +} + +// Expect is part of an exchange, outgoing message from the pivot node +// received by a peer +type Expect struct { + Msg interface{} // type of message to expect + Code uint64 // code of message is now given + Peer discover.NodeID // the peer that expects the message + Timeout time.Duration // timeout duration for receiving +} + +// Disconnect represents a disconnect event, used and checked by TestDisconnected +type Disconnect struct { + Peer discover.NodeID // discconnected peer + Error error // disconnect reason +} + +// trigger sends messages from peers +func (self *ProtocolSession) trigger(trig Trigger) error { + simNode, ok := self.adapter.GetNode(trig.Peer) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", trig.Peer) + } + + errc := make(chan error) + + go func() { + errc <- mockNode.Trigger(&trig) + }() + + t := trig.Timeout + if t == time.Duration(0) { + t = 1000 * time.Millisecond + } + select { + case err := <-errc: + return err + case <-time.After(t): + return fmt.Errorf("timout expecting %v to send to peer %v", trig.Msg, trig.Peer) + } +} + +// expect checks an expectation of a message sent out by the pivot node +func (self *ProtocolSession) expect(exp Expect) error { + if exp.Msg == nil { + return errors.New("no message to expect") + } + simNode, ok := self.adapter.GetNode(exp.Peer) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", exp.Peer, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", exp.Peer) + } + + errc := make(chan error) + go func() { + errc <- mockNode.Expect(&exp) + }() + + t := exp.Timeout + if t == time.Duration(0) { + t = 2000 * time.Millisecond + } + select { + case err := <-errc: + return err + case <-time.After(t): + return fmt.Errorf("timout expecting %v sent to peer %v", exp.Msg, exp.Peer) + } +} + +// TestExchanges tests a series of exchanges against the session +func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { + // launch all triggers of this exchanges + + for _, e := range exchanges { + errc := make(chan error, len(e.Triggers)+len(e.Expects)) + for _, trig := range e.Triggers { + errc <- self.trigger(trig) + } + + // each expectation is spawned in separate go-routine + // expectations of an exchange are conjunctive but unordered, i.e., + // only all of them arriving constitutes a pass + // each expectation is meant to be for a different peer, otherwise they are expected to panic + // testing of an exchange blocks until all expectations are decided + // an expectation is decided if + // expected message arrives OR + // an unexpected message arrives (panic) + // times out on their individual timeout + for _, ex := range e.Expects { + // expect msg spawned to separate go routine + go func(exp Expect) { + errc <- self.expect(exp) + }(ex) + } + + // time out globally or finish when all expectations satisfied + timeout := time.After(5 * time.Second) + for i := 0; i < len(e.Triggers)+len(e.Expects); i++ { + select { + case err := <-errc: + if err != nil { + return fmt.Errorf("exchange failed with: %v", err) + } + case <-timeout: + return fmt.Errorf("exchange %v: '%v' timed out", i, e.Label) + } + } + } + return nil +} + +// TestDisconnected tests the disconnections given as arguments +// the disconnect structs describe what disconnect error is expected on which peer +func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { + expects := make(map[discover.NodeID]error) + for _, disconnect := range disconnects { + expects[disconnect.Peer] = disconnect.Error + } + + timeout := time.After(time.Second) + for len(expects) > 0 { + select { + case event := <-self.events: + if event.Type != p2p.PeerEventTypeDrop { + continue + } + expectErr, ok := expects[event.Peer] + if !ok { + continue + } + + if !(expectErr == nil && event.Error == "" || expectErr != nil && expectErr.Error() == event.Error) { + return fmt.Errorf("unexpected error on peer %v. expected '%v', got '%v'", event.Peer, expectErr, event.Error) + } + delete(expects, event.Peer) + case <-timeout: + return fmt.Errorf("timed out waiting for peers to disconnect") + } + } + return nil +} diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go new file mode 100644 index 0000000000..ea5b106ff8 --- /dev/null +++ b/p2p/testing/protocoltester.go @@ -0,0 +1,200 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +the p2p/testing package provides a unit test scheme to check simple +protocol message exchanges with one pivot node and a number of dummy peers +The pivot test node runs a node.Service, the dummy peers run a mock node +that can be used to send and receive messages +*/ + +package testing + +import ( + "fmt" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" +) + +// ProtocolTester is the tester environment used for unit testing protocol +// message exchanges. It uses p2p/simulations framework +type ProtocolTester struct { + *ProtocolSession + network *simulations.Network +} + +// NewProtocolTester constructs a new ProtocolTester +// it takes as argument the pivot node id, the number of dummy peers and the +// protocol run function called on a peer connection by the p2p server +func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester { + services := adapters.Services{ + "test": func(ctx *adapters.ServiceContext) (node.Service, error) { + return &testNode{run}, nil + }, + "mock": func(ctx *adapters.ServiceContext) (node.Service, error) { + return newMockNode(), nil + }, + } + adapter := adapters.NewSimAdapter(services) + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{}) + if _, err := net.NewNodeWithConfig(&adapters.NodeConfig{ + ID: id, + EnableMsgEvents: true, + Services: []string{"test"}, + }); err != nil { + panic(err.Error()) + } + if err := net.Start(id); err != nil { + panic(err.Error()) + } + + node := net.GetNode(id).Node.(*adapters.SimNode) + peers := make([]*adapters.NodeConfig, n) + peerIDs := make([]discover.NodeID, n) + for i := 0; i < n; i++ { + peers[i] = adapters.RandomNodeConfig() + peers[i].Services = []string{"mock"} + peerIDs[i] = peers[i].ID + } + events := make(chan *p2p.PeerEvent, 1000) + node.SubscribeEvents(events) + ps := &ProtocolSession{ + Server: node.Server(), + IDs: peerIDs, + adapter: adapter, + events: events, + } + self := &ProtocolTester{ + ProtocolSession: ps, + network: net, + } + + self.Connect(id, peers...) + + return self +} + +// Stop stops the p2p server +func (self *ProtocolTester) Stop() error { + self.Server.Stop() + return nil +} + +// Connect brings up the remote peer node and connects it using the +// p2p/simulations network connection with the in memory network adapter +func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { + for _, peer := range peers { + log.Trace(fmt.Sprintf("start node %v", peer.ID)) + if _, err := self.network.NewNodeWithConfig(peer); err != nil { + panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) + } + if err := self.network.Start(peer.ID); err != nil { + panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) + } + log.Trace(fmt.Sprintf("connect to %v", peer.ID)) + if err := self.network.Connect(selfID, peer.ID); err != nil { + panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err)) + } + } + +} + +// testNode wraps a protocol run function and implements the node.Service +// interface +type testNode struct { + run func(*p2p.Peer, p2p.MsgReadWriter) error +} + +func (t *testNode) Protocols() []p2p.Protocol { + return []p2p.Protocol{{ + Length: 100, + Run: t.run, + }} +} + +func (t *testNode) APIs() []rpc.API { + return nil +} + +func (t *testNode) Start(server *p2p.Server) error { + return nil +} + +func (t *testNode) Stop() error { + return nil +} + +// mockNode is a testNode which doesn't actually run a protocol, instead +// exposing channels so that tests can manually trigger and expect certain +// messages +type mockNode struct { + testNode + + trigger chan *Trigger + expect chan *Expect + err chan error + stop chan struct{} + stopOnce sync.Once +} + +func newMockNode() *mockNode { + mock := &mockNode{ + trigger: make(chan *Trigger), + expect: make(chan *Expect), + err: make(chan error), + stop: make(chan struct{}), + } + mock.testNode.run = mock.Run + return mock +} + +// Run is a protocol run function which just loops waiting for tests to +// instruct it to either trigger or expect a message from the peer +func (m *mockNode) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { + for { + select { + case trig := <-m.trigger: + m.err <- p2p.Send(rw, trig.Code, trig.Msg) + case exp := <-m.expect: + m.err <- p2p.ExpectMsg(rw, exp.Code, exp.Msg) + case <-m.stop: + return nil + } + } +} + +func (m *mockNode) Trigger(trig *Trigger) error { + m.trigger <- trig + return <-m.err +} + +func (m *mockNode) Expect(exp *Expect) error { + m.expect <- exp + return <-m.err +} + +func (m *mockNode) Stop() error { + m.stopOnce.Do(func() { close(m.stop) }) + return nil +} From a6787a6308d2109006b036c8a6a331afa938912d Mon Sep 17 00:00:00 2001 From: croath Date: Tue, 23 Jan 2018 19:10:23 +0800 Subject: [PATCH 017/110] accounts/abi: fix the `output` at the beginning instead of making a workaround --- accounts/abi/argument.go | 10 ---------- accounts/abi/unpack_test.go | 36 ------------------------------------ mobile/bind.go | 18 +++++++++++++----- 3 files changed, 13 insertions(+), 51 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 65f79c9d4c..ad17fbf2b1 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -169,16 +169,6 @@ func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { if err != nil { return err } - - // if we reach this part, there is only one output member from the contract event. - // for mobile, the result type is always a slice. - if reflect.Slice == value.Kind() && value.Len() >= 1 { - //check if it's not a byte slice - if reflect.TypeOf([]byte{}) != value.Type() { - value = value.Index(0).Elem() - } - } - return set(value, reflect.ValueOf(marshalledValue), arg) } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index a7901896ab..3910950732 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -287,42 +287,6 @@ func TestUnpack(t *testing.T) { } } -var unpackMobileTests = []unpackTest{ - { - def: `[{"type": "uint256"}]`, - enc: "0000000000000000000000000000000000000000000000000000000000000001", - want: big.NewInt(1), - }, -} - -func TestUnpackMobileOnly(t *testing.T) { - for i, test := range unpackMobileTests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) - abi, err := JSON(strings.NewReader(def)) - if err != nil { - t.Fatalf("invalid ABI definition %s: %v", def, err) - } - encb, err := hex.DecodeString(test.enc) - if err != nil { - t.Fatalf("invalid hex: %s" + test.enc) - } - outptr := reflect.New(reflect.TypeOf(test.want)) - results := make([]interface{}, 1) - copy(results, []interface{}{outptr.Interface()}) - err = abi.Unpack(&results, "method", encb) - if err := test.checkError(err); err != nil { - t.Errorf("test %d (%v) failed: %v", i, test.def, err) - return - } - out := outptr.Elem().Interface() - if !reflect.DeepEqual(test.want, out) { - t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out) - } - }) - } -} - type methodMultiOutput struct { Int *big.Int String string diff --git a/mobile/bind.go b/mobile/bind.go index 7b79bedafd..e8164d523e 100644 --- a/mobile/bind.go +++ b/mobile/bind.go @@ -154,12 +154,20 @@ func (c *BoundContract) GetDeployer() *Transaction { // Call invokes the (constant) contract method with params as input values and // sets the output to result. func (c *BoundContract) Call(opts *CallOpts, out *Interfaces, method string, args *Interfaces) error { - results := make([]interface{}, len(out.objects)) - copy(results, out.objects) - if err := c.contract.Call(&opts.opts, &results, method, args.objects...); err != nil { - return err + if len(out.objects) == 1 { + result := out.objects[0] + if err := c.contract.Call(&opts.opts, result, method, args.objects...); err != nil { + return err + } + out.objects[0] = result + } else { + results := make([]interface{}, len(out.objects)) + copy(results, out.objects) + if err := c.contract.Call(&opts.opts, &results, method, args.objects...); err != nil { + return err + } + copy(out.objects, results) } - copy(out.objects, results) return nil } From 806430a2526b1dd5a85446f56c31df6d37904edd Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 5 Feb 2018 18:18:13 +0100 Subject: [PATCH 018/110] whisper: improve a log message to analyze a travis issue --- whisper/whisperv6/peer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 17f70129b5..dffa7b3507 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -274,7 +274,7 @@ func checkPropagation(t *testing.T, includingNodeZero bool) { time.Sleep(cycle * time.Millisecond) } - t.Fatalf("Test was not complete: timeout %d seconds.", iterations*cycle/1000) + t.Fatalf("Test was not complete: timeout %d seconds. nodes=%v", iterations*cycle/1000, nodes) if !includingNodeZero { f := nodes[0].shh.GetFilter(nodes[0].filerID) From eb07dbb0790fc0f5fe5b3192da6b4d04d844239f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 8 Feb 2018 07:49:23 +0200 Subject: [PATCH 019/110] eth, light: minor light client startup cleanups --- eth/downloader/downloader.go | 1 - light/lightchain.go | 3 +-- light/odr_util.go | 5 +---- light/postprocess.go | 4 ++-- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 7f490d9e9b..3870f10b91 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -266,7 +266,6 @@ func (d *Downloader) Synchronising() bool { // RegisterPeer injects a new download peer into the set of block source to be // used for fetching hashes and blocks from. func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error { - logger := log.New("peer", id) logger.Trace("Registering sync peer") if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil { diff --git a/light/lightchain.go b/light/lightchain.go index 24529ef82e..bc88aeb487 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -100,7 +100,6 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. if cp, ok := trustedCheckpoints[bc.genesisBlock.Hash()]; ok { bc.addTrustedCheckpoint(cp) } - if err := bc.loadLastState(); err != nil { return nil, err } @@ -128,7 +127,7 @@ func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) { if self.odr.BloomIndexer() != nil { self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) } - log.Info("Added trusted checkpoint", "chain name", cp.name) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*ChtFrequency-1, "hash", cp.sectionHead) } func (self *LightChain) getProcInterrupt() bool { diff --git a/light/odr_util.go b/light/odr_util.go index a0eb6303d4..33a8e80ce5 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -58,17 +58,14 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ } } } - if number >= chtCount*ChtFrequency { return nil, ErrNoTrustedCht } - r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number} if err := odr.Retrieve(ctx, r); err != nil { return nil, err - } else { - return r.Header, nil } + return r.Header, nil } func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (common.Hash, error) { diff --git a/light/postprocess.go b/light/postprocess.go index bbac58d121..160d07b175 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -52,7 +52,7 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ - name: "ETH mainnet", + name: "mainnet", sectionIdx: 150, sectionHead: common.HexToHash("1e2e67f289565cbe7bd4367f7960dbd73a3f7c53439e1047cd7ba331c8109e39"), chtRoot: common.HexToHash("f2a6c9ca143d647b44523cc249f1072c8912358ab873a77a5fdc792b8df99e80"), @@ -60,7 +60,7 @@ var ( } ropstenCheckpoint = trustedCheckpoint{ - name: "Ropsten testnet", + name: "ropsten", sectionIdx: 75, sectionHead: common.HexToHash("12e68324f4578ea3e8e7fb3968167686729396c9279287fa1f1a8b51bb2d05b4"), chtRoot: common.HexToHash("3e51dc095c69fa654a4cac766e0afff7357515b4b3c3a379c675f810363e54be"), From 2b4c7e9b37958984525ba63f2ec637662b384090 Mon Sep 17 00:00:00 2001 From: cdetrio Date: Thu, 8 Feb 2018 14:30:26 +0100 Subject: [PATCH 020/110] params: update ropsten bootnodes (#16029) * params: update ropsten bootnodes * params: fix linter --- params/bootnodes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/bootnodes.go b/params/bootnodes.go index f6cbadfc04..c7190ae670 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -33,8 +33,8 @@ var MainnetBootnodes = []string{ // TestnetBootnodes are the enode URLs of the P2P bootstrap nodes running on the // Ropsten test network. var TestnetBootnodes = []string{ - "enode://6ce05930c72abc632c58e2e4324f7c7ea478cec0ed4fa2528982cf34483094e9cbc9216e7aa349691242576d552a2a56aaeae426c5303ded677ce455ba1acd9d@13.84.180.240:30303", // US-TX - "enode://20c9ad97c081d63397d7b685a412227a40e23c8bdc6688c6f37e97cfbc22d2b4d1db1510d8f61e6a8866ad7f0e17c02b14182d37ea7c3c8b9c2683aeb6b733a1@52.169.14.227:30303", // IE + "enode://30b7ab30a01c124a6cceca36863ece12c4f5fa68e3ba9b0b51407ccc002eeed3b3102d20a88f1c1d3c3154e2449317b8ef95090e77b312d5cc39354f86d5d606@52.176.7.10:30303", // US-Azure geth + "enode://865a63255b3bb68023b6bffd5095118fcc13e79dcf014fe4e47e065c350c7cc72af2e53eff895f11ba1bbb6a2b33271c1116ee870f266618eadfc2e78aa7349c@52.176.100.77:30303", // US-Azure parity "enode://6332792c4a00e3e4ee0926ed89e0d27ef985424d97b6a45bf0f23e51f0dcb5e66b875777506458aea7af6f9e4ffb69f43f3778ee73c81ed9d34c51c4b16b0b0f@52.232.243.152:30303", // Parity "enode://94c15d1b9e2fe7ce56e458b9a3b672ef11894ddedd0c6f247e0f1d3487f52b66208fb4aeb8179fce6e3a749ea93ed147c37976d67af557508d199d9594c35f09@192.81.208.223:30303", // @gpip } From c4712bf96bc1bae4a5ad4600e9719e4a74bde7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Thu, 8 Feb 2018 18:06:31 +0100 Subject: [PATCH 021/110] p2p/discv5: fix multiple discovery issues (#16036) * p2p/discv5: add query delay, fix node address update logic, retry refresh if empty * p2p/discv5: remove unnecessary ping before topic query * p2p/discv5: do not filter local address from topicNodes * p2p/discv5: remove canQuery() * p2p/discv5: gofmt --- p2p/discv5/net.go | 40 +++++++++++++++++++++++++--------------- p2p/discv5/ticket.go | 4 ++-- p2p/discv5/udp.go | 20 ++++++++++---------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index f9baf126f1..52c677b623 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -565,11 +565,8 @@ loop: if lookupChn := searchInfo[res.target.topic].lookupChn; lookupChn != nil { lookupChn <- net.ticketStore.radius[res.target.topic].converged } - net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node) []byte { - net.ping(n, n.addr()) - return n.pingEcho - }, func(n *Node, topic Topic) []byte { - if n.state == known { + net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte { + if n.state != nil && n.state.canQuery { return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration } else { if n.state == unknown { @@ -633,15 +630,20 @@ loop: } net.refreshResp <- refreshDone case <-refreshDone: - log.Trace("<-net.refreshDone") - refreshDone = nil - list := searchReqWhenRefreshDone - searchReqWhenRefreshDone = nil - go func() { - for _, req := range list { - net.topicSearchReq <- req - } - }() + log.Trace("<-net.refreshDone", "table size", net.tab.count) + if net.tab.count != 0 { + refreshDone = nil + list := searchReqWhenRefreshDone + searchReqWhenRefreshDone = nil + go func() { + for _, req := range list { + net.topicSearchReq <- req + } + }() + } else { + refreshDone = make(chan struct{}) + net.refresh(refreshDone) + } } } log.Trace("loop stopped") @@ -751,7 +753,15 @@ func (net *Network) internNodeFromNeighbours(sender *net.UDPAddr, rn rpcNode) (n return n, err } if !n.IP.Equal(rn.IP) || n.UDP != rn.UDP || n.TCP != rn.TCP { - err = fmt.Errorf("metadata mismatch: got %v, want %v", rn, n) + if n.state == known { + // reject address change if node is known by us + err = fmt.Errorf("metadata mismatch: got %v, want %v", rn, n) + } else { + // accept otherwise; this will be handled nicer with signed ENRs + n.IP = rn.IP + n.UDP = rn.UDP + n.TCP = rn.TCP + } } return n, err } diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index 37ce8d23cb..b3d1ac4baf 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -494,13 +494,13 @@ func (s *ticketStore) registerLookupDone(lookup lookupInfo, nodes []*Node, ping } } -func (s *ticketStore) searchLookupDone(lookup lookupInfo, nodes []*Node, ping func(n *Node) []byte, query func(n *Node, topic Topic) []byte) { +func (s *ticketStore) searchLookupDone(lookup lookupInfo, nodes []*Node, query func(n *Node, topic Topic) []byte) { now := mclock.Now() for i, n := range nodes { if i == 0 || (binary.BigEndian.Uint64(n.sha[:8])^binary.BigEndian.Uint64(lookup.target[:8])) < s.radius[lookup.topic].minRadius { if lookup.radiusLookup { if lastReq, ok := s.nodeLastReq[n]; !ok || time.Duration(now-lastReq.time) > radiusTC { - s.nodeLastReq[n] = reqInfo{pingHash: ping(n), lookup: lookup, time: now} + s.nodeLastReq[n] = reqInfo{pingHash: nil, lookup: lookup, time: now} } } // else { if s.canQueryTopic(n, lookup.topic) { diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index 5437718173..6ce72d2c15 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -49,7 +49,7 @@ var ( // Timeouts const ( respTimeout = 500 * time.Millisecond - sendTimeout = 500 * time.Millisecond + queryDelay = 1000 * time.Millisecond expiration = 20 * time.Second ntpFailureThreshold = 32 // Continuous timeouts after which to check NTP @@ -318,20 +318,20 @@ func (t *udp) sendTopicRegister(remote *Node, topics []Topic, idx int, pong []by func (t *udp) sendTopicNodes(remote *Node, queryHash common.Hash, nodes []*Node) { p := topicNodes{Echo: queryHash} - if len(nodes) == 0 { - t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p) - return - } - for i, result := range nodes { - if netutil.CheckRelayIP(remote.IP, result.IP) != nil { - continue + var sent bool + for _, result := range nodes { + if result.IP.Equal(t.net.tab.self.IP) || netutil.CheckRelayIP(remote.IP, result.IP) == nil { + p.Nodes = append(p.Nodes, nodeToRPC(result)) } - p.Nodes = append(p.Nodes, nodeToRPC(result)) - if len(p.Nodes) == maxTopicNodes || i == len(nodes)-1 { + if len(p.Nodes) == maxTopicNodes { t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p) p.Nodes = p.Nodes[:0] + sent = true } } + if !sent || len(p.Nodes) > 0 { + t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p) + } } func (t *udp) sendPacket(toid NodeID, toaddr *net.UDPAddr, ptype byte, req interface{}) (hash []byte, err error) { From ccf808353794f835422e02446384bd627f045f1a Mon Sep 17 00:00:00 2001 From: gluk256 Date: Fri, 9 Feb 2018 16:25:03 +0100 Subject: [PATCH 022/110] whisper: Seal function fixed (#16048) --- whisper/whisperv6/envelope.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index 881945e9a8..c7bea2bb90 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -77,15 +77,19 @@ func NewEnvelope(ttl uint32, topic TopicType, msg *sentMessage) *Envelope { // Seal closes the envelope by spending the requested amount of time as a proof // of work on hashing the data. func (e *Envelope) Seal(options *MessageParams) error { - var target, bestBit int if options.PoW == 0 { - // adjust for the duration of Seal() execution only if execution time is predefined unconditionally + // PoW is not required + return nil + } + + var target, bestBit int + if options.PoW < 0 { + // target is not set - the function should run for a period + // of time specified in WorkTime param. Since we can predict + // the execution time, we can also adjust Expiry. e.Expiry += options.WorkTime } else { target = e.powToFirstBit(options.PoW) - if target < 1 { - target = 1 - } } buf := make([]byte, 64) @@ -143,7 +147,11 @@ func (e *Envelope) powToFirstBit(pow float64) int { x *= float64(e.TTL) bits := gmath.Log2(x) bits = gmath.Ceil(bits) - return int(bits) + res := int(bits) + if res < 1 { + res = 1 + } + return res } // Hash returns the SHA3 hash of the envelope, calculating it if not yet done. From 42628ba7eda25830653763ced060f702861d0887 Mon Sep 17 00:00:00 2001 From: gluk256 Date: Fri, 9 Feb 2018 16:25:23 +0100 Subject: [PATCH 023/110] whisper: bloom filter refactoring (#16046) * whisper: bloom filter refactoring * whisper: fixed full node --- whisper/whisperv6/peer.go | 34 ++++++++++++++++++++++------------ whisper/whisperv6/whisper.go | 7 +------ 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 4f9a7c3780..4ef0f3c434 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -36,7 +36,8 @@ type Peer struct { trusted bool powRequirement float64 - bloomFilter []byte // may contain nil in case of full node + bloomFilter []byte + fullNode bool known *set.Set // Messages already known by the peer to avoid wasting bandwidth @@ -53,6 +54,8 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer { powRequirement: 0.0, known: set.New(), quit: make(chan struct{}), + bloomFilter: makeFullNodeBloom(), + fullNode: true, } } @@ -118,11 +121,7 @@ func (peer *Peer) handshake() error { if sz != bloomFilterSize && sz != 0 { return fmt.Errorf("peer [%x] sent bad status message: wrong bloom filter size %d", peer.ID(), sz) } - if isFullNode(bloom) { - peer.bloomFilter = nil - } else { - peer.bloomFilter = bloom - } + peer.setBloomFilter(bloom) } } @@ -226,10 +225,21 @@ func (peer *Peer) notifyAboutBloomFilterChange(bloom []byte) error { } func (peer *Peer) bloomMatch(env *Envelope) bool { - if peer.bloomFilter == nil { - // no filter - full node, accepts all envelops - return true - } - - return bloomFilterMatch(peer.bloomFilter, env.Bloom()) + return peer.fullNode || bloomFilterMatch(peer.bloomFilter, env.Bloom()) +} + +func (peer *Peer) setBloomFilter(bloom []byte) { + peer.bloomFilter = bloom + peer.fullNode = isFullNode(bloom) + if peer.fullNode && peer.bloomFilter == nil { + peer.bloomFilter = makeFullNodeBloom() + } +} + +func makeFullNodeBloom() []byte { + bloom := make([]byte, bloomFilterSize) + for i := 0; i < bloomFilterSize; i++ { + bloom[i] = 0xFF + } + return bloom } diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index d75ad04ac3..600f9cb286 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -710,11 +710,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("failed to decode bloom filter exchange message, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid bloom filter exchange message") } - if isFullNode(bloom) { - p.bloomFilter = nil - } else { - p.bloomFilter = bloom - } + p.setBloomFilter(bloom) case p2pMessageCode: // peer-to-peer message, sent directly to peer bypassing PoW checks, etc. // this message is not supposed to be forwarded to other peers, and @@ -1049,7 +1045,6 @@ func isFullNode(bloom []byte) bool { func bloomFilterMatch(filter, sample []byte) bool { if filter == nil { - // full node, accepts all messages return true } From a00f4a12a95d2bba00b46a58de8232167137fd1e Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Sat, 10 Feb 2018 04:50:14 -0600 Subject: [PATCH 024/110] README: remove --fast and --cache flags and clarify default sync mode (#16043) * Remove --fast flag and clarify default `--fast` is no longer a flag it's `--syncmode "fast"` and that is the default * Remove --cache flag --cache=512 is no longer required as of 1.8 as the default has been increased * README: Minor cache amount fix, mention Rinkeby --- README.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 61e36afec4..527ea048a3 100644 --- a/README.md +++ b/README.md @@ -56,16 +56,14 @@ the user doesn't care about years-old historical data, so we can fast-sync quick state of the network. To do so: ``` -$ geth --fast --cache=512 console +$ geth console ``` This command will: - * Start geth in fast sync mode (`--fast`), causing it to download more data in exchange for avoiding - processing the entire history of the Ethereum network, which is very CPU intensive. - * Bump the memory allowance of the database to 512MB (`--cache=512`), which can help significantly in - sync times especially for HDD users. This flag is optional and you can set it as high or as low as - you'd like, though we'd recommend the 512MB - 2GB range. + * Start geth in fast sync mode (default, can be changed with the `--syncmode` flag), causing it to + download more data in exchange for avoiding processing the entire history of the Ethereum network, + which is very CPU intensive. * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console), (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API) as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs). @@ -80,12 +78,11 @@ entire system. In other words, instead of attaching to the main network, you wan network with your node, which is fully equivalent to the main network, but with play-Ether only. ``` -$ geth --testnet --fast --cache=512 console +$ geth --testnet console ``` -The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they -are equally useful on the testnet too. Please see above for their explanations if you've skipped to -here. +The `console` subcommand have the exact same meaning as above and they are equally useful on the +testnet too. Please see above for their explanations if you've skipped to here. Specifying the `--testnet` flag however will reconfigure your Geth instance a bit: @@ -102,6 +99,14 @@ over between the main network and test network, you should make sure to always u for play-money and real-money. Unless you manually move accounts, Geth will by default correctly separate the two networks and will not make any accounts available between them.* +### Full node on the Rinkeby test network + +The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum. + +``` +$ geth --rinkeby console +``` + ### Configuration As an alternative to passing the numerous flags to the `geth` binary, you can also pass a configuration file via: @@ -125,10 +130,10 @@ One of the quickest ways to get Ethereum up and running on your machine is by us ``` docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ -p 8545:8545 -p 30303:30303 \ - ethereum/client-go --fast --cache=512 + ethereum/client-go ``` -This will start geth in fast sync mode with a DB memory allowance of 512MB just as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. +This will start geth in fast-sync mode with a DB memory allowance of 1GB just as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside. From 2f849ade8204a4b417202d90c66e2f5bef4e965d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Sat, 10 Feb 2018 13:33:52 +0100 Subject: [PATCH 025/110] les: fix server panic when discovery disabled (#16055) --- les/server.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/les/server.go b/les/server.go index 85ebbf8988..65b8c357d2 100644 --- a/les/server.go +++ b/les/server.go @@ -111,15 +111,17 @@ func (s *LesServer) Protocols() []p2p.Protocol { // Start starts the LES server func (s *LesServer) Start(srvr *p2p.Server) { s.protocolManager.Start(s.config.LightPeers) - for _, topic := range s.lesTopics { - topic := topic - go func() { - logger := log.New("topic", topic) - logger.Info("Starting topic registration") - defer logger.Info("Terminated topic registration") + if srvr.DiscV5 != nil { + for _, topic := range s.lesTopics { + topic := topic + go func() { + logger := log.New("topic", topic) + logger.Info("Starting topic registration") + defer logger.Info("Terminated topic registration") - srvr.DiscV5.RegisterTopic(topic, s.quitSync) - }() + srvr.DiscV5.RegisterTopic(topic, s.quitSync) + }() + } } s.privateKey = srvr.PrivateKey s.protocolManager.blockLoop() From 5cf75a30c1ceb0ab35cd6b0532520d556996b21c Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Sat, 10 Feb 2018 14:35:32 +0100 Subject: [PATCH 026/110] whisper: get wnode to work with v6 (#16051) The bulk of the issue was to adapt to the new requirement that a v6 filter has to either contain a symmertric key or an asymmetric one. This commits revert one of the fixes that I made to remove a linter warning: unexporting NewSentMessage. This is not really a problem as I have a cleanup in the pipe that will solve this issue. --- cmd/wnode/main.go | 57 ++++++++++++++++++++-------- whisper/mailserver/mailserver.go | 2 +- whisper/mailserver/server_test.go | 2 +- whisper/whisperv6/api.go | 2 +- whisper/whisperv6/benchmarks_test.go | 14 +++---- whisper/whisperv6/envelope_test.go | 2 +- whisper/whisperv6/filter_test.go | 16 ++++---- whisper/whisperv6/message.go | 2 +- whisper/whisperv6/message_test.go | 20 +++++----- whisper/whisperv6/peer_test.go | 4 +- whisper/whisperv6/whisper_test.go | 12 +++--- 11 files changed, 79 insertions(+), 54 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index e69b57d69f..68e6971dae 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -43,7 +43,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/whisper/mailserver" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "golang.org/x/crypto/pbkdf2" ) @@ -61,15 +61,17 @@ var ( // encryption var ( - symKey []byte - pub *ecdsa.PublicKey - asymKey *ecdsa.PrivateKey - nodeid *ecdsa.PrivateKey - topic whisper.TopicType - asymKeyID string - filterID string - symPass string - msPassword string + symKey []byte + pub *ecdsa.PublicKey + asymKey *ecdsa.PrivateKey + nodeid *ecdsa.PrivateKey + topic whisper.TopicType + + asymKeyID string + asymFilterID string + symFilterID string + symPass string + msPassword string ) // cmd arguments @@ -363,13 +365,22 @@ func configureNode() { } } - filter := whisper.Filter{ + symFilter := whisper.Filter{ KeySym: symKey, + Topics: [][]byte{topic[:]}, + AllowP2P: p2pAccept, + } + symFilterID, err = shh.Subscribe(&symFilter) + if err != nil { + utils.Fatalf("Failed to install filter: %s", err) + } + + asymFilter := whisper.Filter{ KeyAsym: asymKey, Topics: [][]byte{topic[:]}, AllowP2P: p2pAccept, } - filterID, err = shh.Subscribe(&filter) + asymFilterID, err = shh.Subscribe(&asymFilter) if err != nil { utils.Fatalf("Failed to install filter: %s", err) } @@ -522,9 +533,14 @@ func sendMsg(payload []byte) common.Hash { } func messageLoop() { - f := shh.GetFilter(filterID) - if f == nil { - utils.Fatalf("filter is not installed") + sf := shh.GetFilter(symFilterID) + if sf == nil { + utils.Fatalf("symmetric filter is not installed") + } + + af := shh.GetFilter(asymFilterID) + if af == nil { + utils.Fatalf("asymmetric filter is not installed") } ticker := time.NewTicker(time.Millisecond * 50) @@ -532,7 +548,16 @@ func messageLoop() { for { select { case <-ticker.C: - messages := f.Retrieve() + messages := sf.Retrieve() + for _, msg := range messages { + if *fileExMode || len(msg.Payload) > 2048 { + writeMessageToFile(*argSaveDir, msg) + } else { + printMessageInfo(msg) + } + } + + messages = af.Retrieve() for _, msg := range messages { if *fileExMode || len(msg.Payload) > 2048 { writeMessageToFile(*argSaveDir, msg) diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index 0ec6ec570c..6555fd5c0b 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 9155ee85a6..c8e0a553a0 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) const powRequirement = 0.00001 diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index 8ae2882e1f..a2c75a41c7 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -292,7 +292,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er } // encrypt and sent message - whisperMsg, err := newSentMessage(params) + whisperMsg, err := NewSentMessage(params) if err != nil { return false, err } diff --git a/whisper/whisperv6/benchmarks_test.go b/whisper/whisperv6/benchmarks_test.go index 52c8f95ea6..0473179da5 100644 --- a/whisper/whisperv6/benchmarks_test.go +++ b/whisper/whisperv6/benchmarks_test.go @@ -39,7 +39,7 @@ func BenchmarkEncryptionSym(b *testing.B) { } for i := 0; i < b.N; i++ { - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Errorf("failed Wrap with seed %d: %s.", seed, err) @@ -64,7 +64,7 @@ func BenchmarkEncryptionAsym(b *testing.B) { params.Dst = &key.PublicKey for i := 0; i < b.N; i++ { - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -79,7 +79,7 @@ func BenchmarkDecryptionSymValid(b *testing.B) { if err != nil { b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -101,7 +101,7 @@ func BenchmarkDecryptionSymInvalid(b *testing.B) { if err != nil { b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -130,7 +130,7 @@ func BenchmarkDecryptionAsymValid(b *testing.B) { f := Filter{KeyAsym: key} params.KeySym = nil params.Dst = &key.PublicKey - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -157,7 +157,7 @@ func BenchmarkDecryptionAsymInvalid(b *testing.B) { } params.KeySym = nil params.Dst = &key.PublicKey - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -199,7 +199,7 @@ func BenchmarkPoW(b *testing.B) { for i := 0; i < b.N; i++ { increment(params.Payload) - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) diff --git a/whisper/whisperv6/envelope_test.go b/whisper/whisperv6/envelope_test.go index 1ee1bec41b..410b250a3f 100644 --- a/whisper/whisperv6/envelope_test.go +++ b/whisper/whisperv6/envelope_test.go @@ -45,7 +45,7 @@ func TestEnvelopeOpenAcceptsOnlyOneKeyTypeInFilter(t *testing.T) { mrand.Read(params.Payload) - msg, err := newSentMessage(¶ms) + msg, err := NewSentMessage(¶ms) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index fc7db76712..e7230ef388 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -199,7 +199,7 @@ func TestInstallIdenticalFilters(t *testing.T) { filter1.Src = ¶ms.Src.PublicKey filter2.Src = ¶ms.Src.PublicKey - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -306,7 +306,7 @@ func TestMatchEnvelope(t *testing.T) { params.Topic[0] = 0xFF // ensure mismatch // mismatch with pseudo-random data - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -327,7 +327,7 @@ func TestMatchEnvelope(t *testing.T) { i := mrand.Int() % 4 fsym.Topics[i] = params.Topic[:] fasym.Topics[i] = params.Topic[:] - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -372,7 +372,7 @@ func TestMatchEnvelope(t *testing.T) { } params.KeySym = nil params.Dst = &key.PublicKey - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -453,7 +453,7 @@ func TestMatchMessageSym(t *testing.T) { params.KeySym = f.KeySym params.Topic = BytesToTopic(f.Topics[index]) - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -546,7 +546,7 @@ func TestMatchMessageAsym(t *testing.T) { keySymOrig := params.KeySym params.KeySym = nil - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -630,7 +630,7 @@ func generateCompatibeEnvelope(t *testing.T, f *Filter) *Envelope { params.KeySym = f.KeySym params.Topic = BytesToTopic(f.Topics[2]) - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -806,7 +806,7 @@ func TestVariableTopics(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/message.go b/whisper/whisperv6/message.go index 7def35f14f..b8318cbe8f 100644 --- a/whisper/whisperv6/message.go +++ b/whisper/whisperv6/message.go @@ -89,7 +89,7 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { } // NewSentMessage creates and initializes a non-signed, non-encrypted Whisper message. -func newSentMessage(params *MessageParams) (*sentMessage, error) { +func NewSentMessage(params *MessageParams) (*sentMessage, error) { const payloadSizeFieldMaxSize = 4 msg := sentMessage{} msg.Raw = make([]byte, 1, diff --git a/whisper/whisperv6/message_test.go b/whisper/whisperv6/message_test.go index 12a269f5db..0a5c1c8533 100644 --- a/whisper/whisperv6/message_test.go +++ b/whisper/whisperv6/message_test.go @@ -73,7 +73,7 @@ func singleMessageTest(t *testing.T, symmetric bool) { text := make([]byte, 0, 512) text = append(text, params.Payload...) - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -131,7 +131,7 @@ func TestMessageWrap(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -149,7 +149,7 @@ func TestMessageWrap(t *testing.T) { } // set PoW target too high, expect error - msg2, err := newSentMessage(params) + msg2, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -172,7 +172,7 @@ func TestMessageSeal(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -234,7 +234,7 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) { text := make([]byte, 0, 512) text = append(text, params.Payload...) - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -289,7 +289,7 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -303,7 +303,7 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -317,7 +317,7 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -335,7 +335,7 @@ func TestRlpEncode(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -379,7 +379,7 @@ func singlePaddingTest(t *testing.T, padSize int) { if n != padSize { t.Fatalf("padding is not copied (seed %d): %s", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index dffa7b3507..188c8f7467 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -344,7 +344,7 @@ func sendMsg(t *testing.T, expected bool, id int) { opt.Payload = opt.Payload[1:] } - msg, err := newSentMessage(&opt) + msg, err := NewSentMessage(&opt) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -368,7 +368,7 @@ func TestPeerBasic(t *testing.T) { } params.PoW = 0.001 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 838cb7b851..99e5f0bbb4 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -471,7 +471,7 @@ func TestExpiry(t *testing.T) { } params.TTL = 1 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -537,7 +537,7 @@ func TestCustomization(t *testing.T) { params.Topic = BytesToTopic(f.Topics[2]) params.PoW = smallPoW params.TTL = 3600 * 24 // one day - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -558,7 +558,7 @@ func TestCustomization(t *testing.T) { } params.TTL++ - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -647,7 +647,7 @@ func TestSymmetricSendCycle(t *testing.T) { params.PoW = filter1.PoW params.WorkTime = 10 params.TTL = 50 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -725,7 +725,7 @@ func TestSymmetricSendWithoutAKey(t *testing.T) { params.PoW = filter.PoW params.WorkTime = 10 params.TTL = 50 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -791,7 +791,7 @@ func TestSymmetricSendKeyMismatch(t *testing.T) { params.PoW = filter.PoW params.WorkTime = 10 params.TTL = 50 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } From 96dad6b6f6f8db88cca7496665016152272881cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sun, 11 Feb 2018 14:43:56 +0200 Subject: [PATCH 027/110] eth/downloader: don't require state for ancestor lookups --- eth/downloader/downloader.go | 11 ++++++----- eth/downloader/downloader_test.go | 11 +++-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 3870f10b91..7ede530a94 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -173,8 +173,8 @@ type LightChain interface { type BlockChain interface { LightChain - // HasBlockAndState verifies block and associated states' presence in the local chain. - HasBlockAndState(common.Hash, uint64) bool + // HasBlock verifies a block's presence in the local chain. + HasBlock(common.Hash, uint64) bool // GetBlockByHash retrieves a block from the local chain. GetBlockByHash(common.Hash) *types.Block @@ -582,7 +582,6 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err // Figure out the valid ancestor range to prevent rewrite attacks floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64() - p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) if d.mode == FullSync { ceil = d.blockchain.CurrentBlock().NumberU64() } else if d.mode == FastSync { @@ -591,6 +590,8 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err if ceil >= MaxForkAncestry { floor = int64(ceil - MaxForkAncestry) } + p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) + // Request the topmost blocks to short circuit binary ancestor lookup head := ceil if head > height { @@ -646,7 +647,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) { + if (d.mode == FullSync && d.blockchain.HasBlock(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) { number, hash = headers[i].Number.Uint64(), headers[i].Hash() // If every header is known, even future ones, the peer straight out lied about its head @@ -711,7 +712,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) { + if (d.mode == FullSync && !d.blockchain.HasBlock(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) { end = check break } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index d94d55f114..cb671a7df4 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -221,14 +221,9 @@ func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool { return dl.GetHeaderByHash(hash) != nil } -// HasBlockAndState checks if a block and associated state is present in the testers canonical chain. -func (dl *downloadTester) HasBlockAndState(hash common.Hash, number uint64) bool { - block := dl.GetBlockByHash(hash) - if block == nil { - return false - } - _, err := dl.stateDb.Get(block.Root().Bytes()) - return err == nil +// HasBlock checks if a block is present in the testers canonical chain. +func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool { + return dl.GetBlockByHash(hash) != nil } // GetHeader retrieves a header from the testers canonical chain. From 7a0019c63b1297cb5c9a6fdfc4cb00fdae9b05aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sun, 11 Feb 2018 14:57:46 +0200 Subject: [PATCH 028/110] les, light: fix CHT trie retrievals (#16039) * les, light: fix CHT trie retrievals * les, light: minor polishes, test remote CHT retrievals * les, light: deterministic nodeset rlp, bloombits test skeleton * les: add an event emission to the les bloombits test * les: drop dead tester code --- les/handler.go | 44 +++++------- les/handler_test.go | 158 ++++++++++++++++++++++++++++++++----------- les/helper_test.go | 26 +++++-- les/odr_requests.go | 26 +++---- les/peer.go | 5 +- les/server.go | 10 ++- light/lightchain.go | 6 +- light/nodeset.go | 27 +++++--- light/odr_util.go | 4 +- light/postprocess.go | 23 ++++--- 10 files changed, 210 insertions(+), 119 deletions(-) diff --git a/les/handler.go b/les/handler.go index 5c93133fb7..864abe605a 100644 --- a/les/handler.go +++ b/les/handler.go @@ -790,10 +790,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } } - proofs := nodes.NodeList() bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendProofsV2(req.ReqID, bv, proofs) + return p.SendProofsV2(req.ReqID, bv, nodes.NodeList()) case ProofsV1Msg: if pm.odr == nil { @@ -856,15 +855,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) { return errResp(ErrRequestRejected, "") } + trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1) + sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - trie, err := statedb.Database().OpenTrie(root) + trie, err := trie.New(root, trieDb) if err != nil { continue } @@ -878,7 +874,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit { break } - } } } @@ -910,20 +905,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { lastIdx uint64 lastType uint root common.Hash - statedb *state.StateDB - trie state.Trie + auxTrie *trie.Trie ) - nodes := light.NewNodeSet() - for _, req := range req.Reqs { - if trie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx { - statedb, trie, lastType, lastIdx = nil, nil, req.HelperTrieType, req.TrieIdx + if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx { + auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx - if root, _ = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) { - if statedb, _ = pm.blockchain.State(); statedb != nil { - trie, _ = statedb.Database().OpenTrie(root) - } + var prefix string + if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { + auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) } } if req.AuxReq == auxRoot { @@ -934,8 +925,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { auxData = append(auxData, data) auxBytes += len(data) } else { - if trie != nil { - trie.Prove(req.Key, req.FromLevel, nodes) + if auxTrie != nil { + auxTrie.Prove(req.Key, req.FromLevel, nodes) } if req.AuxReq != 0 { data := pm.getHelperTrieAuxData(req) @@ -947,10 +938,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } } - proofs := nodes.NodeList() bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: proofs, AuxData: auxData}) + return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) case HeaderProofsMsg: if pm.odr == nil { @@ -1123,7 +1113,7 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common. func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) { switch id { case htCanonical: - sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.ChtFrequency-1) + sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix case htBloomBits: sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) @@ -1134,10 +1124,8 @@ func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, stri // getHelperTrieAuxData returns requested auxiliary data for the given HelperTrie request func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte { - if req.HelperTrieType == htCanonical && req.AuxReq == auxHeader { - if len(req.Key) != 8 { - return nil - } + switch { + case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8: blockNum := binary.BigEndian.Uint64(req.Key) hash := core.GetCanonicalHash(pm.chainDb, blockNum) return core.GetHeaderRLP(pm.chainDb, hash, blockNum) diff --git a/les/handler_test.go b/les/handler_test.go index e5446c031d..9468032f67 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -17,7 +17,7 @@ package les import ( - "bytes" + "encoding/binary" "math/big" "math/rand" "testing" @@ -45,27 +45,8 @@ func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{} return p2p.ExpectMsg(r, msgcode, resp{reqID, bv, data}) } -func testCheckProof(t *testing.T, exp *light.NodeSet, got light.NodeList) { - if exp.KeyCount() > len(got) { - t.Errorf("proof has fewer nodes than expected") - return - } - if exp.KeyCount() < len(got) { - t.Errorf("proof has more nodes than expected") - return - } - for _, node := range got { - n, _ := exp.Get(crypto.Keccak256(node)) - if !bytes.Equal(n, node) { - t.Errorf("proof contents mismatch") - return - } - } -} - // Tests that block headers can be retrieved from a remote chain based on user queries. func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } - func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { @@ -196,7 +177,6 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Tests that block contents can be retrieved from a remote chain based on their hashes. func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } - func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { @@ -274,7 +254,6 @@ func testGetBlockBodies(t *testing.T, protocol int) { // Tests that the contract codes can be retrieved based on account addresses. func TestGetCodeLes1(t *testing.T) { testGetCode(t, 1) } - func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { @@ -309,7 +288,6 @@ func testGetCode(t *testing.T, protocol int) { // Tests that the transaction receipts can be retrieved based on hashes. func TestGetReceiptLes1(t *testing.T) { testGetReceipt(t, 1) } - func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { @@ -338,7 +316,6 @@ func testGetReceipt(t *testing.T, protocol int) { // Tests that trie merkle proofs can be retrieved func TestGetProofsLes1(t *testing.T) { testGetProofs(t, 1) } - func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { @@ -389,27 +366,126 @@ func testGetProofs(t *testing.T, protocol int) { case 2: cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs)) sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs) - msg, err := peer.app.ReadMsg() - if err != nil { - t.Errorf("Message read error: %v", err) + if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { + t.Errorf("proofs mismatch: %v", err) } - var resp struct { - ReqID, BV uint64 - Data light.NodeList + } +} + +// Tests that CHT proofs can be correctly retrieved. +func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) } +func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) } + +func testGetCHTProofs(t *testing.T, protocol int) { + // Figure out the client's CHT frequency + frequency := uint64(light.CHTFrequencyClient) + if protocol == 1 { + frequency = uint64(light.CHTFrequencyServer) + } + // Assemble the test environment + db, _ := ethdb.NewMemDatabase() + pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) + bc := pm.blockchain.(*core.BlockChain) + peer, _ := newTestPeer(t, "peer", protocol, pm, true) + defer peer.close() + + // Wait a while for the CHT indexer to process the new headers + time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling + time.Sleep(250 * time.Millisecond) // CI tester slack + + // Assemble the proofs from the different protocols + header := bc.GetHeaderByNumber(frequency) + rlp, _ := rlp.EncodeToBytes(header) + + key := make([]byte, 8) + binary.BigEndian.PutUint64(key, frequency) + + proofsV1 := []ChtResp{{ + Header: header, + }} + proofsV2 := HelperTrieResps{ + AuxData: [][]byte{rlp}, + } + switch protocol { + case 1: + root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + + var proof light.NodeList + trie.Prove(key, 0, &proof) + proofsV1[0].Proof = proof + + case 2: + root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + trie.Prove(key, 0, &proofsV2.Proofs) + } + // Assemble the requests for the different protocols + requestsV1 := []ChtReq{{ + ChtNum: 1, + BlockNum: frequency, + }} + requestsV2 := []HelperTrieReq{{ + Type: htCanonical, + TrieIdx: 0, + Key: key, + AuxReq: auxHeader, + }} + // Send the proof request and verify the response + switch protocol { + case 1: + cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1)) + sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1) + if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil { + t.Errorf("proofs mismatch: %v", err) } - if err := msg.Decode(&resp); err != nil { - t.Errorf("reply decode error: %v", err) + case 2: + cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2)) + sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2) + if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { + t.Errorf("proofs mismatch: %v", err) } - if msg.Code != ProofsV2Msg { - t.Errorf("Message code mismatch") + } +} + +// Tests that bloombits proofs can be correctly retrieved. +func TestGetBloombitsProofs(t *testing.T) { + // Assemble the test environment + db, _ := ethdb.NewMemDatabase() + pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) + bc := pm.blockchain.(*core.BlockChain) + peer, _ := newTestPeer(t, "peer", 2, pm, true) + defer peer.close() + + // Wait a while for the bloombits indexer to process the new headers + time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling + time.Sleep(250 * time.Millisecond) // CI tester slack + + // Request and verify each bit of the bloom bits proofs + for bit := 0; bit < 2048; bit++ { + // Assemble therequest and proofs for the bloombits + key := make([]byte, 10) + + binary.BigEndian.PutUint16(key[:2], uint16(bit)) + binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency)) + + requests := []HelperTrieReq{{ + Type: htBloomBits, + TrieIdx: 0, + Key: key, + }} + var proofs HelperTrieResps + + root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix))) + trie.Prove(key, 0, &proofs.Proofs) + + // Send the proof request and verify the response + cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) + sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests) + if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { + t.Errorf("bit %d: proofs mismatch: %v", bit, err) } - if resp.ReqID != 42 { - t.Errorf("ReqID mismatch") - } - if resp.BV != testBufLimit { - t.Errorf("BV mismatch") - } - testCheckProof(t, proofsV2, resp.Data) } } diff --git a/les/helper_test.go b/les/helper_test.go index bf08e1e2f7..6d997a1a36 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/les/flowcontrol" @@ -55,6 +56,9 @@ var ( testContractCodeDeployed = testContractCode[16:] testContractDeployed = uint64(2) + testEventEmitterCode = common.Hex2Bytes("60606040523415600e57600080fd5b7f57050ab73f6b9ebdd9f76b8d4997793f48cf956e965ee070551b9ca0bb71584e60405160405180910390a160358060476000396000f3006060604052600080fd00a165627a7a723058203f727efcad8b5811f8cb1fc2620ce5e8c63570d697aef968172de296ea3994140029") + testEventEmitterAddr common.Address + testBufLimit = uint64(100) ) @@ -85,15 +89,19 @@ func testChainGen(i int, block *core.BlockGen) { // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. // acc1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + // acc1Addr creates a test event. nonce := block.TxNonce(acc1Addr) + + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) - nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key) - testContractAddr = crypto.CreateAddress(acc1Addr, nonce) + tx3, _ := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key) + testContractAddr = crypto.CreateAddress(acc1Addr, nonce+1) + tx4, _ := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, acc1Key) + testEventEmitterAddr = crypto.CreateAddress(acc1Addr, nonce+2) block.AddTx(tx1) block.AddTx(tx2) block.AddTx(tx3) + block.AddTx(tx4) case 2: // Block 3 is empty but was mined by account #2. block.SetCoinbase(acc2Addr) @@ -147,6 +155,16 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain, _ = light.NewLightChain(odr, gspec.Config, engine) } else { blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) + + chtIndexer := light.NewChtIndexer(db, false) + chtIndexer.Start(blockchain) + + bbtIndexer := light.NewBloomTrieIndexer(db, false) + + bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks) + bloomIndexer.AddChildIndexer(bbtIndexer) + bloomIndexer.Start(blockchain) + gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) diff --git a/les/odr_requests.go b/les/odr_requests.go index 937a4f1d9d..34d759dd2a 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -321,7 +321,7 @@ const ( ) type HelperTrieReq struct { - HelperTrieType uint + Type uint TrieIdx uint64 Key []byte FromLevel, AuxReq uint @@ -365,7 +365,7 @@ func (r *ChtRequest) CanSend(peer *peer) bool { peer.lock.RLock() defer peer.lock.RUnlock() - return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.ChtFrequency + return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.CHTFrequencyClient } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -374,10 +374,10 @@ func (r *ChtRequest) Request(reqID uint64, peer *peer) error { var encNum [8]byte binary.BigEndian.PutUint64(encNum[:], r.BlockNum) req := HelperTrieReq{ - HelperTrieType: htCanonical, - TrieIdx: r.ChtNum, - Key: encNum[:], - AuxReq: auxHeader, + Type: htCanonical, + TrieIdx: r.ChtNum, + Key: encNum[:], + AuxReq: auxHeader, } return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req}) } @@ -493,14 +493,14 @@ func (r *BloomRequest) Request(reqID uint64, peer *peer) error { reqs := make([]HelperTrieReq, len(r.SectionIdxList)) var encNumber [10]byte - binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx)) + binary.BigEndian.PutUint16(encNumber[:2], uint16(r.BitIdx)) for i, sectionIdx := range r.SectionIdxList { - binary.BigEndian.PutUint64(encNumber[2:10], sectionIdx) + binary.BigEndian.PutUint64(encNumber[2:], sectionIdx) reqs[i] = HelperTrieReq{ - HelperTrieType: htBloomBits, - TrieIdx: r.BloomTrieNum, - Key: common.CopyBytes(encNumber[:]), + Type: htBloomBits, + TrieIdx: r.BloomTrieNum, + Key: common.CopyBytes(encNumber[:]), } } return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), reqs) @@ -525,10 +525,10 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { // Verify the proofs var encNumber [10]byte - binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx)) + binary.BigEndian.PutUint16(encNumber[:2], uint16(r.BitIdx)) for i, idx := range r.SectionIdxList { - binary.BigEndian.PutUint64(encNumber[2:10], idx) + binary.BigEndian.PutUint64(encNumber[2:], idx) value, err, _ := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads) if err != nil { return err diff --git a/les/peer.go b/les/peer.go index b72c80d35a..caf5680778 100644 --- a/les/peer.go +++ b/les/peer.go @@ -281,7 +281,6 @@ func (p *peer) RequestProofs(reqID, cost uint64, reqs []ProofReq) error { default: panic(nil) } - } // RequestHelperTrieProofs fetches a batch of HelperTrie merkle proofs from a remote node. @@ -291,12 +290,12 @@ func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, reqs []HelperTrieReq) case lpv1: reqsV1 := make([]ChtReq, len(reqs)) for i, req := range reqs { - if req.HelperTrieType != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { + if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { return fmt.Errorf("Request invalid in LES/1 mode") } blockNum := binary.BigEndian.Uint64(req.Key) // convert HelperTrie request to old CHT request - reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.ChtFrequency / light.ChtV1Frequency), BlockNum: blockNum, FromLevel: req.FromLevel} + reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.CHTFrequencyClient / light.CHTFrequencyServer), BlockNum: blockNum, FromLevel: req.FromLevel} } return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqsV1) case lpv2: diff --git a/les/server.go b/les/server.go index 65b8c357d2..28b87008a0 100644 --- a/les/server.go +++ b/les/server.go @@ -20,7 +20,6 @@ package les import ( "crypto/ecdsa" "encoding/binary" - "fmt" "math" "sync" @@ -73,23 +72,22 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { logger := log.New() chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility - chtV2SectionCount := chtV1SectionCount / (light.ChtFrequency / light.ChtV1Frequency) + chtV2SectionCount := chtV1SectionCount / (light.CHTFrequencyClient / light.CHTFrequencyServer) if chtV2SectionCount != 0 { // convert to LES/2 section chtLastSection := chtV2SectionCount - 1 // convert last LES/2 section index back to LES/1 index for chtIndexer.SectionHead - chtLastSectionV1 := (chtLastSection+1)*(light.ChtFrequency/light.ChtV1Frequency) - 1 + chtLastSectionV1 := (chtLastSection+1)*(light.CHTFrequencyClient/light.CHTFrequencyServer) - 1 chtSectionHead := srv.chtIndexer.SectionHead(chtLastSectionV1) chtRoot := light.GetChtV2Root(pm.chainDb, chtLastSection, chtSectionHead) - logger.Info("CHT", "section", chtLastSection, "sectionHead", fmt.Sprintf("%064x", chtSectionHead), "root", fmt.Sprintf("%064x", chtRoot)) + logger.Info("Loaded CHT", "section", chtLastSection, "head", chtSectionHead, "root", chtRoot) } - bloomTrieSectionCount, _, _ := srv.bloomTrieIndexer.Sections() if bloomTrieSectionCount != 0 { bloomTrieLastSection := bloomTrieSectionCount - 1 bloomTrieSectionHead := srv.bloomTrieIndexer.SectionHead(bloomTrieLastSection) bloomTrieRoot := light.GetBloomTrieRoot(pm.chainDb, bloomTrieLastSection, bloomTrieSectionHead) - logger.Info("BloomTrie", "section", bloomTrieLastSection, "sectionHead", fmt.Sprintf("%064x", bloomTrieSectionHead), "root", fmt.Sprintf("%064x", bloomTrieRoot)) + logger.Info("Loaded bloom trie", "section", bloomTrieLastSection, "head", bloomTrieSectionHead, "root", bloomTrieRoot) } srv.chtIndexer.Start(eth.BlockChain()) diff --git a/light/lightchain.go b/light/lightchain.go index bc88aeb487..181a1c2a62 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -127,7 +127,7 @@ func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) { if self.odr.BloomIndexer() != nil { self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) } - log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*ChtFrequency-1, "hash", cp.sectionHead) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*CHTFrequencyClient-1, "hash", cp.sectionHead) } func (self *LightChain) getProcInterrupt() bool { @@ -453,8 +453,8 @@ func (self *LightChain) SyncCht(ctx context.Context) bool { } headNum := self.CurrentHeader().Number.Uint64() chtCount, _, _ := self.odr.ChtIndexer().Sections() - if headNum+1 < chtCount*ChtFrequency { - num := chtCount*ChtFrequency - 1 + if headNum+1 < chtCount*CHTFrequencyClient { + num := chtCount*CHTFrequencyClient - 1 header, err := GetHeaderByNumber(ctx, self.odr, num) if header != nil && err == nil { self.mu.Lock() diff --git a/light/nodeset.go b/light/nodeset.go index ffdb71bb79..245b5eb766 100644 --- a/light/nodeset.go +++ b/light/nodeset.go @@ -29,7 +29,9 @@ import ( // NodeSet stores a set of trie nodes. It implements trie.Database and can also // act as a cache for another trie.Database. type NodeSet struct { - db map[string][]byte + nodes map[string][]byte + order []string + dataSize int lock sync.RWMutex } @@ -37,7 +39,7 @@ type NodeSet struct { // NewNodeSet creates an empty node set func NewNodeSet() *NodeSet { return &NodeSet{ - db: make(map[string][]byte), + nodes: make(map[string][]byte), } } @@ -46,10 +48,15 @@ func (db *NodeSet) Put(key []byte, value []byte) error { db.lock.Lock() defer db.lock.Unlock() - if _, ok := db.db[string(key)]; !ok { - db.db[string(key)] = common.CopyBytes(value) - db.dataSize += len(value) + if _, ok := db.nodes[string(key)]; ok { + return nil } + keystr := string(key) + + db.nodes[keystr] = common.CopyBytes(value) + db.order = append(db.order, keystr) + db.dataSize += len(value) + return nil } @@ -58,7 +65,7 @@ func (db *NodeSet) Get(key []byte) ([]byte, error) { db.lock.RLock() defer db.lock.RUnlock() - if entry, ok := db.db[string(key)]; ok { + if entry, ok := db.nodes[string(key)]; ok { return entry, nil } return nil, errors.New("not found") @@ -75,7 +82,7 @@ func (db *NodeSet) KeyCount() int { db.lock.RLock() defer db.lock.RUnlock() - return len(db.db) + return len(db.nodes) } // DataSize returns the aggregated data size of nodes in the set @@ -92,8 +99,8 @@ func (db *NodeSet) NodeList() NodeList { defer db.lock.RUnlock() var values NodeList - for _, value := range db.db { - values = append(values, value) + for _, key := range db.order { + values = append(values, db.nodes[key]) } return values } @@ -103,7 +110,7 @@ func (db *NodeSet) Store(target ethdb.Putter) { db.lock.RLock() defer db.lock.RUnlock() - for key, value := range db.db { + for key, value := range db.nodes { target.Put([]byte(key), value) } } diff --git a/light/odr_util.go b/light/odr_util.go index 33a8e80ce5..8f92d64426 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -52,13 +52,13 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { chtCount-- if chtCount > 0 { - sectionHeadNum = chtCount*ChtFrequency - 1 + sectionHeadNum = chtCount*CHTFrequencyClient - 1 sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1) canonicalHash = core.GetCanonicalHash(db, sectionHeadNum) } } } - if number >= chtCount*ChtFrequency { + if number >= chtCount*CHTFrequencyClient { return nil, ErrNoTrustedCht } r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number} diff --git a/light/postprocess.go b/light/postprocess.go index 160d07b175..b6756de510 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -19,7 +19,6 @@ package light import ( "encoding/binary" "errors" - "fmt" "math/big" "time" @@ -35,8 +34,14 @@ import ( ) const ( - ChtFrequency = 32768 - ChtV1Frequency = 4096 // as long as we want to retain LES/1 compatibility, servers generate CHTs with the old, higher frequency + // CHTFrequencyClient is the block frequency for creating CHTs on the client side. + CHTFrequencyClient = 32768 + + // CHTFrequencyServer is the block frequency for creating CHTs on the server side. + // Eventually this can be merged back with the client version, but that requires a + // full database upgrade, so that should be left for a suitable moment. + CHTFrequencyServer = 4096 + HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated ) @@ -100,7 +105,7 @@ func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) c // GetChtV2Root reads the CHT root assoctiated to the given section from the database // Note that sectionIdx is specified according to LES/2 CHT section size func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { - return GetChtRoot(db, (sectionIdx+1)*(ChtFrequency/ChtV1Frequency)-1, sectionHead) + return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) } // StoreChtRoot writes the CHT root assoctiated to the given section into the database @@ -124,10 +129,10 @@ type ChtIndexerBackend struct { func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { var sectionSize, confirmReq uint64 if clientMode { - sectionSize = ChtFrequency + sectionSize = CHTFrequencyClient confirmReq = HelperTrieConfirmations } else { - sectionSize = ChtV1Frequency + sectionSize = CHTFrequencyServer confirmReq = HelperTrieProcessConfirmations } idb := ethdb.NewTable(db, "chtIndex-") @@ -174,8 +179,8 @@ func (c *ChtIndexerBackend) Commit() error { } c.triedb.Commit(root, false) - if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 { - log.Info("Storing CHT", "idx", c.section*c.sectionSize/ChtFrequency, "sectionHead", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) + if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { + log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil @@ -294,7 +299,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.triedb.Commit(root, false) sectionHead := b.sectionHeads[b.bloomTrieRatio-1] - log.Info("Storing BloomTrie", "section", b.section, "sectionHead", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression ratio", float64(compSize)/float64(decompSize)) + log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) return nil From 57fd2da0fe2ec4d48db7fa936d9e5b6ddb547bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sun, 11 Feb 2018 17:25:00 +0200 Subject: [PATCH 029/110] eth: only disable fast sync after success --- eth/sync.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/eth/sync.go b/eth/sync.go index a8ae646170..2da1464bc5 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -189,18 +189,13 @@ func (pm *ProtocolManager) synchronise(peer *peer) { mode = downloader.FastSync } // Run the sync cycle, and disable fast sync if we've went past the pivot block - err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode) - - if atomic.LoadUint32(&pm.fastSync) == 1 { - // Disable fast sync if we indeed have something in our chain - if pm.blockchain.CurrentBlock().NumberU64() > 0 { - log.Info("Fast sync complete, auto disabling") - atomic.StoreUint32(&pm.fastSync, 0) - } - } - if err != nil { + if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { return } + if atomic.LoadUint32(&pm.fastSync) == 1 { + log.Info("Fast sync complete, auto disabling") + atomic.StoreUint32(&pm.fastSync, 0) + } atomic.StoreUint32(&pm.acceptTxs, 1) // Mark initial sync done if head := pm.blockchain.CurrentBlock(); head.NumberU64() > 0 { // We've completed a sync cycle, notify all peers of new state. This path is From 969474f60ab0c6800abb49cc34eb1f9ce4015bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sun, 11 Feb 2018 19:07:11 +0200 Subject: [PATCH 030/110] build: deprecate zesty, add bionic PPA --- build/ci.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/ci.go b/build/ci.go index 1f98bb8438..544483c42c 100644 --- a/build/ci.go +++ b/build/ci.go @@ -121,7 +121,8 @@ var ( // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: wily is unsupported because it was officially deprecated on lanchpad. // Note: yakkety is unsupported because it was officially deprecated on lanchpad. - debDistros = []string{"trusty", "xenial", "zesty", "artful"} + // Note: zesty is unsupported because it was officially deprecated on lanchpad. + debDistros = []string{"trusty", "xenial", "artful", "bionic"} ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From 52ad848b2ef5f4c03156f18898e57f303636f52b Mon Sep 17 00:00:00 2001 From: ferhat elmas Date: Mon, 12 Feb 2018 10:18:35 +0100 Subject: [PATCH 031/110] internal/build: fix usage of strings.TrimLeft (#16066) --- internal/build/env.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/build/env.go b/internal/build/env.go index c9848bf82c..b553e0ed80 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -94,7 +94,7 @@ func LocalEnv() Environment { } if env.Branch == "" { if head != "HEAD" { - env.Branch = strings.TrimLeft(head, "refs/heads/") + env.Branch = strings.TrimPrefix(head, "refs/heads/") } } if info, err := os.Stat(".git/objects"); err == nil && info.IsDir() && env.Tag == "" { From 69c1f2c2a760fcb1c4229bff06719c73aa8cce31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 12 Feb 2018 11:54:14 +0200 Subject: [PATCH 032/110] core: force import known but rolled back blocks --- core/blockchain.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 8d141fddb5..e498dedefc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1070,8 +1070,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } switch { case err == ErrKnownBlock: - stats.ignored++ - continue + // Block and state both already known. However if the current block is below + // this number we did a rollback and we should reimport it nonetheless. + if bc.CurrentBlock().NumberU64() >= block.NumberU64() { + stats.ignored++ + continue + } case err == consensus.ErrFutureBlock: // Allow up to MaxFuture second in the future blocks. If this limit is exceeded From 9123eceb0f78f69e88d909a56ad7fadb75570198 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 12 Feb 2018 13:36:09 +0100 Subject: [PATCH 033/110] p2p, p2p/discover: misc connectivity improvements (#16069) * p2p: add DialRatio for configuration of inbound vs. dialed connections * p2p: add connection flags to PeerInfo * p2p/netutil: add SameNet, DistinctNetSet * p2p/discover: improve revalidation and seeding This changes node revalidation to be periodic instead of on-demand. This should prevent issues where dead nodes get stuck in closer buckets because no other node will ever come along to replace them. Every 5 seconds (on average), the last node in a random bucket is checked and moved to the front of the bucket if it is still responding. If revalidation fails, the last node is replaced by an entry of the 'replacement list' containing recently-seen nodes. Most close buckets are removed because it's very unlikely we'll ever encounter a node that would fall into any of those buckets. Table seeding is also improved: we now require a few minutes of table membership before considering a node as a potential seed node. This should make it less likely to store short-lived nodes as potential seeds. * p2p/discover: fix nits in UDP transport We would skip sending neighbors replies if there were fewer than maxNeighbors results and CheckRelayIP returned an error for the last one. While here, also resolve a TODO about pong reply tokens. --- cmd/bootnode/main.go | 7 +- p2p/discover/node.go | 6 +- p2p/discover/table.go | 484 +++++++++++++++++++++++++------------ p2p/discover/table_test.go | 181 +++++++++----- p2p/discover/udp.go | 86 ++++--- p2p/discover/udp_test.go | 21 +- p2p/netutil/net.go | 131 ++++++++++ p2p/netutil/net_test.go | 89 +++++++ p2p/peer.go | 6 + p2p/server.go | 77 +++--- 10 files changed, 806 insertions(+), 282 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index ecfc6fc24e..2e93cc04d2 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -122,7 +122,12 @@ func main() { utils.Fatalf("%v", err) } } else { - if _, err := discover.ListenUDP(nodeKey, conn, realaddr, nil, "", restrictList); err != nil { + cfg := discover.Config{ + PrivateKey: nodeKey, + AnnounceAddr: realaddr, + NetRestrict: restrictList, + } + if _, err := discover.ListenUDP(conn, cfg); err != nil { utils.Fatalf("%v", err) } } diff --git a/p2p/discover/node.go b/p2p/discover/node.go index fc928a91af..3b0c84115c 100644 --- a/p2p/discover/node.go +++ b/p2p/discover/node.go @@ -29,6 +29,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -51,9 +52,8 @@ type Node struct { // with ID. sha common.Hash - // whether this node is currently being pinged in order to replace - // it in a bucket - contested bool + // Time when the node was added to the table. + addedAt time.Time } // NewNode creates a new node. It is mostly meant to be used for diff --git a/p2p/discover/table.go b/p2p/discover/table.go index ec4eb94ad5..84c54dac12 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -23,10 +23,11 @@ package discover import ( - "crypto/rand" + crand "crypto/rand" "encoding/binary" "errors" "fmt" + mrand "math/rand" "net" "sort" "sync" @@ -35,29 +36,45 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/netutil" ) const ( - alpha = 3 // Kademlia concurrency factor - bucketSize = 16 // Kademlia bucket size - hashBits = len(common.Hash{}) * 8 - nBuckets = hashBits + 1 // Number of buckets + alpha = 3 // Kademlia concurrency factor + bucketSize = 16 // Kademlia bucket size + maxReplacements = 10 // Size of per-bucket replacement list - maxBondingPingPongs = 16 - maxFindnodeFailures = 5 + // We keep buckets for the upper 1/15 of distances because + // it's very unlikely we'll ever encounter a node that's closer. + hashBits = len(common.Hash{}) * 8 + nBuckets = hashBits / 15 // Number of buckets + bucketMinDistance = hashBits - nBuckets // Log distance of closest bucket - autoRefreshInterval = 1 * time.Hour - seedCount = 30 - seedMaxAge = 5 * 24 * time.Hour + // IP address limits. + bucketIPLimit, bucketSubnet = 2, 24 // at most 2 addresses from the same /24 + tableIPLimit, tableSubnet = 10, 24 + + maxBondingPingPongs = 16 // Limit on the number of concurrent ping/pong interactions + maxFindnodeFailures = 5 // Nodes exceeding this limit are dropped + + refreshInterval = 30 * time.Minute + revalidateInterval = 10 * time.Second + copyNodesInterval = 30 * time.Second + seedMinTableTime = 5 * time.Minute + seedCount = 30 + seedMaxAge = 5 * 24 * time.Hour ) type Table struct { - mutex sync.Mutex // protects buckets, their content, and nursery + mutex sync.Mutex // protects buckets, bucket content, nursery, rand buckets [nBuckets]*bucket // index of known nodes by distance nursery []*Node // bootstrap nodes - db *nodeDB // database of known nodes + rand *mrand.Rand // source of randomness, periodically reseeded + ips netutil.DistinctNetSet + db *nodeDB // database of known nodes refreshReq chan chan struct{} + initDone chan struct{} closeReq chan struct{} closed chan struct{} @@ -89,9 +106,13 @@ type transport interface { // bucket contains nodes, ordered by their last activity. the entry // that was most recently active is the first element in entries. -type bucket struct{ entries []*Node } +type bucket struct { + entries []*Node // live entries, sorted by time of last contact + replacements []*Node // recently seen nodes to be used if revalidation fails + ips netutil.DistinctNetSet +} -func newTable(t transport, ourID NodeID, ourAddr *net.UDPAddr, nodeDBPath string) (*Table, error) { +func newTable(t transport, ourID NodeID, ourAddr *net.UDPAddr, nodeDBPath string, bootnodes []*Node) (*Table, error) { // If no node database was given, use an in-memory one db, err := newNodeDB(nodeDBPath, Version, ourID) if err != nil { @@ -104,19 +125,42 @@ func newTable(t transport, ourID NodeID, ourAddr *net.UDPAddr, nodeDBPath string bonding: make(map[NodeID]*bondproc), bondslots: make(chan struct{}, maxBondingPingPongs), refreshReq: make(chan chan struct{}), + initDone: make(chan struct{}), closeReq: make(chan struct{}), closed: make(chan struct{}), + rand: mrand.New(mrand.NewSource(0)), + ips: netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit}, + } + if err := tab.setFallbackNodes(bootnodes); err != nil { + return nil, err } for i := 0; i < cap(tab.bondslots); i++ { tab.bondslots <- struct{}{} } for i := range tab.buckets { - tab.buckets[i] = new(bucket) + tab.buckets[i] = &bucket{ + ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit}, + } } - go tab.refreshLoop() + tab.seedRand() + tab.loadSeedNodes(false) + // Start the background expiration goroutine after loading seeds so that the search for + // seed nodes also considers older nodes that would otherwise be removed by the + // expiration. + tab.db.ensureExpirer() + go tab.loop() return tab, nil } +func (tab *Table) seedRand() { + var b [8]byte + crand.Read(b[:]) + + tab.mutex.Lock() + tab.rand.Seed(int64(binary.BigEndian.Uint64(b[:]))) + tab.mutex.Unlock() +} + // Self returns the local node. // The returned node should not be modified by the caller. func (tab *Table) Self() *Node { @@ -127,9 +171,12 @@ func (tab *Table) Self() *Node { // table. It will not write the same node more than once. The nodes in // the slice are copies and can be modified by the caller. func (tab *Table) ReadRandomNodes(buf []*Node) (n int) { + if !tab.isInitDone() { + return 0 + } tab.mutex.Lock() defer tab.mutex.Unlock() - // TODO: tree-based buckets would help here + // Find all non-empty buckets and get a fresh slice of their entries. var buckets [][]*Node for _, b := range tab.buckets { @@ -141,8 +188,8 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) { return 0 } // Shuffle the buckets. - for i := uint32(len(buckets)) - 1; i > 0; i-- { - j := randUint(i) + for i := len(buckets) - 1; i > 0; i-- { + j := tab.rand.Intn(len(buckets)) buckets[i], buckets[j] = buckets[j], buckets[i] } // Move head of each bucket into buf, removing buckets that become empty. @@ -161,15 +208,6 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) { return i + 1 } -func randUint(max uint32) uint32 { - if max == 0 { - return 0 - } - var b [4]byte - rand.Read(b[:]) - return binary.BigEndian.Uint32(b[:]) % max -} - // Close terminates the network listener and flushes the node database. func (tab *Table) Close() { select { @@ -180,16 +218,15 @@ func (tab *Table) Close() { } } -// SetFallbackNodes sets the initial points of contact. These nodes +// setFallbackNodes sets the initial points of contact. These nodes // are used to connect to the network if the table is empty and there // are no known nodes in the database. -func (tab *Table) SetFallbackNodes(nodes []*Node) error { +func (tab *Table) setFallbackNodes(nodes []*Node) error { for _, n := range nodes { if err := n.validateComplete(); err != nil { return fmt.Errorf("bad bootstrap/fallback node %q (%v)", n, err) } } - tab.mutex.Lock() tab.nursery = make([]*Node, 0, len(nodes)) for _, n := range nodes { cpy := *n @@ -198,11 +235,19 @@ func (tab *Table) SetFallbackNodes(nodes []*Node) error { cpy.sha = crypto.Keccak256Hash(n.ID[:]) tab.nursery = append(tab.nursery, &cpy) } - tab.mutex.Unlock() - tab.refresh() return nil } +// isInitDone returns whether the table's initial seeding procedure has completed. +func (tab *Table) isInitDone() bool { + select { + case <-tab.initDone: + return true + default: + return false + } +} + // Resolve searches for a specific node with the given ID. // It returns nil if the node could not be found. func (tab *Table) Resolve(targetID NodeID) *Node { @@ -314,33 +359,49 @@ func (tab *Table) refresh() <-chan struct{} { return done } -// refreshLoop schedules doRefresh runs and coordinates shutdown. -func (tab *Table) refreshLoop() { +// loop schedules refresh, revalidate runs and coordinates shutdown. +func (tab *Table) loop() { var ( - timer = time.NewTicker(autoRefreshInterval) - waiting []chan struct{} // accumulates waiting callers while doRefresh runs - done chan struct{} // where doRefresh reports completion + revalidate = time.NewTimer(tab.nextRevalidateTime()) + refresh = time.NewTicker(refreshInterval) + copyNodes = time.NewTicker(copyNodesInterval) + revalidateDone = make(chan struct{}) + refreshDone = make(chan struct{}) // where doRefresh reports completion + waiting = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs ) + defer refresh.Stop() + defer revalidate.Stop() + defer copyNodes.Stop() + + // Start initial refresh. + go tab.doRefresh(refreshDone) + loop: for { select { - case <-timer.C: - if done == nil { - done = make(chan struct{}) - go tab.doRefresh(done) + case <-refresh.C: + tab.seedRand() + if refreshDone == nil { + refreshDone = make(chan struct{}) + go tab.doRefresh(refreshDone) } case req := <-tab.refreshReq: waiting = append(waiting, req) - if done == nil { - done = make(chan struct{}) - go tab.doRefresh(done) + if refreshDone == nil { + refreshDone = make(chan struct{}) + go tab.doRefresh(refreshDone) } - case <-done: + case <-refreshDone: for _, ch := range waiting { close(ch) } - waiting = nil - done = nil + waiting, refreshDone = nil, nil + case <-revalidate.C: + go tab.doRevalidate(revalidateDone) + case <-revalidateDone: + revalidate.Reset(tab.nextRevalidateTime()) + case <-copyNodes.C: + go tab.copyBondedNodes() case <-tab.closeReq: break loop } @@ -349,8 +410,8 @@ loop: if tab.net != nil { tab.net.close() } - if done != nil { - <-done + if refreshDone != nil { + <-refreshDone } for _, ch := range waiting { close(ch) @@ -365,38 +426,109 @@ loop: func (tab *Table) doRefresh(done chan struct{}) { defer close(done) + // Load nodes from the database and insert + // them. This should yield a few previously seen nodes that are + // (hopefully) still alive. + tab.loadSeedNodes(true) + + // Run self lookup to discover new neighbor nodes. + tab.lookup(tab.self.ID, false) + // The Kademlia paper specifies that the bucket refresh should // perform a lookup in the least recently used bucket. We cannot // adhere to this because the findnode target is a 512bit value // (not hash-sized) and it is not easily possible to generate a // sha3 preimage that falls into a chosen bucket. - // We perform a lookup with a random target instead. - var target NodeID - rand.Read(target[:]) - result := tab.lookup(target, false) - if len(result) > 0 { + // We perform a few lookups with a random target instead. + for i := 0; i < 3; i++ { + var target NodeID + crand.Read(target[:]) + tab.lookup(target, false) + } +} + +func (tab *Table) loadSeedNodes(bond bool) { + seeds := tab.db.querySeeds(seedCount, seedMaxAge) + seeds = append(seeds, tab.nursery...) + if bond { + seeds = tab.bondall(seeds) + } + for i := range seeds { + seed := seeds[i] + age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.lastPong(seed.ID)) }} + log.Debug("Found seed node in database", "id", seed.ID, "addr", seed.addr(), "age", age) + tab.add(seed) + } +} + +// doRevalidate checks that the last node in a random bucket is still live +// and replaces or deletes the node if it isn't. +func (tab *Table) doRevalidate(done chan<- struct{}) { + defer func() { done <- struct{}{} }() + + last, bi := tab.nodeToRevalidate() + if last == nil { + // No non-empty bucket found. return } - // The table is empty. Load nodes from the database and insert - // them. This should yield a few previously seen nodes that are - // (hopefully) still alive. - seeds := tab.db.querySeeds(seedCount, seedMaxAge) - seeds = tab.bondall(append(seeds, tab.nursery...)) + // Ping the selected node and wait for a pong. + err := tab.ping(last.ID, last.addr()) - if len(seeds) == 0 { - log.Debug("No discv4 seed nodes found") - } - for _, n := range seeds { - age := log.Lazy{Fn: func() time.Duration { return time.Since(tab.db.lastPong(n.ID)) }} - log.Trace("Found seed node in database", "id", n.ID, "addr", n.addr(), "age", age) - } tab.mutex.Lock() - tab.stuff(seeds) - tab.mutex.Unlock() + defer tab.mutex.Unlock() + b := tab.buckets[bi] + if err == nil { + // The node responded, move it to the front. + log.Debug("Revalidated node", "b", bi, "id", last.ID) + b.bump(last) + return + } + // No reply received, pick a replacement or delete the node if there aren't + // any replacements. + if r := tab.replace(b, last); r != nil { + log.Debug("Replaced dead node", "b", bi, "id", last.ID, "ip", last.IP, "r", r.ID, "rip", r.IP) + } else { + log.Debug("Removed dead node", "b", bi, "id", last.ID, "ip", last.IP) + } +} - // Finally, do a self lookup to fill up the buckets. - tab.lookup(tab.self.ID, false) +// nodeToRevalidate returns the last node in a random, non-empty bucket. +func (tab *Table) nodeToRevalidate() (n *Node, bi int) { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + for _, bi = range tab.rand.Perm(len(tab.buckets)) { + b := tab.buckets[bi] + if len(b.entries) > 0 { + last := b.entries[len(b.entries)-1] + return last, bi + } + } + return nil, 0 +} + +func (tab *Table) nextRevalidateTime() time.Duration { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + return time.Duration(tab.rand.Int63n(int64(revalidateInterval))) +} + +// copyBondedNodes adds nodes from the table to the database if they have been in the table +// longer then minTableTime. +func (tab *Table) copyBondedNodes() { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + now := time.Now() + for _, b := range tab.buckets { + for _, n := range b.entries { + if now.Sub(n.addedAt) >= seedMinTableTime { + tab.db.updateNode(n) + } + } + } } // closest returns the n nodes in the table that are closest to the @@ -459,15 +591,14 @@ func (tab *Table) bond(pinged bool, id NodeID, addr *net.UDPAddr, tcpPort uint16 if id == tab.self.ID { return nil, errors.New("is self") } - // Retrieve a previously known node and any recent findnode failures - node, fails := tab.db.node(id), 0 - if node != nil { - fails = tab.db.findFails(id) + if pinged && !tab.isInitDone() { + return nil, errors.New("still initializing") } - // If the node is unknown (non-bonded) or failed (remotely unknown), bond from scratch - var result error + // Start bonding if we haven't seen this node for a while or if it failed findnode too often. + node, fails := tab.db.node(id), tab.db.findFails(id) age := time.Since(tab.db.lastPong(id)) - if node == nil || fails > 0 || age > nodeDBNodeExpiration { + var result error + if fails > 0 || age > nodeDBNodeExpiration { log.Trace("Starting bonding ping/pong", "id", id, "known", node != nil, "failcount", fails, "age", age) tab.bondmu.Lock() @@ -494,10 +625,10 @@ func (tab *Table) bond(pinged bool, id NodeID, addr *net.UDPAddr, tcpPort uint16 node = w.n } } + // Add the node to the table even if the bonding ping/pong + // fails. It will be relaced quickly if it continues to be + // unresponsive. if node != nil { - // Add the node to the table even if the bonding ping/pong - // fails. It will be relaced quickly if it continues to be - // unresponsive. tab.add(node) tab.db.updateFindFails(id, 0) } @@ -522,7 +653,6 @@ func (tab *Table) pingpong(w *bondproc, pinged bool, id NodeID, addr *net.UDPAdd } // Bonding succeeded, update the node database. w.n = NewNode(id, addr.IP, uint16(addr.Port), tcpPort) - tab.db.updateNode(w.n) close(w.done) } @@ -534,16 +664,18 @@ func (tab *Table) ping(id NodeID, addr *net.UDPAddr) error { return err } tab.db.updateLastPong(id, time.Now()) - - // Start the background expiration goroutine after the first - // successful communication. Subsequent calls have no effect if it - // is already running. We do this here instead of somewhere else - // so that the search for seed nodes also considers older nodes - // that would otherwise be removed by the expiration. - tab.db.ensureExpirer() return nil } +// bucket returns the bucket for the given node ID hash. +func (tab *Table) bucket(sha common.Hash) *bucket { + d := logdist(tab.self.sha, sha) + if d <= bucketMinDistance { + return tab.buckets[0] + } + return tab.buckets[d-bucketMinDistance-1] +} + // add attempts to add the given node its corresponding bucket. If the // bucket has space available, adding the node succeeds immediately. // Otherwise, the node is added if the least recently active node in @@ -551,57 +683,29 @@ func (tab *Table) ping(id NodeID, addr *net.UDPAddr) error { // // The caller must not hold tab.mutex. func (tab *Table) add(new *Node) { - b := tab.buckets[logdist(tab.self.sha, new.sha)] tab.mutex.Lock() defer tab.mutex.Unlock() - if b.bump(new) { - return - } - var oldest *Node - if len(b.entries) == bucketSize { - oldest = b.entries[bucketSize-1] - if oldest.contested { - // The node is already being replaced, don't attempt - // to replace it. - return - } - oldest.contested = true - // Let go of the mutex so other goroutines can access - // the table while we ping the least recently active node. - tab.mutex.Unlock() - err := tab.ping(oldest.ID, oldest.addr()) - tab.mutex.Lock() - oldest.contested = false - if err == nil { - // The node responded, don't replace it. - return - } - } - added := b.replace(new, oldest) - if added && tab.nodeAddedHook != nil { - tab.nodeAddedHook(new) + + b := tab.bucket(new.sha) + if !tab.bumpOrAdd(b, new) { + // Node is not in table. Add it to the replacement list. + tab.addReplacement(b, new) } } // stuff adds nodes the table to the end of their corresponding bucket -// if the bucket is not full. The caller must hold tab.mutex. +// if the bucket is not full. The caller must not hold tab.mutex. func (tab *Table) stuff(nodes []*Node) { -outer: + tab.mutex.Lock() + defer tab.mutex.Unlock() + for _, n := range nodes { if n.ID == tab.self.ID { continue // don't add self } - bucket := tab.buckets[logdist(tab.self.sha, n.sha)] - for i := range bucket.entries { - if bucket.entries[i].ID == n.ID { - continue outer // already in bucket - } - } - if len(bucket.entries) < bucketSize { - bucket.entries = append(bucket.entries, n) - if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) - } + b := tab.bucket(n.sha) + if len(b.entries) < bucketSize { + tab.bumpOrAdd(b, n) } } } @@ -611,36 +715,72 @@ outer: func (tab *Table) delete(node *Node) { tab.mutex.Lock() defer tab.mutex.Unlock() - bucket := tab.buckets[logdist(tab.self.sha, node.sha)] - for i := range bucket.entries { - if bucket.entries[i].ID == node.ID { - bucket.entries = append(bucket.entries[:i], bucket.entries[i+1:]...) - return - } - } + + tab.deleteInBucket(tab.bucket(node.sha), node) } -func (b *bucket) replace(n *Node, last *Node) bool { - // Don't add if b already contains n. - for i := range b.entries { - if b.entries[i].ID == n.ID { - return false - } +func (tab *Table) addIP(b *bucket, ip net.IP) bool { + if netutil.IsLAN(ip) { + return true } - // Replace last if it is still the last entry or just add n if b - // isn't full. If is no longer the last entry, it has either been - // replaced with someone else or became active. - if len(b.entries) == bucketSize && (last == nil || b.entries[bucketSize-1].ID != last.ID) { + if !tab.ips.Add(ip) { + log.Debug("IP exceeds table limit", "ip", ip) return false } - if len(b.entries) < bucketSize { - b.entries = append(b.entries, nil) + if !b.ips.Add(ip) { + log.Debug("IP exceeds bucket limit", "ip", ip) + tab.ips.Remove(ip) + return false } - copy(b.entries[1:], b.entries) - b.entries[0] = n return true } +func (tab *Table) removeIP(b *bucket, ip net.IP) { + if netutil.IsLAN(ip) { + return + } + tab.ips.Remove(ip) + b.ips.Remove(ip) +} + +func (tab *Table) addReplacement(b *bucket, n *Node) { + for _, e := range b.replacements { + if e.ID == n.ID { + return // already in list + } + } + if !tab.addIP(b, n.IP) { + return + } + var removed *Node + b.replacements, removed = pushNode(b.replacements, n, maxReplacements) + if removed != nil { + tab.removeIP(b, removed.IP) + } +} + +// replace removes n from the replacement list and replaces 'last' with it if it is the +// last entry in the bucket. If 'last' isn't the last entry, it has either been replaced +// with someone else or became active. +func (tab *Table) replace(b *bucket, last *Node) *Node { + if len(b.entries) >= 0 && b.entries[len(b.entries)-1].ID != last.ID { + // Entry has moved, don't replace it. + return nil + } + // Still the last entry. + if len(b.replacements) == 0 { + tab.deleteInBucket(b, last) + return nil + } + r := b.replacements[tab.rand.Intn(len(b.replacements))] + b.replacements = deleteNode(b.replacements, r) + b.entries[len(b.entries)-1] = r + tab.removeIP(b, last.IP) + return r +} + +// bump moves the given node to the front of the bucket entry list +// if it is contained in that list. func (b *bucket) bump(n *Node) bool { for i := range b.entries { if b.entries[i].ID == n.ID { @@ -653,6 +793,50 @@ func (b *bucket) bump(n *Node) bool { return false } +// bumpOrAdd moves n to the front of the bucket entry list or adds it if the list isn't +// full. The return value is true if n is in the bucket. +func (tab *Table) bumpOrAdd(b *bucket, n *Node) bool { + if b.bump(n) { + return true + } + if len(b.entries) >= bucketSize || !tab.addIP(b, n.IP) { + return false + } + b.entries, _ = pushNode(b.entries, n, bucketSize) + b.replacements = deleteNode(b.replacements, n) + n.addedAt = time.Now() + if tab.nodeAddedHook != nil { + tab.nodeAddedHook(n) + } + return true +} + +func (tab *Table) deleteInBucket(b *bucket, n *Node) { + b.entries = deleteNode(b.entries, n) + tab.removeIP(b, n.IP) +} + +// pushNode adds n to the front of list, keeping at most max items. +func pushNode(list []*Node, n *Node, max int) ([]*Node, *Node) { + if len(list) < max { + list = append(list, nil) + } + removed := list[len(list)-1] + copy(list[1:], list) + list[0] = n + return list, removed +} + +// deleteNode removes n from list. +func deleteNode(list []*Node, n *Node) []*Node { + for i := range list { + if list[i].ID == n.ID { + return append(list[:i], list[i+1:]...) + } + } + return list +} + // nodesByDistance is a list of nodes, ordered by // distance to target. type nodesByDistance struct { diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 1037cc6099..3ce48d2995 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -20,6 +20,7 @@ import ( "crypto/ecdsa" "fmt" "math/rand" + "sync" "net" "reflect" @@ -32,60 +33,65 @@ import ( ) func TestTable_pingReplace(t *testing.T) { - doit := func(newNodeIsResponding, lastInBucketIsResponding bool) { - transport := newPingRecorder() - tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "") - defer tab.Close() - pingSender := NewNode(MustHexID("a502af0f59b2aab7746995408c79e9ca312d2793cc997e44fc55eda62f0150bbb8c59a6f9269ba3a081518b62699ee807c7c19c20125ddfccca872608af9e370"), net.IP{}, 99, 99) - - // fill up the sender's bucket. - last := fillBucket(tab, 253) - - // this call to bond should replace the last node - // in its bucket if the node is not responding. - transport.responding[last.ID] = lastInBucketIsResponding - transport.responding[pingSender.ID] = newNodeIsResponding - tab.bond(true, pingSender.ID, &net.UDPAddr{}, 0) - - // first ping goes to sender (bonding pingback) - if !transport.pinged[pingSender.ID] { - t.Error("table did not ping back sender") - } - if newNodeIsResponding { - // second ping goes to oldest node in bucket - // to see whether it is still alive. - if !transport.pinged[last.ID] { - t.Error("table did not ping last node in bucket") - } - } - - tab.mutex.Lock() - defer tab.mutex.Unlock() - if l := len(tab.buckets[253].entries); l != bucketSize { - t.Errorf("wrong bucket size after bond: got %d, want %d", l, bucketSize) - } - - if lastInBucketIsResponding || !newNodeIsResponding { - if !contains(tab.buckets[253].entries, last.ID) { - t.Error("last entry was removed") - } - if contains(tab.buckets[253].entries, pingSender.ID) { - t.Error("new entry was added") - } - } else { - if contains(tab.buckets[253].entries, last.ID) { - t.Error("last entry was not removed") - } - if !contains(tab.buckets[253].entries, pingSender.ID) { - t.Error("new entry was not added") - } - } + run := func(newNodeResponding, lastInBucketResponding bool) { + name := fmt.Sprintf("newNodeResponding=%t/lastInBucketResponding=%t", newNodeResponding, lastInBucketResponding) + t.Run(name, func(t *testing.T) { + t.Parallel() + testPingReplace(t, newNodeResponding, lastInBucketResponding) + }) } - doit(true, true) - doit(false, true) - doit(true, false) - doit(false, false) + run(true, true) + run(false, true) + run(true, false) + run(false, false) +} + +func testPingReplace(t *testing.T, newNodeIsResponding, lastInBucketIsResponding bool) { + transport := newPingRecorder() + tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil) + defer tab.Close() + + // Wait for init so bond is accepted. + <-tab.initDone + + // fill up the sender's bucket. + pingSender := NewNode(MustHexID("a502af0f59b2aab7746995408c79e9ca312d2793cc997e44fc55eda62f0150bbb8c59a6f9269ba3a081518b62699ee807c7c19c20125ddfccca872608af9e370"), net.IP{}, 99, 99) + last := fillBucket(tab, pingSender) + + // this call to bond should replace the last node + // in its bucket if the node is not responding. + transport.dead[last.ID] = !lastInBucketIsResponding + transport.dead[pingSender.ID] = !newNodeIsResponding + tab.bond(true, pingSender.ID, &net.UDPAddr{}, 0) + tab.doRevalidate(make(chan struct{}, 1)) + + // first ping goes to sender (bonding pingback) + if !transport.pinged[pingSender.ID] { + t.Error("table did not ping back sender") + } + if !transport.pinged[last.ID] { + // second ping goes to oldest node in bucket + // to see whether it is still alive. + t.Error("table did not ping last node in bucket") + } + + tab.mutex.Lock() + defer tab.mutex.Unlock() + wantSize := bucketSize + if !lastInBucketIsResponding && !newNodeIsResponding { + wantSize-- + } + if l := len(tab.bucket(pingSender.sha).entries); l != wantSize { + t.Errorf("wrong bucket size after bond: got %d, want %d", l, wantSize) + } + if found := contains(tab.bucket(pingSender.sha).entries, last.ID); found != lastInBucketIsResponding { + t.Errorf("last entry found: %t, want: %t", found, lastInBucketIsResponding) + } + wantNewEntry := newNodeIsResponding && !lastInBucketIsResponding + if found := contains(tab.bucket(pingSender.sha).entries, pingSender.ID); found != wantNewEntry { + t.Errorf("new entry found: %t, want: %t", found, wantNewEntry) + } } func TestBucket_bumpNoDuplicates(t *testing.T) { @@ -130,11 +136,45 @@ func TestBucket_bumpNoDuplicates(t *testing.T) { } } +// This checks that the table-wide IP limit is applied correctly. +func TestTable_IPLimit(t *testing.T) { + transport := newPingRecorder() + tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil) + defer tab.Close() + + for i := 0; i < tableIPLimit+1; i++ { + n := nodeAtDistance(tab.self.sha, i) + n.IP = net.IP{172, 0, 1, byte(i)} + tab.add(n) + } + if tab.len() > tableIPLimit { + t.Errorf("too many nodes in table") + } +} + +// This checks that the table-wide IP limit is applied correctly. +func TestTable_BucketIPLimit(t *testing.T) { + transport := newPingRecorder() + tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil) + defer tab.Close() + + d := 3 + for i := 0; i < bucketIPLimit+1; i++ { + n := nodeAtDistance(tab.self.sha, d) + n.IP = net.IP{172, 0, 1, byte(i)} + tab.add(n) + } + if tab.len() > bucketIPLimit { + t.Errorf("too many nodes in table") + } +} + // fillBucket inserts nodes into the given bucket until // it is full. The node's IDs dont correspond to their // hashes. -func fillBucket(tab *Table, ld int) (last *Node) { - b := tab.buckets[ld] +func fillBucket(tab *Table, n *Node) (last *Node) { + ld := logdist(tab.self.sha, n.sha) + b := tab.bucket(n.sha) for len(b.entries) < bucketSize { b.entries = append(b.entries, nodeAtDistance(tab.self.sha, ld)) } @@ -146,30 +186,39 @@ func fillBucket(tab *Table, ld int) (last *Node) { func nodeAtDistance(base common.Hash, ld int) (n *Node) { n = new(Node) n.sha = hashAtDistance(base, ld) - n.IP = net.IP{10, 0, 2, byte(ld)} + n.IP = net.IP{byte(ld), 0, 2, byte(ld)} copy(n.ID[:], n.sha[:]) // ensure the node still has a unique ID return n } -type pingRecorder struct{ responding, pinged map[NodeID]bool } +type pingRecorder struct { + mu sync.Mutex + dead, pinged map[NodeID]bool +} func newPingRecorder() *pingRecorder { - return &pingRecorder{make(map[NodeID]bool), make(map[NodeID]bool)} + return &pingRecorder{ + dead: make(map[NodeID]bool), + pinged: make(map[NodeID]bool), + } } func (t *pingRecorder) findnode(toid NodeID, toaddr *net.UDPAddr, target NodeID) ([]*Node, error) { - panic("findnode called on pingRecorder") + return nil, nil } func (t *pingRecorder) close() {} func (t *pingRecorder) waitping(from NodeID) error { return nil // remote always pings } func (t *pingRecorder) ping(toid NodeID, toaddr *net.UDPAddr) error { + t.mu.Lock() + defer t.mu.Unlock() + t.pinged[toid] = true - if t.responding[toid] { - return nil - } else { + if t.dead[toid] { return errTimeout + } else { + return nil } } @@ -178,7 +227,8 @@ func TestTable_closest(t *testing.T) { test := func(test *closeTest) bool { // for any node table, Target and N - tab, _ := newTable(nil, test.Self, &net.UDPAddr{}, "") + transport := newPingRecorder() + tab, _ := newTable(transport, test.Self, &net.UDPAddr{}, "", nil) defer tab.Close() tab.stuff(test.All) @@ -237,8 +287,11 @@ func TestTable_ReadRandomNodesGetAll(t *testing.T) { }, } test := func(buf []*Node) bool { - tab, _ := newTable(nil, NodeID{}, &net.UDPAddr{}, "") + transport := newPingRecorder() + tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil) defer tab.Close() + <-tab.initDone + for i := 0; i < len(buf); i++ { ld := cfg.Rand.Intn(len(tab.buckets)) tab.stuff([]*Node{nodeAtDistance(tab.self.sha, ld)}) @@ -280,7 +333,7 @@ func (*closeTest) Generate(rand *rand.Rand, size int) reflect.Value { func TestTable_Lookup(t *testing.T) { self := nodeAtDistance(common.Hash{}, 0) - tab, _ := newTable(lookupTestnet, self.ID, &net.UDPAddr{}, "") + tab, _ := newTable(lookupTestnet, self.ID, &net.UDPAddr{}, "", nil) defer tab.Close() // lookup on empty table returns no nodes diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index 60436952d8..e40de2c36f 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -216,9 +216,22 @@ type ReadPacket struct { Addr *net.UDPAddr } +// Config holds Table-related settings. +type Config struct { + // These settings are required and configure the UDP listener: + PrivateKey *ecdsa.PrivateKey + + // These settings are optional: + AnnounceAddr *net.UDPAddr // local address announced in the DHT + NodeDBPath string // if set, the node database is stored at this filesystem location + NetRestrict *netutil.Netlist // network whitelist + Bootnodes []*Node // list of bootstrap nodes + Unhandled chan<- ReadPacket // unhandled packets are sent on this channel +} + // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, error) { - tab, _, err := newUDP(priv, conn, realaddr, unhandled, nodeDBPath, netrestrict) +func ListenUDP(c conn, cfg Config) (*Table, error) { + tab, _, err := newUDP(c, cfg) if err != nil { return nil, err } @@ -226,25 +239,29 @@ func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, unhandl return tab, nil } -func newUDP(priv *ecdsa.PrivateKey, c conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, *udp, error) { +func newUDP(c conn, cfg Config) (*Table, *udp, error) { udp := &udp{ conn: c, - priv: priv, - netrestrict: netrestrict, + priv: cfg.PrivateKey, + netrestrict: cfg.NetRestrict, closing: make(chan struct{}), gotreply: make(chan reply), addpending: make(chan *pending), } + realaddr := c.LocalAddr().(*net.UDPAddr) + if cfg.AnnounceAddr != nil { + realaddr = cfg.AnnounceAddr + } // TODO: separate TCP port udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port)) - tab, err := newTable(udp, PubkeyID(&priv.PublicKey), realaddr, nodeDBPath) + tab, err := newTable(udp, PubkeyID(&cfg.PrivateKey.PublicKey), realaddr, cfg.NodeDBPath, cfg.Bootnodes) if err != nil { return nil, nil, err } udp.Table = tab go udp.loop() - go udp.readLoop(unhandled) + go udp.readLoop(cfg.Unhandled) return udp.Table, udp, nil } @@ -256,14 +273,20 @@ func (t *udp) close() { // ping sends a ping message to the given node and waits for a reply. func (t *udp) ping(toid NodeID, toaddr *net.UDPAddr) error { - // TODO: maybe check for ReplyTo field in callback to measure RTT - errc := t.pending(toid, pongPacket, func(interface{}) bool { return true }) - t.send(toaddr, pingPacket, &ping{ + req := &ping{ Version: Version, From: t.ourEndpoint, To: makeEndpoint(toaddr, 0), // TODO: maybe use known TCP port from DB Expiration: uint64(time.Now().Add(expiration).Unix()), + } + packet, hash, err := encodePacket(t.priv, pingPacket, req) + if err != nil { + return err + } + errc := t.pending(toid, pongPacket, func(p interface{}) bool { + return bytes.Equal(p.(*pong).ReplyTok, hash) }) + t.write(toaddr, req.name(), packet) return <-errc } @@ -447,40 +470,45 @@ func init() { } } -func (t *udp) send(toaddr *net.UDPAddr, ptype byte, req packet) error { - packet, err := encodePacket(t.priv, ptype, req) +func (t *udp) send(toaddr *net.UDPAddr, ptype byte, req packet) ([]byte, error) { + packet, hash, err := encodePacket(t.priv, ptype, req) if err != nil { - return err + return hash, err } - _, err = t.conn.WriteToUDP(packet, toaddr) - log.Trace(">> "+req.name(), "addr", toaddr, "err", err) + return hash, t.write(toaddr, req.name(), packet) +} + +func (t *udp) write(toaddr *net.UDPAddr, what string, packet []byte) error { + _, err := t.conn.WriteToUDP(packet, toaddr) + log.Trace(">> "+what, "addr", toaddr, "err", err) return err } -func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) ([]byte, error) { +func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (packet, hash []byte, err error) { b := new(bytes.Buffer) b.Write(headSpace) b.WriteByte(ptype) if err := rlp.Encode(b, req); err != nil { log.Error("Can't encode discv4 packet", "err", err) - return nil, err + return nil, nil, err } - packet := b.Bytes() + packet = b.Bytes() sig, err := crypto.Sign(crypto.Keccak256(packet[headSize:]), priv) if err != nil { log.Error("Can't sign discv4 packet", "err", err) - return nil, err + return nil, nil, err } copy(packet[macSize:], sig) // add the hash to the front. Note: this doesn't protect the // packet in any way. Our public key will be part of this hash in // The future. - copy(packet, crypto.Keccak256(packet[macSize:])) - return packet, nil + hash = crypto.Keccak256(packet[macSize:]) + copy(packet, hash) + return packet, hash, nil } // readLoop runs in its own goroutine. it handles incoming UDP packets. -func (t *udp) readLoop(unhandled chan ReadPacket) { +func (t *udp) readLoop(unhandled chan<- ReadPacket) { defer t.conn.Close() if unhandled != nil { defer close(unhandled) @@ -601,18 +629,22 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromID NodeID, mac []byte t.mutex.Unlock() p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())} + var sent bool // Send neighbors in chunks with at most maxNeighbors per packet // to stay below the 1280 byte limit. - for i, n := range closest { - if netutil.CheckRelayIP(from.IP, n.IP) != nil { - continue + for _, n := range closest { + if netutil.CheckRelayIP(from.IP, n.IP) == nil { + p.Nodes = append(p.Nodes, nodeToRPC(n)) } - p.Nodes = append(p.Nodes, nodeToRPC(n)) - if len(p.Nodes) == maxNeighbors || i == len(closest)-1 { + if len(p.Nodes) == maxNeighbors { t.send(from, neighborsPacket, &p) p.Nodes = p.Nodes[:0] + sent = true } } + if len(p.Nodes) > 0 || !sent { + t.send(from, neighborsPacket, &p) + } return nil } diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go index b81caf8392..3ffa5c4dd1 100644 --- a/p2p/discover/udp_test.go +++ b/p2p/discover/udp_test.go @@ -70,14 +70,15 @@ func newUDPTest(t *testing.T) *udpTest { remotekey: newkey(), remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, } - realaddr := test.pipe.LocalAddr().(*net.UDPAddr) - test.table, test.udp, _ = newUDP(test.localkey, test.pipe, realaddr, nil, "", nil) + test.table, test.udp, _ = newUDP(test.pipe, Config{PrivateKey: test.localkey}) + // Wait for initial refresh so the table doesn't send unexpected findnode. + <-test.table.initDone return test } // handles a packet as if it had been sent to the transport. func (test *udpTest) packetIn(wantError error, ptype byte, data packet) error { - enc, err := encodePacket(test.remotekey, ptype, data) + enc, _, err := encodePacket(test.remotekey, ptype, data) if err != nil { return test.errorf("packet (%d) encode error: %v", ptype, err) } @@ -90,19 +91,19 @@ func (test *udpTest) packetIn(wantError error, ptype byte, data packet) error { // waits for a packet to be sent by the transport. // validate should have type func(*udpTest, X) error, where X is a packet type. -func (test *udpTest) waitPacketOut(validate interface{}) error { +func (test *udpTest) waitPacketOut(validate interface{}) ([]byte, error) { dgram := test.pipe.waitPacketOut() - p, _, _, err := decodePacket(dgram) + p, _, hash, err := decodePacket(dgram) if err != nil { - return test.errorf("sent packet decode error: %v", err) + return hash, test.errorf("sent packet decode error: %v", err) } fn := reflect.ValueOf(validate) exptype := fn.Type().In(0) if reflect.TypeOf(p) != exptype { - return test.errorf("sent packet type mismatch, got: %v, want: %v", reflect.TypeOf(p), exptype) + return hash, test.errorf("sent packet type mismatch, got: %v, want: %v", reflect.TypeOf(p), exptype) } fn.Call([]reflect.Value{reflect.ValueOf(p)}) - return nil + return hash, nil } func (test *udpTest) errorf(format string, args ...interface{}) error { @@ -351,7 +352,7 @@ func TestUDP_successfulPing(t *testing.T) { }) // remote is unknown, the table pings back. - test.waitPacketOut(func(p *ping) error { + hash, _ := test.waitPacketOut(func(p *ping) error { if !reflect.DeepEqual(p.From, test.udp.ourEndpoint) { t.Errorf("got ping.From %v, want %v", p.From, test.udp.ourEndpoint) } @@ -365,7 +366,7 @@ func TestUDP_successfulPing(t *testing.T) { } return nil }) - test.packetIn(nil, pongPacket, &pong{Expiration: futureExp}) + test.packetIn(nil, pongPacket, &pong{ReplyTok: hash, Expiration: futureExp}) // the node should be added to the table shortly after getting the // pong packet. diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go index f6005afd21..656abb6825 100644 --- a/p2p/netutil/net.go +++ b/p2p/netutil/net.go @@ -18,8 +18,11 @@ package netutil import ( + "bytes" "errors" + "fmt" "net" + "sort" "strings" ) @@ -189,3 +192,131 @@ func CheckRelayIP(sender, addr net.IP) error { } return nil } + +// SameNet reports whether two IP addresses have an equal prefix of the given bit length. +func SameNet(bits uint, ip, other net.IP) bool { + ip4, other4 := ip.To4(), other.To4() + switch { + case (ip4 == nil) != (other4 == nil): + return false + case ip4 != nil: + return sameNet(bits, ip4, other4) + default: + return sameNet(bits, ip.To16(), other.To16()) + } +} + +func sameNet(bits uint, ip, other net.IP) bool { + nb := int(bits / 8) + mask := ^byte(0xFF >> (bits % 8)) + if mask != 0 && nb < len(ip) && ip[nb]&mask != other[nb]&mask { + return false + } + return nb <= len(ip) && bytes.Equal(ip[:nb], other[:nb]) +} + +// DistinctNetSet tracks IPs, ensuring that at most N of them +// fall into the same network range. +type DistinctNetSet struct { + Subnet uint // number of common prefix bits + Limit uint // maximum number of IPs in each subnet + + members map[string]uint + buf net.IP +} + +// Add adds an IP address to the set. It returns false (and doesn't add the IP) if the +// number of existing IPs in the defined range exceeds the limit. +func (s *DistinctNetSet) Add(ip net.IP) bool { + key := s.key(ip) + n := s.members[string(key)] + if n < s.Limit { + s.members[string(key)] = n + 1 + return true + } + return false +} + +// Remove removes an IP from the set. +func (s *DistinctNetSet) Remove(ip net.IP) { + key := s.key(ip) + if n, ok := s.members[string(key)]; ok { + if n == 1 { + delete(s.members, string(key)) + } else { + s.members[string(key)] = n - 1 + } + } +} + +// Contains whether the given IP is contained in the set. +func (s DistinctNetSet) Contains(ip net.IP) bool { + key := s.key(ip) + _, ok := s.members[string(key)] + return ok +} + +// Len returns the number of tracked IPs. +func (s DistinctNetSet) Len() int { + n := uint(0) + for _, i := range s.members { + n += i + } + return int(n) +} + +// key encodes the map key for an address into a temporary buffer. +// +// The first byte of key is '4' or '6' to distinguish IPv4/IPv6 address types. +// The remainder of the key is the IP, truncated to the number of bits. +func (s *DistinctNetSet) key(ip net.IP) net.IP { + // Lazily initialize storage. + if s.members == nil { + s.members = make(map[string]uint) + s.buf = make(net.IP, 17) + } + // Canonicalize ip and bits. + typ := byte('6') + if ip4 := ip.To4(); ip4 != nil { + typ, ip = '4', ip4 + } + bits := s.Subnet + if bits > uint(len(ip)*8) { + bits = uint(len(ip) * 8) + } + // Encode the prefix into s.buf. + nb := int(bits / 8) + mask := ^byte(0xFF >> (bits % 8)) + s.buf[0] = typ + buf := append(s.buf[:1], ip[:nb]...) + if nb < len(ip) && mask != 0 { + buf = append(buf, ip[nb]&mask) + } + return buf +} + +// String implements fmt.Stringer +func (s DistinctNetSet) String() string { + var buf bytes.Buffer + buf.WriteString("{") + keys := make([]string, 0, len(s.members)) + for k := range s.members { + keys = append(keys, k) + } + sort.Strings(keys) + for i, k := range keys { + var ip net.IP + if k[0] == '4' { + ip = make(net.IP, 4) + } else { + ip = make(net.IP, 16) + } + copy(ip, k[1:]) + fmt.Fprintf(&buf, "%v×%d", ip, s.members[k]) + if i != len(keys)-1 { + buf.WriteString(" ") + } + } + buf.WriteString("}") + return buf.String() +} diff --git a/p2p/netutil/net_test.go b/p2p/netutil/net_test.go index 1ee1fcb4d6..3a6aa081f2 100644 --- a/p2p/netutil/net_test.go +++ b/p2p/netutil/net_test.go @@ -17,9 +17,11 @@ package netutil import ( + "fmt" "net" "reflect" "testing" + "testing/quick" "github.com/davecgh/go-spew/spew" ) @@ -171,3 +173,90 @@ func BenchmarkCheckRelayIP(b *testing.B) { CheckRelayIP(sender, addr) } } + +func TestSameNet(t *testing.T) { + tests := []struct { + ip, other string + bits uint + want bool + }{ + {"0.0.0.0", "0.0.0.0", 32, true}, + {"0.0.0.0", "0.0.0.1", 0, true}, + {"0.0.0.0", "0.0.0.1", 31, true}, + {"0.0.0.0", "0.0.0.1", 32, false}, + {"0.33.0.1", "0.34.0.2", 8, true}, + {"0.33.0.1", "0.34.0.2", 13, true}, + {"0.33.0.1", "0.34.0.2", 15, false}, + } + + for _, test := range tests { + if ok := SameNet(test.bits, parseIP(test.ip), parseIP(test.other)); ok != test.want { + t.Errorf("SameNet(%d, %s, %s) == %t, want %t", test.bits, test.ip, test.other, ok, test.want) + } + } +} + +func ExampleSameNet() { + // This returns true because the IPs are in the same /24 network: + fmt.Println(SameNet(24, net.IP{127, 0, 0, 1}, net.IP{127, 0, 0, 3})) + // This call returns false: + fmt.Println(SameNet(24, net.IP{127, 3, 0, 1}, net.IP{127, 5, 0, 3})) + // Output: + // true + // false +} + +func TestDistinctNetSet(t *testing.T) { + ops := []struct { + add, remove string + fails bool + }{ + {add: "127.0.0.1"}, + {add: "127.0.0.2"}, + {add: "127.0.0.3", fails: true}, + {add: "127.32.0.1"}, + {add: "127.32.0.2"}, + {add: "127.32.0.3", fails: true}, + {add: "127.33.0.1", fails: true}, + {add: "127.34.0.1"}, + {add: "127.34.0.2"}, + {add: "127.34.0.3", fails: true}, + // Make room for an address, then add again. + {remove: "127.0.0.1"}, + {add: "127.0.0.3"}, + {add: "127.0.0.3", fails: true}, + } + + set := DistinctNetSet{Subnet: 15, Limit: 2} + for _, op := range ops { + var desc string + if op.add != "" { + desc = fmt.Sprintf("Add(%s)", op.add) + if ok := set.Add(parseIP(op.add)); ok != !op.fails { + t.Errorf("%s == %t, want %t", desc, ok, !op.fails) + } + } else { + desc = fmt.Sprintf("Remove(%s)", op.remove) + set.Remove(parseIP(op.remove)) + } + t.Logf("%s: %v", desc, set) + } +} + +func TestDistinctNetSetAddRemove(t *testing.T) { + cfg := &quick.Config{} + fn := func(ips []net.IP) bool { + s := DistinctNetSet{Limit: 3, Subnet: 2} + for _, ip := range ips { + s.Add(ip) + } + for _, ip := range ips { + s.Remove(ip) + } + return s.Len() == 0 + } + + if err := quick.Check(fn, cfg); err != nil { + t.Fatal(err) + } +} diff --git a/p2p/peer.go b/p2p/peer.go index bad1c8c8b2..477d8c2190 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -419,6 +419,9 @@ type PeerInfo struct { Network struct { LocalAddress string `json:"localAddress"` // Local endpoint of the TCP data connection RemoteAddress string `json:"remoteAddress"` // Remote endpoint of the TCP data connection + Inbound bool `json:"inbound"` + Trusted bool `json:"trusted"` + Static bool `json:"static"` } `json:"network"` Protocols map[string]interface{} `json:"protocols"` // Sub-protocol specific metadata fields } @@ -439,6 +442,9 @@ func (p *Peer) Info() *PeerInfo { } info.Network.LocalAddress = p.LocalAddr().String() info.Network.RemoteAddress = p.RemoteAddr().String() + info.Network.Inbound = p.rw.is(inboundConn) + info.Network.Trusted = p.rw.is(trustedConn) + info.Network.Static = p.rw.is(staticDialedConn) // Gather all the running protocol infos for _, proto := range p.running { diff --git a/p2p/server.go b/p2p/server.go index 2cff94ea5b..edc1d9d219 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -40,11 +40,10 @@ const ( refreshPeersInterval = 30 * time.Second staticPeerCheckInterval = 15 * time.Second - // Maximum number of concurrently handshaking inbound connections. - maxAcceptConns = 50 - - // Maximum number of concurrently dialing outbound connections. - maxActiveDialTasks = 16 + // Connectivity defaults. + maxActiveDialTasks = 16 + defaultMaxPendingPeers = 50 + defaultDialRatio = 3 // Maximum time allowed for reading a complete message. // This is effectively the amount of time a connection can be idle. @@ -70,6 +69,11 @@ type Config struct { // Zero defaults to preset values. MaxPendingPeers int `toml:",omitempty"` + // DialRatio controls the ratio of inbound to dialed connections. + // Example: a DialRatio of 2 allows 1/2 of connections to be dialed. + // Setting DialRatio to zero defaults it to 3. + DialRatio int `toml:",omitempty"` + // NoDiscovery can be used to disable the peer discovery mechanism. // Disabling is useful for protocol debugging (manual topology). NoDiscovery bool @@ -427,7 +431,6 @@ func (srv *Server) Start() (err error) { if err != nil { return err } - realaddr = conn.LocalAddr().(*net.UDPAddr) if srv.NAT != nil { if !realaddr.IP.IsLoopback() { @@ -447,11 +450,16 @@ func (srv *Server) Start() (err error) { // node table if !srv.NoDiscovery { - ntab, err := discover.ListenUDP(srv.PrivateKey, conn, realaddr, unhandled, srv.NodeDatabase, srv.NetRestrict) - if err != nil { - return err + cfg := discover.Config{ + PrivateKey: srv.PrivateKey, + AnnounceAddr: realaddr, + NodeDBPath: srv.NodeDatabase, + NetRestrict: srv.NetRestrict, + Bootnodes: srv.BootstrapNodes, + Unhandled: unhandled, } - if err := ntab.SetFallbackNodes(srv.BootstrapNodes); err != nil { + ntab, err := discover.ListenUDP(conn, cfg) + if err != nil { return err } srv.ntab = ntab @@ -476,10 +484,7 @@ func (srv *Server) Start() (err error) { srv.DiscV5 = ntab } - dynPeers := (srv.MaxPeers + 1) / 2 - if srv.NoDiscovery { - dynPeers = 0 - } + dynPeers := srv.maxDialedConns() dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) // handshake @@ -536,6 +541,7 @@ func (srv *Server) run(dialstate dialer) { defer srv.loopWG.Done() var ( peers = make(map[discover.NodeID]*Peer) + inboundCount = 0 trusted = make(map[discover.NodeID]bool, len(srv.TrustedNodes)) taskdone = make(chan task, maxActiveDialTasks) runningTasks []task @@ -621,14 +627,14 @@ running: } // TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them. select { - case c.cont <- srv.encHandshakeChecks(peers, c): + case c.cont <- srv.encHandshakeChecks(peers, inboundCount, c): case <-srv.quit: break running } case c := <-srv.addpeer: // At this point the connection is past the protocol handshake. // Its capabilities are known and the remote identity is verified. - err := srv.protoHandshakeChecks(peers, c) + err := srv.protoHandshakeChecks(peers, inboundCount, c) if err == nil { // The handshakes are done and it passed all checks. p := newPeer(c, srv.Protocols) @@ -639,8 +645,11 @@ running: } name := truncateName(c.name) srv.log.Debug("Adding p2p peer", "name", name, "addr", c.fd.RemoteAddr(), "peers", len(peers)+1) - peers[c.id] = p go srv.runPeer(p) + peers[c.id] = p + if p.Inbound() { + inboundCount++ + } } // The dialer logic relies on the assumption that // dial tasks complete after the peer has been added or @@ -655,6 +664,9 @@ running: d := common.PrettyDuration(mclock.Now() - pd.created) pd.log.Debug("Removing p2p peer", "duration", d, "peers", len(peers)-1, "req", pd.requested, "err", pd.err) delete(peers, pd.ID()) + if pd.Inbound() { + inboundCount-- + } } } @@ -681,20 +693,22 @@ running: } } -func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, c *conn) error { +func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error { // Drop connections with no matching protocols. if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 { return DiscUselessPeer } // Repeat the encryption handshake checks because the // peer set might have changed between the handshakes. - return srv.encHandshakeChecks(peers, c) + return srv.encHandshakeChecks(peers, inboundCount, c) } -func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, c *conn) error { +func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error { switch { case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers: return DiscTooManyPeers + case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): + return DiscTooManyPeers case peers[c.id] != nil: return DiscAlreadyConnected case c.id == srv.Self().ID: @@ -704,6 +718,21 @@ func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, c *conn) } } +func (srv *Server) maxInboundConns() int { + return srv.MaxPeers - srv.maxDialedConns() +} + +func (srv *Server) maxDialedConns() int { + if srv.NoDiscovery || srv.NoDial { + return 0 + } + r := srv.DialRatio + if r == 0 { + r = defaultDialRatio + } + return srv.MaxPeers / r +} + type tempError interface { Temporary() bool } @@ -714,10 +743,7 @@ func (srv *Server) listenLoop() { defer srv.loopWG.Done() srv.log.Info("RLPx listener up", "self", srv.makeSelf(srv.listener, srv.ntab)) - // This channel acts as a semaphore limiting - // active inbound connections that are lingering pre-handshake. - // If all slots are taken, no further connections are accepted. - tokens := maxAcceptConns + tokens := defaultMaxPendingPeers if srv.MaxPendingPeers > 0 { tokens = srv.MaxPendingPeers } @@ -758,9 +784,6 @@ func (srv *Server) listenLoop() { fd = newMeteredConn(fd, true) srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) - - // Spawn the handler. It will give the slot back when the connection - // has been established. go func() { srv.SetupConn(fd, inboundConn, nil) slots <- struct{}{} From 589b603a9b1e17930d1e83ca64ce7cdc4c3d5c85 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 12 Feb 2018 13:52:07 +0100 Subject: [PATCH 034/110] rpc: dns rebind protection (#15962) * cmd,node,rpc: add allowedHosts to prevent dns rebinding attacks * p2p,node: Fix bug with dumpconfig introduced in r54aeb8e4c0bb9f0e7a6c67258af67df3b266af3d * rpc: add wildcard support for rpcallowedhosts + go fmt * cmd/geth, cmd/utils, node, rpc: ignore direct ip(v4/6) addresses in rpc virtual hostnames check * http, rpc, utils: make vhosts into map, address review concerns * node: change log messages to use geth standard (not sprintf) * rpc: fix spelling --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 ++++++ node/api.go | 12 ++++++++-- node/config.go | 11 ++++++++- node/node.go | 29 ++++++++++++----------- p2p/server.go | 2 +- rpc/http.go | 57 +++++++++++++++++++++++++++++++++++++++++++--- 8 files changed, 98 insertions(+), 22 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index cb8d63bf71..a82e5c89cf 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -114,6 +114,7 @@ var ( utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.RPCCORSDomainFlag, + utils.RPCVirtualHostsFlag, utils.EthStatsURLFlag, utils.MetricsEnabledFlag, utils.FakePoWFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index a2bcaff027..a1558c2330 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -156,6 +156,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.IPCDisabledFlag, utils.IPCPathFlag, utils.RPCCORSDomainFlag, + utils.RPCVirtualHostsFlag, utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 2a2909ff2c..5fd5013f00 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -397,6 +397,11 @@ var ( Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", Value: "", } + RPCVirtualHostsFlag = cli.StringFlag{ + Name: "rpcvhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: "localhost", + } RPCApiFlag = cli.StringFlag{ Name: "rpcapi", Usage: "API's offered over the HTTP-RPC interface", @@ -690,6 +695,8 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(RPCApiFlag.Name) { cfg.HTTPModules = splitAndTrim(ctx.GlobalString(RPCApiFlag.Name)) } + + cfg.HTTPVirtualHosts = splitAndTrim(ctx.GlobalString(RPCVirtualHostsFlag.Name)) } // setWS creates the WebSocket RPC listener interface string from the set diff --git a/node/api.go b/node/api.go index 1b04b70938..4e9b1edc47 100644 --- a/node/api.go +++ b/node/api.go @@ -114,7 +114,7 @@ func (api *PrivateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, } // StartRPC starts the HTTP RPC API server. -func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string) (bool, error) { +func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() @@ -141,6 +141,14 @@ func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis } } + allowedVHosts := api.node.config.HTTPVirtualHosts + if vhosts != nil { + allowedVHosts = nil + for _, vhost := range strings.Split(*host, ",") { + allowedVHosts = append(allowedVHosts, strings.TrimSpace(vhost)) + } + } + modules := api.node.httpWhitelist if apis != nil { modules = nil @@ -149,7 +157,7 @@ func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis } } - if err := api.node.startHTTP(fmt.Sprintf("%s:%d", *host, *port), api.node.rpcAPIs, modules, allowedOrigins); err != nil { + if err := api.node.startHTTP(fmt.Sprintf("%s:%d", *host, *port), api.node.rpcAPIs, modules, allowedOrigins, allowedVHosts); err != nil { return false, err } return true, nil diff --git a/node/config.go b/node/config.go index 7a0c1688ec..dda24583ee 100644 --- a/node/config.go +++ b/node/config.go @@ -105,6 +105,15 @@ type Config struct { // useless for custom HTTP clients. HTTPCors []string `toml:",omitempty"` + // HTTPVirtualHosts is the list of virtual hostnames which are allowed on incoming requests. + // This is by default {'localhost'}. Using this prevents attacks like + // DNS rebinding, which bypasses SOP by simply masquerading as being within the same + // origin. These attacks do not utilize CORS, since they are not cross-domain. + // By explicitly checking the Host-header, the server will not allow requests + // made against the server with a malicious host domain. + // Requests using ip address directly are not affected + HTTPVirtualHosts []string `toml:",omitempty"` + // HTTPModules is a list of API modules to expose via the HTTP RPC interface. // If the module list is empty, all RPC API endpoints designated public will be // exposed. @@ -137,7 +146,7 @@ type Config struct { WSExposeAll bool `toml:",omitempty"` // Logger is a custom logger to use with the p2p.Server. - Logger log.Logger + Logger log.Logger `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/node/node.go b/node/node.go index ff7258033e..37bd2eb3ce 100644 --- a/node/node.go +++ b/node/node.go @@ -263,7 +263,7 @@ func (n *Node) startRPC(services map[reflect.Type]Service) error { n.stopInProc() return err } - if err := n.startHTTP(n.httpEndpoint, apis, n.config.HTTPModules, n.config.HTTPCors); err != nil { + if err := n.startHTTP(n.httpEndpoint, apis, n.config.HTTPModules, n.config.HTTPCors, n.config.HTTPVirtualHosts); err != nil { n.stopIPC() n.stopInProc() return err @@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error { if err := handler.RegisterName(api.Namespace, api.Service); err != nil { return err } - n.log.Debug(fmt.Sprintf("InProc registered %T under '%s'", api.Service, api.Namespace)) + n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace) } n.inprocHandler = handler return nil @@ -313,7 +313,7 @@ func (n *Node) startIPC(apis []rpc.API) error { if err := handler.RegisterName(api.Namespace, api.Service); err != nil { return err } - n.log.Debug(fmt.Sprintf("IPC registered %T under '%s'", api.Service, api.Namespace)) + n.log.Debug("IPC registered", "service", api.Service, "namespace", api.Namespace) } // All APIs registered, start the IPC listener var ( @@ -324,7 +324,7 @@ func (n *Node) startIPC(apis []rpc.API) error { return err } go func() { - n.log.Info(fmt.Sprintf("IPC endpoint opened: %s", n.ipcEndpoint)) + n.log.Info("IPC endpoint opened", "url", fmt.Sprintf("%s", n.ipcEndpoint)) for { conn, err := listener.Accept() @@ -337,7 +337,7 @@ func (n *Node) startIPC(apis []rpc.API) error { return } // Not closed, just some error; report and continue - n.log.Error(fmt.Sprintf("IPC accept failed: %v", err)) + n.log.Error("IPC accept failed", "err", err) continue } go handler.ServeCodec(rpc.NewJSONCodec(conn), rpc.OptionMethodInvocation|rpc.OptionSubscriptions) @@ -356,7 +356,7 @@ func (n *Node) stopIPC() { n.ipcListener.Close() n.ipcListener = nil - n.log.Info(fmt.Sprintf("IPC endpoint closed: %s", n.ipcEndpoint)) + n.log.Info("IPC endpoint closed", "endpoint", n.ipcEndpoint) } if n.ipcHandler != nil { n.ipcHandler.Stop() @@ -365,7 +365,7 @@ func (n *Node) stopIPC() { } // startHTTP initializes and starts the HTTP RPC endpoint. -func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors []string) error { +func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string) error { // Short circuit if the HTTP endpoint isn't being exposed if endpoint == "" { return nil @@ -382,7 +382,7 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors if err := handler.RegisterName(api.Namespace, api.Service); err != nil { return err } - n.log.Debug(fmt.Sprintf("HTTP registered %T under '%s'", api.Service, api.Namespace)) + n.log.Debug("HTTP registered", "service", api.Service, "namespace", api.Namespace) } } // All APIs registered, start the HTTP listener @@ -393,9 +393,8 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors if listener, err = net.Listen("tcp", endpoint); err != nil { return err } - go rpc.NewHTTPServer(cors, handler).Serve(listener) - n.log.Info(fmt.Sprintf("HTTP endpoint opened: http://%s", endpoint)) - + go rpc.NewHTTPServer(cors, vhosts, handler).Serve(listener) + n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "hvosts", strings.Join(vhosts, ",")) // All listeners booted successfully n.httpEndpoint = endpoint n.httpListener = listener @@ -410,7 +409,7 @@ func (n *Node) stopHTTP() { n.httpListener.Close() n.httpListener = nil - n.log.Info(fmt.Sprintf("HTTP endpoint closed: http://%s", n.httpEndpoint)) + n.log.Info("HTTP endpoint closed", "url", fmt.Sprintf("http://%s", n.httpEndpoint)) } if n.httpHandler != nil { n.httpHandler.Stop() @@ -436,7 +435,7 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig if err := handler.RegisterName(api.Namespace, api.Service); err != nil { return err } - n.log.Debug(fmt.Sprintf("WebSocket registered %T under '%s'", api.Service, api.Namespace)) + n.log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace) } } // All APIs registered, start the HTTP listener @@ -448,7 +447,7 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig return err } go rpc.NewWSServer(wsOrigins, handler).Serve(listener) - n.log.Info(fmt.Sprintf("WebSocket endpoint opened: ws://%s", listener.Addr())) + n.log.Info("WebSocket endpoint opened", "url", fmt.Sprintf("ws://%s", listener.Addr())) // All listeners booted successfully n.wsEndpoint = endpoint @@ -464,7 +463,7 @@ func (n *Node) stopWS() { n.wsListener.Close() n.wsListener = nil - n.log.Info(fmt.Sprintf("WebSocket endpoint closed: ws://%s", n.wsEndpoint)) + n.log.Info("WebSocket endpoint closed", "url", fmt.Sprintf("ws://%s", n.wsEndpoint)) } if n.wsHandler != nil { n.wsHandler.Stop() diff --git a/p2p/server.go b/p2p/server.go index edc1d9d219..90e92dc05b 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -142,7 +142,7 @@ type Config struct { EnableMsgEvents bool // Logger is a custom logger to use with the p2p.Server. - Logger log.Logger + Logger log.Logger `toml:",omitempty"` } // Server manages all peer connections. diff --git a/rpc/http.go b/rpc/http.go index 6717899b53..277f093a23 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -31,6 +31,7 @@ import ( "time" "github.com/rs/cors" + "strings" ) const ( @@ -148,8 +149,11 @@ func (t *httpReadWriteNopCloser) Close() error { // NewHTTPServer creates a new HTTP RPC server around an API provider. // // Deprecated: Server implements http.Handler -func NewHTTPServer(cors []string, srv *Server) *http.Server { - return &http.Server{Handler: newCorsHandler(srv, cors)} +func NewHTTPServer(cors []string, vhosts []string, srv *Server) *http.Server { + // Wrap the CORS-handler within a host-handler + handler := newCorsHandler(srv, cors) + handler = newVHostHandler(vhosts, handler) + return &http.Server{Handler: handler} } // ServeHTTP serves JSON-RPC requests over HTTP. @@ -195,7 +199,6 @@ func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { if len(allowedOrigins) == 0 { return srv } - c := cors.New(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: []string{http.MethodPost, http.MethodGet}, @@ -204,3 +207,51 @@ func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { }) return c.Handler(srv) } + +// virtualHostHandler is a handler which validates the Host-header of incoming requests. +// The virtualHostHandler can prevent DNS rebinding attacks, which do not utilize CORS-headers, +// since they do in-domain requests against the RPC api. Instead, we can see on the Host-header +// which domain was used, and validate that against a whitelist. +type virtualHostHandler struct { + vhosts map[string]struct{} + next http.Handler +} + +// ServeHTTP serves JSON-RPC requests over HTTP, implements http.Handler +func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // if r.Host is not set, we can continue serving since a browser would set the Host header + if r.Host == "" { + h.next.ServeHTTP(w, r) + return + } + host, _, err := net.SplitHostPort(r.Host) + if err != nil { + // Either invalid (too many colons) or no port specified + host = r.Host + } + if ipAddr := net.ParseIP(host); ipAddr != nil { + // It's an IP address, we can serve that + h.next.ServeHTTP(w, r) + return + + } + // Not an ip address, but a hostname. Need to validate + if _, exist := h.vhosts["*"]; exist { + h.next.ServeHTTP(w, r) + return + } + if _, exist := h.vhosts[host]; exist { + h.next.ServeHTTP(w, r) + return + } + http.Error(w, "invalid host specified", http.StatusForbidden) + return +} + +func newVHostHandler(vhosts []string, next http.Handler) http.Handler { + vhostMap := make(map[string]struct{}) + for _, allowedHost := range vhosts { + vhostMap[strings.ToLower(allowedHost)] = struct{}{} + } + return &virtualHostHandler{vhostMap, next} +} From 6c6247a690edbc9856b823fe9adfad6b43ce8e71 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 12 Feb 2018 14:12:55 +0100 Subject: [PATCH 035/110] node, rpc: fix linter issues --- node/node.go | 2 +- rpc/http.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/node/node.go b/node/node.go index 37bd2eb3ce..a1dd5166da 100644 --- a/node/node.go +++ b/node/node.go @@ -324,7 +324,7 @@ func (n *Node) startIPC(apis []rpc.API) error { return err } go func() { - n.log.Info("IPC endpoint opened", "url", fmt.Sprintf("%s", n.ipcEndpoint)) + n.log.Info("IPC endpoint opened", "url", n.ipcEndpoint) for { conn, err := listener.Accept() diff --git a/rpc/http.go b/rpc/http.go index 277f093a23..a46d8c2b39 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -245,7 +245,6 @@ func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } http.Error(w, "invalid host specified", http.StatusForbidden) - return } func newVHostHandler(vhosts []string, next http.Handler) http.Handler { From 12dab534958c5a91fdba470836a91de545681288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 12 Feb 2018 16:27:53 +0200 Subject: [PATCH 036/110] cmd/puppeth: unify discv4 and discv5 ports --- cmd/puppeth/module_dashboard.go | 78 ++++++++++++++++----------------- cmd/puppeth/module_faucet.go | 8 ++-- cmd/puppeth/module_node.go | 54 +++++++++-------------- cmd/puppeth/wizard.go | 3 +- cmd/puppeth/wizard_explorer.go | 2 +- cmd/puppeth/wizard_faucet.go | 8 ++-- cmd/puppeth/wizard_netstats.go | 14 ++---- cmd/puppeth/wizard_node.go | 10 ++--- cmd/puppeth/wizard_wallet.go | 2 +- 9 files changed, 78 insertions(+), 101 deletions(-) diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 1092c4c88e..1cb2d45491 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -117,7 +117,7 @@ var dashboardContent = `

To run an archive node, download {{.GethGenesis}} and start Geth with: 

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
- 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=1024 --syncmode=full{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFullFlat}}
+ 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=1024 --syncmode=full{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -136,7 +136,7 @@ var dashboardContent = `

To run a full node, download {{.GethGenesis}} and start Geth with: 

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
- 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=512{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFullFlat}}
+ 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=512{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -158,7 +158,7 @@ var dashboardContent = `

To run a light node, download {{.GethGenesis}} and start Geth with: 

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
- 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}
+ 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -177,7 +177,7 @@ var dashboardContent = `

To run an embedded node, download {{.GethGenesis}} and start Geth with: 

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
- 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}
+ 
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -208,7 +208,7 @@ var dashboardContent = ` 
geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}

With your local chain initialized, you can start the Ethereum Wallet: - 

ethereumwallet --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFullFlat}}
+ 
ethereumwallet --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFlat}}


You can download the Ethereum Wallet from https://github.com/ethereum/mist/releases.

@@ -229,7 +229,7 @@ var dashboardContent = ` 
geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}

With your local chain initialized, you can start Mist: - 

mist --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFullFlat}}
+ 
mist --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFlat}}


You can download the Mist browser from https://github.com/ethereum/mist/releases.

@@ -261,7 +261,7 @@ var dashboardContent = `

Inside your Java code you can now import the geth archive and connect to Ethereum: 

import org.ethereum.geth.*;
-Enodes bootnodes = new Enodes();{{range .BootnodesLight}}
+Enodes bootnodes = new Enodes();{{range .Bootnodes}}
 bootnodes.append(new Enode("{{.}}"));{{end}}
 
 NodeConfig config = new NodeConfig();
@@ -294,7 +294,7 @@ node.start();
 
 var error: NSError?
 
-let bootnodes = GethNewEnodesEmpty(){{range .BootnodesLight}}
+let bootnodes = GethNewEnodesEmpty(){{range .Bootnodes}}
 bootnodes?.append(GethNewEnode("{{.}}", &error)){{end}}
 
 let config = GethNewNodeConfig()
@@ -595,44 +595,42 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 		statsLogin = ""
 	}
 	indexfile := new(bytes.Buffer)
-	bootCpp := make([]string, len(conf.bootFull))
-	for i, boot := range conf.bootFull {
+	bootCpp := make([]string, len(conf.bootnodes))
+	for i, boot := range conf.bootnodes {
 		bootCpp[i] = "required:" + strings.TrimPrefix(boot, "enode://")
 	}
-	bootHarmony := make([]string, len(conf.bootFull))
-	for i, boot := range conf.bootFull {
+	bootHarmony := make([]string, len(conf.bootnodes))
+	for i, boot := range conf.bootnodes {
 		bootHarmony[i] = fmt.Sprintf("-Dpeer.active.%d.url=%s", i, boot)
 	}
-	bootPython := make([]string, len(conf.bootFull))
-	for i, boot := range conf.bootFull {
+	bootPython := make([]string, len(conf.bootnodes))
+	for i, boot := range conf.bootnodes {
 		bootPython[i] = "'" + boot + "'"
 	}
 	template.Must(template.New("").Parse(dashboardContent)).Execute(indexfile, map[string]interface{}{
-		"Network":            network,
-		"NetworkID":          conf.Genesis.Config.ChainId,
-		"NetworkTitle":       strings.Title(network),
-		"EthstatsPage":       config.ethstats,
-		"ExplorerPage":       config.explorer,
-		"WalletPage":         config.wallet,
-		"FaucetPage":         config.faucet,
-		"GethGenesis":        network + ".json",
-		"BootnodesFull":      conf.bootFull,
-		"BootnodesLight":     conf.bootLight,
-		"BootnodesFullFlat":  strings.Join(conf.bootFull, ","),
-		"BootnodesLightFlat": strings.Join(conf.bootLight, ","),
-		"Ethstats":           statsLogin,
-		"Ethash":             conf.Genesis.Config.Ethash != nil,
-		"CppGenesis":         network + "-cpp.json",
-		"CppBootnodes":       strings.Join(bootCpp, " "),
-		"HarmonyGenesis":     network + "-harmony.json",
-		"HarmonyBootnodes":   strings.Join(bootHarmony, " "),
-		"ParityGenesis":      network + "-parity.json",
-		"PythonGenesis":      network + "-python.json",
-		"PythonBootnodes":    strings.Join(bootPython, ","),
-		"Homestead":          conf.Genesis.Config.HomesteadBlock,
-		"Tangerine":          conf.Genesis.Config.EIP150Block,
-		"Spurious":           conf.Genesis.Config.EIP155Block,
-		"Byzantium":          conf.Genesis.Config.ByzantiumBlock,
+		"Network":          network,
+		"NetworkID":        conf.Genesis.Config.ChainId,
+		"NetworkTitle":     strings.Title(network),
+		"EthstatsPage":     config.ethstats,
+		"ExplorerPage":     config.explorer,
+		"WalletPage":       config.wallet,
+		"FaucetPage":       config.faucet,
+		"GethGenesis":      network + ".json",
+		"Bootnodes":        conf.bootnodes,
+		"BootnodesFlat":    strings.Join(conf.bootnodes, ","),
+		"Ethstats":         statsLogin,
+		"Ethash":           conf.Genesis.Config.Ethash != nil,
+		"CppGenesis":       network + "-cpp.json",
+		"CppBootnodes":     strings.Join(bootCpp, " "),
+		"HarmonyGenesis":   network + "-harmony.json",
+		"HarmonyBootnodes": strings.Join(bootHarmony, " "),
+		"ParityGenesis":    network + "-parity.json",
+		"PythonGenesis":    network + "-python.json",
+		"PythonBootnodes":  strings.Join(bootPython, ","),
+		"Homestead":        conf.Genesis.Config.HomesteadBlock,
+		"Tangerine":        conf.Genesis.Config.EIP150Block,
+		"Spurious":         conf.Genesis.Config.EIP155Block,
+		"Byzantium":        conf.Genesis.Config.ByzantiumBlock,
 	})
 	files[filepath.Join(workdir, "index.html")] = indexfile.Bytes()
 
@@ -651,7 +649,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 		harmonySpecJSON, _ := conf.Genesis.MarshalJSON()
 		files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON
 
-		paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootFull)
+		paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootnodes)
 		if err != nil {
 			return nil, err
 		}
diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go
index 92b4cb2861..976bf04d00 100644
--- a/cmd/puppeth/module_faucet.go
+++ b/cmd/puppeth/module_faucet.go
@@ -93,7 +93,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"NetworkID":     config.node.network,
 		"Bootnodes":     strings.Join(bootnodes, ","),
 		"Ethstats":      config.node.ethstats,
-		"EthPort":       config.node.portFull,
+		"EthPort":       config.node.port,
 		"CaptchaToken":  config.captchaToken,
 		"CaptchaSecret": config.captchaSecret,
 		"FaucetName":    strings.Title(network),
@@ -110,7 +110,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"Datadir":       config.node.datadir,
 		"VHost":         config.host,
 		"ApiPort":       config.port,
-		"EthPort":       config.node.portFull,
+		"EthPort":       config.node.port,
 		"EthName":       config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
 		"CaptchaToken":  config.captchaToken,
 		"CaptchaSecret": config.captchaSecret,
@@ -158,7 +158,7 @@ func (info *faucetInfos) Report() map[string]string {
 	report := map[string]string{
 		"Website address":              info.host,
 		"Website listener port":        strconv.Itoa(info.port),
-		"Ethereum listener port":       strconv.Itoa(info.node.portFull),
+		"Ethereum listener port":       strconv.Itoa(info.node.port),
 		"Funding amount (base tier)":   fmt.Sprintf("%d Ethers", info.amount),
 		"Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes),
 		"Funding tiers":                strconv.Itoa(info.tiers),
@@ -228,7 +228,7 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
 	return &faucetInfos{
 		node: &nodeInfos{
 			datadir:  infos.volumes["/root/.faucet"],
-			portFull: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
+			port:     infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
 			ethstats: infos.envvars["ETH_NAME"],
 			keyJSON:  keyJSON,
 			keyPass:  keyPass,
diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go
index 69cb19c349..2609fd976e 100644
--- a/cmd/puppeth/module_node.go
+++ b/cmd/puppeth/module_node.go
@@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -56,15 +56,13 @@ services:
     build: .
     image: {{.Network}}/{{.Type}}
     ports:
-      - "{{.FullPort}}:{{.FullPort}}"
-      - "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
-      - "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
+      - "{{.Port}}:{{.Port}}"
+      - "{{.Port}}:{{.Port}}/udp"
     volumes:
       - {{.Datadir}}:/root/.ethereum{{if .Ethashdir}}
       - {{.Ethashdir}}:/root/.ethash{{end}}
     environment:
-      - FULL_PORT={{.FullPort}}/tcp
-      - LIGHT_PORT={{.LightPort}}/udp
+      - PORT={{.Port}}/tcp
       - TOTAL_PEERS={{.TotalPeers}}
       - LIGHT_PEERS={{.LightPeers}}
       - STATS_NAME={{.Ethstats}}
@@ -82,12 +80,11 @@ services:
 // deployNode deploys a new Ethereum node container to a remote machine via SSH,
 // docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos, nocache bool) ([]byte, error) {
+func deployNode(client *sshClient, network string, bootnodes []string, config *nodeInfos, nocache bool) ([]byte, error) {
 	kind := "sealnode"
 	if config.keyJSON == "" && config.etherbase == "" {
 		kind = "bootnode"
-		bootv4 = make([]string, 0)
-		bootv5 = make([]string, 0)
+		bootnodes = make([]string, 0)
 	}
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
@@ -100,11 +97,10 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 	dockerfile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
 		"NetworkID": config.network,
-		"Port":      config.portFull,
+		"Port":      config.port,
 		"Peers":     config.peersTotal,
 		"LightFlag": lightFlag,
-		"BootV4":    strings.Join(bootv4, ","),
-		"BootV5":    strings.Join(bootv5, ","),
+		"Bootnodes": strings.Join(bootnodes, ","),
 		"Ethstats":  config.ethstats,
 		"Etherbase": config.etherbase,
 		"GasTarget": uint64(1000000 * config.gasTarget),
@@ -119,10 +115,9 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 		"Datadir":    config.datadir,
 		"Ethashdir":  config.ethashdir,
 		"Network":    network,
-		"FullPort":   config.portFull,
+		"Port":       config.port,
 		"TotalPeers": config.peersTotal,
 		"Light":      config.peersLight > 0,
-		"LightPort":  config.portFull + 1,
 		"LightPeers": config.peersLight,
 		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
 		"Etherbase":  config.etherbase,
@@ -157,10 +152,8 @@ type nodeInfos struct {
 	datadir    string
 	ethashdir  string
 	ethstats   string
-	portFull   int
-	portLight  int
-	enodeFull  string
-	enodeLight string
+	port       int
+	enode      string
 	peersTotal int
 	peersLight int
 	etherbase  string
@@ -174,15 +167,11 @@ type nodeInfos struct {
 // most - but not all - fields for reporting to the user.
 func (info *nodeInfos) Report() map[string]string {
 	report := map[string]string{
-		"Data directory":             info.datadir,
-		"Listener port (full nodes)": strconv.Itoa(info.portFull),
-		"Peer count (all total)":     strconv.Itoa(info.peersTotal),
-		"Peer count (light nodes)":   strconv.Itoa(info.peersLight),
-		"Ethstats username":          info.ethstats,
-	}
-	if info.peersLight > 0 {
-		// Light server enabled
-		report["Listener port (light nodes)"] = strconv.Itoa(info.portLight)
+		"Data directory":           info.datadir,
+		"Listener port":            strconv.Itoa(info.port),
+		"Peer count (all total)":   strconv.Itoa(info.peersTotal),
+		"Peer count (light nodes)": strconv.Itoa(info.peersLight),
+		"Ethstats username":        info.ethstats,
 	}
 	if info.gasTarget > 0 {
 		// Miner or signer node
@@ -250,7 +239,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		keyPass = string(bytes.TrimSpace(out))
 	}
 	// Run a sanity check to see if the devp2p is reachable
-	port := infos.portmap[infos.envvars["FULL_PORT"]]
+	port := infos.portmap[infos.envvars["PORT"]]
 	if err = checkPort(client.server, port); err != nil {
 		log.Warn(fmt.Sprintf("%s devp2p port seems unreachable", strings.Title(kind)), "server", client.server, "port", port, "err", err)
 	}
@@ -259,8 +248,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		genesis:    genesis,
 		datadir:    infos.volumes["/root/.ethereum"],
 		ethashdir:  infos.volumes["/root/.ethash"],
-		portFull:   infos.portmap[infos.envvars["FULL_PORT"]],
-		portLight:  infos.portmap[infos.envvars["LIGHT_PORT"]],
+		port:       port,
 		peersTotal: totalPeers,
 		peersLight: lightPeers,
 		ethstats:   infos.envvars["STATS_NAME"],
@@ -270,9 +258,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		gasTarget:  gasTarget,
 		gasPrice:   gasPrice,
 	}
-	stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull)
-	if stats.portLight != 0 {
-		stats.enodeLight = fmt.Sprintf("enode://%s@%s:%d?discport=%d", id, client.address, stats.portFull, stats.portLight)
-	}
+	stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
+
 	return stats, nil
 }
diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go
index 2e2b4644cf..b88a61de7d 100644
--- a/cmd/puppeth/wizard.go
+++ b/cmd/puppeth/wizard.go
@@ -40,8 +40,7 @@ import (
 // between sessions.
 type config struct {
 	path      string   // File containing the configuration values
-	bootFull  []string // Bootnodes to always connect to by full nodes
-	bootLight []string // Bootnodes to always connect to by light nodes
+	bootnodes []string // Bootnodes to always connect to by all nodes
 	ethstats  string   // Ethstats settings to cache for node deploys
 
 	Genesis *core.Genesis     `json:"genesis,omitempty"` // Genesis block to cache for node deploys
diff --git a/cmd/puppeth/wizard_explorer.go b/cmd/puppeth/wizard_explorer.go
index 10ef72f788..413511c1c3 100644
--- a/cmd/puppeth/wizard_explorer.go
+++ b/cmd/puppeth/wizard_explorer.go
@@ -55,7 +55,7 @@ func (w *wizard) deployExplorer() {
 	}
 	existed := err == nil
 
-	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootFull)
+	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootnodes)
 	if err != nil {
 		log.Error("Failed to create chain spec for explorer", "err", err)
 		return
diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go
index 191575b168..9a429bc96d 100644
--- a/cmd/puppeth/wizard_faucet.go
+++ b/cmd/puppeth/wizard_faucet.go
@@ -38,7 +38,7 @@ func (w *wizard) deployFaucet() {
 	infos, err := checkFaucet(client, w.network)
 	if err != nil {
 		infos = &faucetInfos{
-			node:    &nodeInfos{portFull: 30303, peersTotal: 25},
+			node:    &nodeInfos{port: 30303, peersTotal: 25},
 			port:    80,
 			host:    client.server,
 			amount:  1,
@@ -113,8 +113,8 @@ func (w *wizard) deployFaucet() {
 	}
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.portFull)
-	infos.node.portFull = w.readDefaultInt(infos.node.portFull)
+	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.port)
+	infos.node.port = w.readDefaultInt(infos.node.port)
 
 	// Set a proper name to report on the stats page
 	fmt.Println()
@@ -168,7 +168,7 @@ func (w *wizard) deployFaucet() {
 		fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultString("n") != "n"
 	}
-	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos, nocache); err != nil {
+	if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy faucet container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
diff --git a/cmd/puppeth/wizard_netstats.go b/cmd/puppeth/wizard_netstats.go
index e19180bb16..90bf7ae3c8 100644
--- a/cmd/puppeth/wizard_netstats.go
+++ b/cmd/puppeth/wizard_netstats.go
@@ -37,8 +37,7 @@ func (w *wizard) networkStats() {
 	}
 	// Clear out some previous configs to refill from current scan
 	w.conf.ethstats = ""
-	w.conf.bootFull = w.conf.bootFull[:0]
-	w.conf.bootLight = w.conf.bootLight[:0]
+	w.conf.bootnodes = w.conf.bootnodes[:0]
 
 	// Iterate over all the specified hosts and check their status
 	var pend sync.WaitGroup
@@ -76,8 +75,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
 	var (
 		genesis   string
 		ethstats  string
-		bootFull  []string
-		bootLight []string
+		bootnodes []string
 	)
 	// Ensure a valid SSH connection to the remote server
 	logger := log.New("server", server)
@@ -123,10 +121,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
 		stat.services["bootnode"] = infos.Report()
 
 		genesis = string(infos.genesis)
-		bootFull = append(bootFull, infos.enodeFull)
-		if infos.enodeLight != "" {
-			bootLight = append(bootLight, infos.enodeLight)
-		}
+		bootnodes = append(bootnodes, infos.enode)
 	}
 	logger.Debug("Checking for sealnode availability")
 	if infos, err := checkNode(client, w.network, false); err != nil {
@@ -184,8 +179,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
 	if ethstats != "" {
 		w.conf.ethstats = ethstats
 	}
-	w.conf.bootFull = append(w.conf.bootFull, bootFull...)
-	w.conf.bootLight = append(w.conf.bootLight, bootLight...)
+	w.conf.bootnodes = append(w.conf.bootnodes, bootnodes...)
 
 	return stat
 }
diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go
index 097e2e41aa..a60948bc67 100644
--- a/cmd/puppeth/wizard_node.go
+++ b/cmd/puppeth/wizard_node.go
@@ -48,9 +48,9 @@ func (w *wizard) deployNode(boot bool) {
 	infos, err := checkNode(client, w.network, boot)
 	if err != nil {
 		if boot {
-			infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256}
+			infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256}
 		} else {
-			infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
+			infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
 		}
 	}
 	existed := err == nil
@@ -79,8 +79,8 @@ func (w *wizard) deployNode(boot bool) {
 	}
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
-	infos.portFull = w.readDefaultInt(infos.portFull)
+	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
 
 	// Figure out how many peers to allow (different based on node type)
 	fmt.Println()
@@ -163,7 +163,7 @@ func (w *wizard) deployNode(boot bool) {
 		fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultString("n") != "n"
 	}
-	if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos, nocache); err != nil {
+	if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy Ethereum node container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
diff --git a/cmd/puppeth/wizard_wallet.go b/cmd/puppeth/wizard_wallet.go
index 7c3896a17c..933cd9ae59 100644
--- a/cmd/puppeth/wizard_wallet.go
+++ b/cmd/puppeth/wizard_wallet.go
@@ -98,7 +98,7 @@ func (w *wizard) deployWallet() {
 		fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultString("n") != "n"
 	}
-	if out, err := deployWallet(client, w.network, w.conf.bootFull, infos, nocache); err != nil {
+	if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy wallet container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)

From 8d32c4b9906d2a97149c4312998d46e924d863d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= 
Date: Mon, 12 Feb 2018 17:03:17 +0100
Subject: [PATCH 037/110] light: new CHTs (#16074)

---
 light/postprocess.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/light/postprocess.go b/light/postprocess.go
index b6756de510..82c5027335 100644
--- a/light/postprocess.go
+++ b/light/postprocess.go
@@ -58,18 +58,18 @@ type trustedCheckpoint struct {
 var (
 	mainnetCheckpoint = trustedCheckpoint{
 		name:          "mainnet",
-		sectionIdx:    150,
-		sectionHead:   common.HexToHash("1e2e67f289565cbe7bd4367f7960dbd73a3f7c53439e1047cd7ba331c8109e39"),
-		chtRoot:       common.HexToHash("f2a6c9ca143d647b44523cc249f1072c8912358ab873a77a5fdc792b8df99e80"),
-		bloomTrieRoot: common.HexToHash("c018952fa1513c97857e79fbb9a37acaf8432d5b85e52a78eca7dff5fd5900ee"),
+		sectionIdx:    153,
+		sectionHead:   common.HexToHash("04c2114a8cbe49ba5c37a03cc4b4b8d3adfc0bd2c78e0e726405dd84afca1d63"),
+		chtRoot:       common.HexToHash("d7ec603e5d30b567a6e894ee7704e4603232f206d3e5a589794cec0c57bf318e"),
+		bloomTrieRoot: common.HexToHash("0b139b8fb692e21f663ff200da287192201c28ef5813c1ac6ba02a0a4799eef9"),
 	}
 
 	ropstenCheckpoint = trustedCheckpoint{
 		name:          "ropsten",
-		sectionIdx:    75,
-		sectionHead:   common.HexToHash("12e68324f4578ea3e8e7fb3968167686729396c9279287fa1f1a8b51bb2d05b4"),
-		chtRoot:       common.HexToHash("3e51dc095c69fa654a4cac766e0afff7357515b4b3c3a379c675f810363e54be"),
-		bloomTrieRoot: common.HexToHash("33e3a70b33c1d73aa698d496a80615e98ed31fa8f56969876180553b32333339"),
+		sectionIdx:    79,
+		sectionHead:   common.HexToHash("1b1ba890510e06411fdee9bb64ca7705c56a1a4ce3559ddb34b3680c526cb419"),
+		chtRoot:       common.HexToHash("71d60207af74e5a22a3e1cfbfc89f9944f91b49aa980c86fba94d568369eaf44"),
+		bloomTrieRoot: common.HexToHash("70aca4b3b6d08dde8704c95cedb1420394453c1aec390947751e69ff8c436360"),
 	}
 )
 

From b007412db165e22640eb4cef6cb41013f54ddd32 Mon Sep 17 00:00:00 2001
From: Felix Lange 
Date: Tue, 13 Feb 2018 14:12:55 +0100
Subject: [PATCH 038/110] core: soften up state memory force-commit log
 messages (#16080)

Talk about "state" instead of "trie timing", "trie memory" and remove
the overzealous warning when the limit is just reached. Since the time
limit is always reached on slow machines, move the message to info level
so users don't freak out about internal details.
---
 core/blockchain.go | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/core/blockchain.go b/core/blockchain.go
index e498dedefc..4ae0e4f4ec 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -926,13 +926,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 				if chosen < lastWrite+triesInMemory {
 					switch {
 					case size >= 2*limit:
-						log.Error("Trie memory critical, forcing to disk", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+						log.Warn("State memory usage too high, committing", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
 					case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit:
-						log.Error("Trie timing critical, forcing to disk", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
-					case size > limit:
-						log.Warn("Trie memory at dangerous levels", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
-					case bc.gcproc > bc.cacheConfig.TrieTimeLimit:
-						log.Warn("Trie timing at dangerous levels", "time", bc.gcproc, "limit", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+						log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
 					}
 				}
 				// If optimum or critical limits reached, write to disk

From 88f2839da4993ab51c32fef8383136f3addaeb24 Mon Sep 17 00:00:00 2001
From: Felix Lange 
Date: Tue, 13 Feb 2018 18:32:20 +0100
Subject: [PATCH 039/110] travis.yml: work around Go 1.9.4 issue (#16082)

* travis.yml: work around Go 1.9.4 issue

* travis: workaround the workaround
---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index ba62b87bf5..3941fa785b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -185,6 +185,8 @@ matrix:
         - xctool -version
         - xcrun simctl list
 
+        # Workaround for https://github.com/golang/go/issues/23749
+        - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk

From 20797348ca9d037dc5b7a830bafdfe1ea703eac0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= 
Date: Tue, 13 Feb 2018 20:59:43 +0200
Subject: [PATCH 040/110] p2p/discover: fix out-of-bounds issue

---
 p2p/discover/table.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/p2p/discover/table.go b/p2p/discover/table.go
index 84c54dac12..17c9db7774 100644
--- a/p2p/discover/table.go
+++ b/p2p/discover/table.go
@@ -763,7 +763,7 @@ func (tab *Table) addReplacement(b *bucket, n *Node) {
 // last entry in the bucket. If 'last' isn't the last entry, it has either been replaced
 // with someone else or became active.
 func (tab *Table) replace(b *bucket, last *Node) *Node {
-	if len(b.entries) >= 0 && b.entries[len(b.entries)-1].ID != last.ID {
+	if len(b.entries) == 0 || b.entries[len(b.entries)-1].ID != last.ID {
 		// Entry has moved, don't replace it.
 		return nil
 	}

From a5c0bbb4f4c321c355637ef57fff807857128c6b Mon Sep 17 00:00:00 2001
From: Felix Lange 
Date: Wed, 14 Feb 2018 13:49:11 +0100
Subject: [PATCH 041/110] all: update license information (#16089)

---
 .mailmap                          | 14 ++++-
 AUTHORS                           | 92 ++++++++++++++++++++++++++++++-
 accounts/abi/pack_test.go         |  2 +-
 accounts/abi/unpack.go            |  2 +-
 accounts/abi/unpack_test.go       |  2 +-
 build/update-license.go           |  5 +-
 cmd/ethkey/inspect.go             | 16 ++++++
 cmd/ethkey/message_test.go        |  2 +-
 cmd/ethkey/run_test.go            |  2 +-
 cmd/evm/json_logger.go            | 14 ++---
 cmd/p2psim/main.go                | 16 ++++++
 cmd/swarm/run_test.go             |  2 +-
 cmd/swarm/upload_test.go          |  2 +-
 common/fdlimit/fdlimit_freebsd.go | 14 ++---
 common/fdlimit/fdlimit_test.go    | 14 ++---
 common/fdlimit/fdlimit_unix.go    | 14 ++---
 common/fdlimit/fdlimit_windows.go | 16 +++---
 core/vm/contracts_test.go         | 16 ++++++
 core/vm/instructions_test.go      | 16 ++++++
 eth/api_test.go                   |  2 +-
 eth/tracers/tracer.go             |  2 +-
 eth/tracers/tracer_test.go        |  2 +-
 internal/cmdtest/test_cmd.go      | 16 +++---
 internal/ethapi/addrlock.go       |  2 +-
 les/retrieve.go                   |  2 +-
 light/nodeset.go                  |  2 +-
 light/postprocess.go              |  2 +-
 p2p/simulations/adapters/state.go |  1 +
 rpc/types_test.go                 |  2 +-
 swarm/api/http/error_test.go      |  2 +-
 tests/difficulty_test.go          |  1 -
 tests/difficulty_test_util.go     |  1 -
 tests/init.go                     |  2 +-
 tests/init_test.go                |  2 +-
 tests/state_test.go               |  2 +-
 tests/state_test_util.go          |  2 +-
 trie/database.go                  |  2 +-
 37 files changed, 235 insertions(+), 73 deletions(-)

diff --git a/.mailmap b/.mailmap
index d51c7b6093..cc4b871a3c 100644
--- a/.mailmap
+++ b/.mailmap
@@ -65,7 +65,8 @@ Enrique Fynn 
 
 Vincent G 
 
-RJ Catalano 
+RJ Catalano 
+RJ Catalano  
 
 Nchinda Nchinda 
 
@@ -109,3 +110,14 @@ Frank Wang 
 Gary Rong 
 
 Guillaume Nicolas 
+
+Sorin Neacsu 
+Sorin Neacsu  
+
+Valentin Wüstholz 
+Valentin Wüstholz  
+
+Armin Braun 
+
+Ernesto del Toro 
+Ernesto del Toro  
diff --git a/AUTHORS b/AUTHORS
index faa19d281c..bd44a3de55 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -1,85 +1,173 @@
 # This is the official list of go-ethereum authors for copyright purposes.
 
+Afri Schoedon <5chdn@users.noreply.github.com>
+Agustin Armellini Fischer 
+Airead 
+Alan Chen 
+Alejandro Isaza 
 Ales Katona 
 Alex Leverington 
+Alex Wu 
 Alexandre Van de Sande 
+Ali Hajimirza 
+Anton Evangelatov 
+Arba Sasmoyo 
+Armani Ferrante 
+Armin Braun 
 Aron Fischer 
 Bas van Kervel 
 Benjamin Brent 
+Benoit Verkindt 
+Bo 
+Bo Ye 
+Bob Glickstein 
 Brian Schroeder 
 Casey Detrio 
+Chase Wright 
 Christoph Jentzsch 
 Daniel A. Nagy 
+Daniel Sloof 
+Darrel Herbst 
+Dave Appleton 
 Diego Siqueira 
+Dmitry Shulyak 
+Egon Elbre 
+Elias Naur 
 Elliot Shepherd 
 Enrique Fynn 
+Ernesto del Toro 
 Ethan Buchman 
+Eugene Valeyev 
+Evangelos Pappas 
+Evgeny Danilenko <6655321@bk.ru>
 Fabian Vogelsteller 
+Fabio Barone 
 Fabio Berger 
+FaceHo 
 Felix Lange 
+Fiisio 
 Frank Wang 
+Furkan KAMACI 
 Gary Rong 
+George Ornbo 
 Gregg Dourgarian 
+Guillaume Ballet 
 Guillaume Nicolas 
 Gustav Simonsson 
 Hao Bryan Cheng 
 Henning Diedrich 
 Isidoro Ghezzi 
+Ivan Daniluk 
 Jae Kwon 
 Jamie Pitts 
+Janoš Guljaš 
 Jason Carver 
+Jay Guo 
 Jeff R. Allen 
 Jeffrey Wilcke 
 Jens Agerberg 
+Jia Chenhui 
+Jim McDonald 
+Joel Burget 
 Jonathan Brown 
 Joseph Chow 
 Justin Clark-Casey 
 Justin Drake 
 Kenji Siu 
 Kobi Gurkan 
+Konrad Feldmeier 
+Kurkó Mihály 
+Kyuntae Ethan Kim 
 Lefteris Karapetsas 
 Leif Jurvetson 
+Leo Shklovskii 
 Lewis Marshall 
+Lio李欧 
 Louis Holbrook 
 Luca Zeug 
+Magicking 
 Maran Hidskes 
 Marek Kotewicz 
+Mark 
 Martin Holst Swende 
 Matthew Di Ferrante 
 Matthew Wampler-Doty 
+Maximilian Meister 
 Micah Zoltu 
+Michael Ruminer 
+Miguel Mota 
+Miya Chen 
 Nchinda Nchinda 
 Nick Dodson 
 Nick Johnson 
+Nicolas Guillaume 
+Noman 
+Oli Bye 
+Paul Litvak 
 Paulo L F Casaretto 
+Paweł Bylica 
 Peter Pratscher 
+Petr Mikusek 
 Péter Szilágyi 
-RJ Catalano 
+RJ Catalano 
 Ramesh Nair 
 Ricardo Catalinas Jiménez 
+Ricardo Domingos 
+Richard Hart 
+Rob 
+Robert Zaremba 
+Russ Cox 
 Rémy Roy 
+S. Matthew English 
 Shintaro Kaneko 
+Sorin Neacsu 
 Stein Dekker 
+Steve Waldman 
 Steven Roose 
 Taylor Gerring 
 Thomas Bocek 
+Ti Zhou 
 Tosh Camille 
-Valentin Wüstholz 
+Valentin Wüstholz 
 Victor Farazdagi 
 Victor Tran 
 Viktor Trón 
 Ville Sundell 
 Vincent G 
 Vitalik Buterin 
+Vitaly V 
 Vivek Anand 
 Vlad Gluhovsky 
 Yohann Léon 
 Yoichi Hirai 
+Yondon Fu 
+Zach 
 Zahoor Mohamed 
+Zoe Nolan 
 Zsolt Felföldi 
+am2rican5 
+ayeowch 
+b00ris 
+bailantaotao 
+baizhenxuan 
+bloonfield 
+changhong 
+evgk 
+ferhat elmas 
 holisticode 
+jtakalai 
 ken10100147 
 ligi 
+mark.lin 
+necaremus 
+njupt-moon <1015041018@njupt.edu.cn>
+nkbai 
+rhaps107 
+slumber1122 
+sunxiaojun2014 
+terasum 
+tsarpaul 
 xiekeyang 
+yoza 
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
 Максим Чусовлянов 
diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go
index 36401ee677..14ab516ac2 100644
--- a/accounts/abi/pack_test.go
+++ b/accounts/abi/pack_test.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go
index 80efb3f7ea..3342456618 100644
--- a/accounts/abi/unpack.go
+++ b/accounts/abi/unpack.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go
index 4d7fe638c4..a65426a30b 100644
--- a/accounts/abi/unpack_test.go
+++ b/accounts/abi/unpack_test.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/build/update-license.go b/build/update-license.go
index 3d69598b75..22e4033428 100644
--- a/build/update-license.go
+++ b/build/update-license.go
@@ -55,10 +55,9 @@ var (
 		"crypto/sha3/",
 		"internal/jsre/deps",
 		"log/",
+		"common/bitutil/bitutil",
 		// don't license generated files
-		"contracts/chequebook/contract/",
-		"contracts/ens/contract/",
-		"contracts/release/contract.go",
+		"contracts/chequebook/contract/code.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.
diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go
index 219a5460b8..dbf5afc0ce 100644
--- a/cmd/ethkey/inspect.go
+++ b/cmd/ethkey/inspect.go
@@ -1,3 +1,19 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see .
+
 package main
 
 import (
diff --git a/cmd/ethkey/message_test.go b/cmd/ethkey/message_test.go
index fb16f03d02..39352b1d22 100644
--- a/cmd/ethkey/message_test.go
+++ b/cmd/ethkey/message_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
diff --git a/cmd/ethkey/run_test.go b/cmd/ethkey/run_test.go
index 8ce4fe5cde..6006f6b5bb 100644
--- a/cmd/ethkey/run_test.go
+++ b/cmd/ethkey/run_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go
index 47daf7dbbc..0e7a911896 100644
--- a/cmd/evm/json_logger.go
+++ b/cmd/evm/json_logger.go
@@ -1,18 +1,18 @@
 // Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
+// This file is part of go-ethereum.
 //
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// The go-ethereum library is distributed in the hope that it will be useful,
+// go-ethereum is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see .
 
 package main
 
diff --git a/cmd/p2psim/main.go b/cmd/p2psim/main.go
index 56b74d135b..0c8ed038d5 100644
--- a/cmd/p2psim/main.go
+++ b/cmd/p2psim/main.go
@@ -1,3 +1,19 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see .
+
 // p2psim provides a command-line client for a simulation HTTP API.
 //
 // Here is an example of creating a 2 node network with the first node
diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go
index ed15028685..594cfa55cb 100644
--- a/cmd/swarm/run_test.go
+++ b/cmd/swarm/run_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go
index 5656186e1c..df7fc216af 100644
--- a/cmd/swarm/upload_test.go
+++ b/cmd/swarm/upload_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
diff --git a/common/fdlimit/fdlimit_freebsd.go b/common/fdlimit/fdlimit_freebsd.go
index 25caaafe21..c126b0c265 100644
--- a/common/fdlimit/fdlimit_freebsd.go
+++ b/common/fdlimit/fdlimit_freebsd.go
@@ -1,18 +1,18 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 // +build freebsd
 
diff --git a/common/fdlimit/fdlimit_test.go b/common/fdlimit/fdlimit_test.go
index 05e9f0b658..a9ee9ab36a 100644
--- a/common/fdlimit/fdlimit_test.go
+++ b/common/fdlimit/fdlimit_test.go
@@ -1,18 +1,18 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 package fdlimit
 
diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go
index 27c7e783f7..a258132353 100644
--- a/common/fdlimit/fdlimit_unix.go
+++ b/common/fdlimit/fdlimit_unix.go
@@ -1,18 +1,18 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 // +build linux darwin netbsd openbsd solaris
 
diff --git a/common/fdlimit/fdlimit_windows.go b/common/fdlimit/fdlimit_windows.go
index efcd3220ea..863c58bedf 100644
--- a/common/fdlimit/fdlimit_windows.go
+++ b/common/fdlimit/fdlimit_windows.go
@@ -1,18 +1,18 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 package fdlimit
 
diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go
index 513651835a..96083337c9 100644
--- a/core/vm/contracts_test.go
+++ b/core/vm/contracts_test.go
@@ -1,3 +1,19 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
 package vm
 
 import (
diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go
index 18644989c1..180433ea88 100644
--- a/core/vm/instructions_test.go
+++ b/core/vm/instructions_test.go
@@ -1,3 +1,19 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
 package vm
 
 import (
diff --git a/eth/api_test.go b/eth/api_test.go
index 248bc3ab6a..900a82bb6a 100644
--- a/eth/api_test.go
+++ b/eth/api_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go
index f3f848fc1b..4cec9e633c 100644
--- a/eth/tracers/tracer.go
+++ b/eth/tracers/tracer.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go
index 7224a1489f..117c376b81 100644
--- a/eth/tracers/tracer_test.go
+++ b/eth/tracers/tracer_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go
index fae61cfe32..20e82ec2a9 100644
--- a/internal/cmdtest/test_cmd.go
+++ b/internal/cmdtest/test_cmd.go
@@ -1,18 +1,18 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 package cmdtest
 
diff --git a/internal/ethapi/addrlock.go b/internal/ethapi/addrlock.go
index 5a9c948b83..61ddff688c 100644
--- a/internal/ethapi/addrlock.go
+++ b/internal/ethapi/addrlock.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/les/retrieve.go b/les/retrieve.go
index dd15b56acf..e262a3cb47 100644
--- a/les/retrieve.go
+++ b/les/retrieve.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/light/nodeset.go b/light/nodeset.go
index 245b5eb766..6f25219c13 100644
--- a/light/nodeset.go
+++ b/light/nodeset.go
@@ -1,4 +1,4 @@
-// Copyright 2014 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/light/postprocess.go b/light/postprocess.go
index 82c5027335..84149fdaae 100644
--- a/light/postprocess.go
+++ b/light/postprocess.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/p2p/simulations/adapters/state.go b/p2p/simulations/adapters/state.go
index 8b1dfef904..0d4ecfb0ff 100644
--- a/p2p/simulations/adapters/state.go
+++ b/p2p/simulations/adapters/state.go
@@ -13,6 +13,7 @@
 //
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see .
+
 package adapters
 
 type SimStateStore struct {
diff --git a/rpc/types_test.go b/rpc/types_test.go
index 30cef9b22e..68b6d3c54f 100644
--- a/rpc/types_test.go
+++ b/rpc/types_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go
index ed52bafbd2..c2c8b908b8 100644
--- a/swarm/api/http/error_test.go
+++ b/swarm/api/http/error_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go
index a449b1cfa6..6006373009 100644
--- a/tests/difficulty_test.go
+++ b/tests/difficulty_test.go
@@ -13,7 +13,6 @@
 //
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see .
-//
 
 package tests
 
diff --git a/tests/difficulty_test_util.go b/tests/difficulty_test_util.go
index 7541477933..00d699cf75 100644
--- a/tests/difficulty_test_util.go
+++ b/tests/difficulty_test_util.go
@@ -13,7 +13,6 @@
 //
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see .
-//
 
 package tests
 
diff --git a/tests/init.go b/tests/init.go
index 9e884efe39..ff8ee7da18 100644
--- a/tests/init.go
+++ b/tests/init.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/tests/init_test.go b/tests/init_test.go
index ebb0d32c34..fbb214b08c 100644
--- a/tests/init_test.go
+++ b/tests/init_test.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/tests/state_test.go b/tests/state_test.go
index 100c776c1a..9ca5f18303 100644
--- a/tests/state_test.go
+++ b/tests/state_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/tests/state_test_util.go b/tests/state_test_util.go
index 18280d2a46..3b761bd771 100644
--- a/tests/state_test_util.go
+++ b/tests/state_test_util.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/trie/database.go b/trie/database.go
index d79120813d..da36e72f98 100644
--- a/trie/database.go
+++ b/trie/database.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

From 57bca0af8c51de54b8b0e6a42a9cdcf78a5fac89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= 
Date: Wed, 14 Feb 2018 14:49:53 +0200
Subject: [PATCH 042/110] containers/docker: bump legacy images to 1.8 branch
 (#16084)

---
 containers/docker/master-alpine/Dockerfile | 2 +-
 containers/docker/master-ubuntu/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile
index c7b71c7263..8d4e7fe81a 100644
--- a/containers/docker/master-alpine/Dockerfile
+++ b/containers/docker/master-alpine/Dockerfile
@@ -2,7 +2,7 @@ FROM alpine:3.7
 
 RUN \
   apk add --update go git make gcc musl-dev linux-headers ca-certificates && \
-  git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \
+  git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
   (cd go-ethereum && make geth) && \
   cp go-ethereum/build/bin/geth /geth && \
   apk del go git make gcc musl-dev linux-headers && \
diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile
index bba70abfdd..4cfc4f58c0 100644
--- a/containers/docker/master-ubuntu/Dockerfile
+++ b/containers/docker/master-ubuntu/Dockerfile
@@ -5,7 +5,7 @@ ENV PATH=/usr/lib/go-1.9/bin:$PATH
 RUN \
   apt-get update && apt-get upgrade -q -y && \
   apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \
-  git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \
+  git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
   (cd go-ethereum && make geth) && \
   cp go-ethereum/build/bin/geth /geth && \
   apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \

From 5f54075760748ae7f249bf735565924ea885c477 Mon Sep 17 00:00:00 2001
From: Felix Lange 
Date: Wed, 14 Feb 2018 13:51:30 +0100
Subject: [PATCH 043/110] params: v1.8.0 stable

---
 params/version.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/params/version.go b/params/version.go
index 32d4a2e23c..e485ef0098 100644
--- a/params/version.go
+++ b/params/version.go
@@ -21,10 +21,10 @@ import (
 )
 
 const (
-	VersionMajor = 1          // Major version component of the current release
-	VersionMinor = 8          // Minor version component of the current release
-	VersionPatch = 0          // Patch version component of the current release
-	VersionMeta  = "unstable" // Version metadata to append to the version string
+	VersionMajor = 1        // Major version component of the current release
+	VersionMinor = 8        // Minor version component of the current release
+	VersionPatch = 0        // Patch version component of the current release
+	VersionMeta  = "stable" // Version metadata to append to the version string
 )
 
 // Version holds the textual version string.

From 752761cb578f19aa1d81bbb060d7a0166553ae01 Mon Sep 17 00:00:00 2001
From: Felix Lange 
Date: Wed, 14 Feb 2018 13:55:21 +0100
Subject: [PATCH 044/110] params, VERSION: v1.8.1 unstable

---
 VERSION           | 2 +-
 params/version.go | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/VERSION b/VERSION
index 27f9cd322b..a8fdfda1c7 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.8.0
+1.8.1
diff --git a/params/version.go b/params/version.go
index e485ef0098..2775859349 100644
--- a/params/version.go
+++ b/params/version.go
@@ -21,10 +21,10 @@ import (
 )
 
 const (
-	VersionMajor = 1        // Major version component of the current release
-	VersionMinor = 8        // Minor version component of the current release
-	VersionPatch = 0        // Patch version component of the current release
-	VersionMeta  = "stable" // Version metadata to append to the version string
+	VersionMajor = 1          // Major version component of the current release
+	VersionMinor = 8          // Minor version component of the current release
+	VersionPatch = 1          // Patch version component of the current release
+	VersionMeta  = "unstable" // Version metadata to append to the version string
 )
 
 // Version holds the textual version string.

From ff225db813b6d56ecd35db301bc582ca04e42b27 Mon Sep 17 00:00:00 2001
From: ferhat elmas 
Date: Wed, 14 Feb 2018 14:41:05 +0100
Subject: [PATCH 045/110] core/vm: remove unused hashing (#16075)

---
 core/vm/interpreter.go | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go
index 482e67a3a9..82a6d3de6d 100644
--- a/core/vm/interpreter.go
+++ b/core/vm/interpreter.go
@@ -20,9 +20,7 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -123,11 +121,6 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
 		return nil, nil
 	}
 
-	codehash := contract.CodeHash // codehash is used when doing jump dest caching
-	if codehash == (common.Hash{}) {
-		codehash = crypto.Keccak256Hash(contract.Code)
-	}
-
 	var (
 		op    OpCode        // current opcode
 		mem   = NewMemory() // bound memory

From dfc5842a89781c22e847913f723d6b0a4e439479 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= 
Date: Wed, 14 Feb 2018 21:09:20 +0200
Subject: [PATCH 046/110] les: add missing lock around peer access

---
 les/fetcher.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/les/fetcher.go b/les/fetcher.go
index 3fc4df30b9..9d224176fc 100644
--- a/les/fetcher.go
+++ b/les/fetcher.go
@@ -425,6 +425,9 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
 			},
 			canSend: func(dp distPeer) bool {
 				p := dp.(*peer)
+				f.lock.Lock()
+				defer f.lock.Unlock()
+
 				fp := f.peers[p]
 				return fp != nil && fp.nodeByHash[bestHash] != nil
 			},

From dc7ca52b3b7c84e8371ea0c1acde327149df6c50 Mon Sep 17 00:00:00 2001
From: ferhat elmas 
Date: Wed, 14 Feb 2018 21:02:51 +0100
Subject: [PATCH 047/110] core: handle ignored error (#16065)

- according to implementation of `IntrinsicGas`
we can continue execution since problem will be detected
later. However, early return is future-proof for changes.
---
 core/state_transition.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/state_transition.go b/core/state_transition.go
index 390473fffd..b19bc12e42 100644
--- a/core/state_transition.go
+++ b/core/state_transition.go
@@ -215,6 +215,9 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 
 	// Pay intrinsic gas
 	gas, err := IntrinsicGas(st.data, contractCreation, homestead)
+	if err != nil {
+		return nil, 0, false, err
+	}
 	if err = st.useGas(gas); err != nil {
 		return nil, 0, false, err
 	}

From de93a9d43799bd4e9d6e3966c23ff9d7d053b50f Mon Sep 17 00:00:00 2001
From: Martin Holst Swende 
Date: Thu, 15 Feb 2018 09:16:59 +0100
Subject: [PATCH 048/110] main: add gc flags to import-command

---
 cmd/geth/chaincmd.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go
index 35bf576e1d..85d0c3acaa 100644
--- a/cmd/geth/chaincmd.go
+++ b/cmd/geth/chaincmd.go
@@ -67,6 +67,9 @@ It expects the genesis file as argument.`,
 			utils.DataDirFlag,
 			utils.CacheFlag,
 			utils.LightModeFlag,
+			utils.GCModeFlag,
+			utils.CacheDatabaseFlag,
+			utils.CacheGCFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `

From e2f2bb3e2e57118a3c206be91554c67cc9b2622b Mon Sep 17 00:00:00 2001
From: GuiltyMorishita 
Date: Thu, 15 Feb 2018 19:38:39 +0900
Subject: [PATCH 049/110] node: fix typo hvosts -> vhosts (#16096)

---
 node/node.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/node.go b/node/node.go
index a1dd5166da..b02aecfad1 100644
--- a/node/node.go
+++ b/node/node.go
@@ -394,7 +394,7 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors
 		return err
 	}
 	go rpc.NewHTTPServer(cors, vhosts, handler).Serve(listener)
-	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "hvosts", strings.Join(vhosts, ","))
+	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "vhosts", strings.Join(vhosts, ","))
 	// All listeners booted successfully
 	n.httpEndpoint = endpoint
 	n.httpListener = listener

From fac6d9ce77b636013013cce8eff9f6b218bdf380 Mon Sep 17 00:00:00 2001
From: gluk256 
Date: Thu, 15 Feb 2018 13:42:44 +0100
Subject: [PATCH 050/110] whisper: test timeout extended (#16088)

* whisper: timeout extended

* whisper: test updated

* whisper: test updated
---
 whisper/whisperv6/peer_test.go | 99 +++++++++++++++++++++++++---------
 1 file changed, 74 insertions(+), 25 deletions(-)

diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go
index 188c8f7467..9ce5eed8bc 100644
--- a/whisper/whisperv6/peer_test.go
+++ b/whisper/whisperv6/peer_test.go
@@ -70,9 +70,8 @@ var keys = []string{
 	"7184c1701569e3a4c4d2ddce691edd983b81e42e09196d332e1ae2f1e062cff4",
 }
 
-const NumNodes = 16 // must not exceed the number of keys (32)
-
 type TestData struct {
+	started int
 	counter [NumNodes]int
 	mutex   sync.RWMutex
 }
@@ -84,21 +83,29 @@ type TestNode struct {
 	filerID string
 }
 
+const NumNodes = 8 // must not exceed the number of keys (32)
+
 var result TestData
 var nodes [NumNodes]*TestNode
 var sharedKey = hexutil.MustDecode("0x03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31")
+var wrongKey = hexutil.MustDecode("0xf91156714d7ec88d3edc1c652c2181dbb3044e8771c683f3b30d33c12b986b11")
 var sharedTopic = TopicType{0xF, 0x1, 0x2, 0}
-var expectedMessage = []byte("per rectum ad astra")
+var wrongTopic = TopicType{0, 0, 0, 0}
+var expectedMessage = []byte("per aspera ad astra")
+var unexpectedMessage = []byte("per rectum ad astra")
 var masterBloomFilter []byte
 var masterPow = 0.00000001
 var round = 1
+var debugMode = false
+var prevTime time.Time
+var cntPrev int
 
 func TestSimulation(t *testing.T) {
 	// create a chain of whisper nodes,
 	// installs the filters with shared (predefined) parameters
 	initialize(t)
 
-	// each node sends a number of random (undecryptable) messages
+	// each node sends one random (not decryptable) message
 	for i := 0; i < NumNodes; i++ {
 		sendMsg(t, false, i)
 	}
@@ -115,7 +122,6 @@ func TestSimulation(t *testing.T) {
 
 	// send new pow and bloom exchange messages
 	resetParams(t)
-	round++
 
 	// node #1 sends one expected (decryptable) message
 	sendMsg(t, true, 1)
@@ -140,6 +146,8 @@ func resetParams(t *testing.T) {
 	for i := 0; i < NumNodes; i++ {
 		nodes[i].shh.SetBloomFilter(masterBloomFilter)
 	}
+
+	round++
 }
 
 func initBloom(t *testing.T) {
@@ -219,15 +227,22 @@ func initialize(t *testing.T) {
 		nodes[i] = &node
 	}
 
-	for i := 1; i < NumNodes; i++ {
-		go nodes[i].server.Start()
+	for i := 0; i < NumNodes; i++ {
+		go startServer(t, nodes[i].server)
 	}
 
-	// we need to wait until the first node actually starts
-	err = nodes[0].server.Start()
+	waitForServersToStart(t)
+}
+
+func startServer(t *testing.T, s *p2p.Server) {
+	err := s.Start()
 	if err != nil {
 		t.Fatalf("failed to start the fisrt server.")
 	}
+
+	result.mutex.Lock()
+	defer result.mutex.Unlock()
+	result.started++
 }
 
 func stopServers() {
@@ -246,8 +261,10 @@ func checkPropagation(t *testing.T, includingNodeZero bool) {
 		return
 	}
 
-	const cycle = 50
-	const iterations = 200
+	prevTime = time.Now()
+	// (cycle * iterations) should not exceed 50 seconds, since TTL=50
+	const cycle = 200 // time in milliseconds
+	const iterations = 250
 
 	first := 0
 	if !includingNodeZero {
@@ -262,29 +279,29 @@ func checkPropagation(t *testing.T, includingNodeZero bool) {
 			}
 
 			mail := f.Retrieve()
-			if !validateMail(t, i, mail) {
-				return
-			}
+			validateMail(t, i, mail)
 
 			if isTestComplete() {
+				checkTestStatus()
 				return
 			}
 		}
 
+		checkTestStatus()
 		time.Sleep(cycle * time.Millisecond)
 	}
 
-	t.Fatalf("Test was not complete: timeout %d seconds. nodes=%v", iterations*cycle/1000, nodes)
-
 	if !includingNodeZero {
 		f := nodes[0].shh.GetFilter(nodes[0].filerID)
 		if f != nil {
 			t.Fatalf("node zero received a message with low PoW.")
 		}
 	}
+
+	t.Fatalf("Test was not complete (%d round): timeout %d seconds. nodes=%v", round, iterations*cycle/1000, nodes)
 }
 
-func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool {
+func validateMail(t *testing.T, index int, mail []*ReceivedMessage) {
 	var cnt int
 	for _, m := range mail {
 		if bytes.Equal(m.Payload, expectedMessage) {
@@ -294,14 +311,13 @@ func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool {
 
 	if cnt == 0 {
 		// no messages received yet: nothing is wrong
-		return true
+		return
 	}
 	if cnt > 1 {
 		t.Fatalf("node %d received %d.", index, cnt)
-		return false
 	}
 
-	if cnt > 0 {
+	if cnt == 1 {
 		result.mutex.Lock()
 		defer result.mutex.Unlock()
 		result.counter[index] += cnt
@@ -309,7 +325,28 @@ func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool {
 			t.Fatalf("node %d accumulated %d.", index, result.counter[index])
 		}
 	}
-	return true
+}
+
+func checkTestStatus() {
+	var cnt int
+	var arr [NumNodes]int
+
+	for i := 0; i < NumNodes; i++ {
+		arr[i] = nodes[i].server.PeerCount()
+		envelopes := nodes[i].shh.Envelopes()
+		if len(envelopes) >= NumNodes {
+			cnt++
+		}
+	}
+
+	if debugMode {
+		if cntPrev != cnt {
+			fmt.Printf(" %v \t number of nodes that have received all msgs: %d, number of peers per node: %v \n",
+				time.Since(prevTime), cnt, arr)
+			prevTime = time.Now()
+			cntPrev = cnt
+		}
+	}
 }
 
 func isTestComplete() bool {
@@ -324,7 +361,7 @@ func isTestComplete() bool {
 
 	for i := 0; i < NumNodes; i++ {
 		envelopes := nodes[i].shh.Envelopes()
-		if len(envelopes) < 2 {
+		if len(envelopes) < NumNodes+1 {
 			return false
 		}
 	}
@@ -339,9 +376,10 @@ func sendMsg(t *testing.T, expected bool, id int) {
 
 	opt := MessageParams{KeySym: sharedKey, Topic: sharedTopic, Payload: expectedMessage, PoW: 0.00000001, WorkTime: 1}
 	if !expected {
-		opt.KeySym[0]++
-		opt.Topic[0]++
-		opt.Payload = opt.Payload[1:]
+		opt.KeySym = wrongKey
+		opt.Topic = wrongTopic
+		opt.Payload = unexpectedMessage
+		opt.Payload[0] = byte(id)
 	}
 
 	msg, err := NewSentMessage(&opt)
@@ -459,3 +497,14 @@ func checkBloomFilterExchange(t *testing.T) {
 		time.Sleep(50 * time.Millisecond)
 	}
 }
+
+func waitForServersToStart(t *testing.T) {
+	const iterations = 200
+	for j := 0; j < iterations; j++ {
+		time.Sleep(50 * time.Millisecond)
+		if result.started == NumNodes {
+			return
+		}
+	}
+	t.Fatalf("Failed to start all the servers, running: %d", result.started)
+}

From 5f9b01a2839111a97cfecdb9d746025f433c5276 Mon Sep 17 00:00:00 2001
From: Guillaume Ballet 
Date: Thu, 15 Feb 2018 13:43:48 +0100
Subject: [PATCH 051/110] whisper: only use the node id as a p2p id, not for
 sending messages (#16102)

This is in preparation for the switch to libp2p: the ID generated
will be from a private key created with the help of libp2p's crypto
library, while Whisper will still use Go's default crypto libraries
for encrypting its messages. This change removes a conflict.

It shouldn't have any impact as the person receiving emails is
the user, not the node.
---
 cmd/wnode/main.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go
index 68e6971dae..971b1c0ab8 100644
--- a/cmd/wnode/main.go
+++ b/cmd/wnode/main.go
@@ -265,7 +265,7 @@ func initialize() {
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
 			MaxPeers:       maxPeers,
-			Name:           common.MakeName("wnode", "5.0"),
+			Name:           common.MakeName("wnode", "6.0"),
 			Protocols:      shh.Protocols(),
 			ListenAddr:     *argIP,
 			NAT:            nat.Any(),
@@ -656,7 +656,7 @@ func requestExpiredMessagesLoop() {
 		params.PoW = *argServerPoW
 		params.Payload = data
 		params.KeySym = key
-		params.Src = nodeid
+		params.Src = asymKey
 		params.WorkTime = 5
 
 		msg, err := whisper.NewSentMessage(¶ms)

From 4e61ed02e2d32aa38f5a2f37b87bc52a71657809 Mon Sep 17 00:00:00 2001
From: cooganb 
Date: Thu, 15 Feb 2018 08:24:20 -0600
Subject: [PATCH 052/110] swarm: add favicon for Swarm templates served by
 browser (#15958)

* swarm: added script to HTML header to create favicon addresses #153

* swarm: moved data blob direclty into link tag, removed script

* swarm: added favicon info to other html templates

* swarm: fixing test errors

* swarm: fixing favicon test

* swarm: fixing travis tests

* swarm: added script to HTML header to create favicon addresses #153

* swarm: moved data blob direclty into link tag, removed script

* swarm: added favicon info to other html templates

* swarm: fixing test errors

* swarm: fixing favicon test

* swarm: fixing travis tests
---
 swarm/api/http/error_templates.go | 20 ++++++++++----------
 swarm/api/http/server_test.go     |  6 +++---
 swarm/api/http/templates.go       |  3 ++-
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go
index 2c20ba8f99..0457cb8a70 100644
--- a/swarm/api/http/error_templates.go
+++ b/swarm/api/http/error_templates.go
@@ -37,7 +37,7 @@ func GetGenericErrorPage() string {
     
     
     
-
+		
     
+    Swarm :: Welcome to Swarm
+  
+  
+
+
+      

+        

+          
+        

+        

+          
Welcome to Swarm

+        

+      

+
+			
+				
+
+	            
Enter the hash or ENS of a Swarm-hosted file below:

+								
+								
+
+				
+      

+        

+          Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution

+          Swarm
+        

+      

+
+  
+
+`[1:]))

From 764878d988e7c6673f1edfeea5a47a02c2579d84 Mon Sep 17 00:00:00 2001
From: Anton Evangelatov 
Date: Mon, 26 Feb 2018 10:36:26 +0100
Subject: [PATCH 088/110] contracts/chequebook: increase interval between auto
 deposits (#16178)

---
 contracts/chequebook/cheque_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go
index b7555d0815..6b6b28e657 100644
--- a/contracts/chequebook/cheque_test.go
+++ b/contracts/chequebook/cheque_test.go
@@ -281,8 +281,8 @@ func TestDeposit(t *testing.T) {
 		t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
 	}
 
-	// autodeposit every 30ms if new cheque issued
-	interval := 30 * time.Millisecond
+	// autodeposit every 200ms if new cheque issued
+	interval := 200 * time.Millisecond
 	chbook.AutoDeposit(interval, common.Big1, balance)
 	_, err = chbook.Issue(addr1, amount)
 	if err != nil {

From d398d04e2779c5ebb9713929518f9305dd10af09 Mon Sep 17 00:00:00 2001
From: Domino Valdano 
Date: Mon, 26 Feb 2018 01:38:17 -0800
Subject: [PATCH 089/110] cmd/geth: fix broken links to JavaScript-Console wiki
 in cmd line help (#16183)

* Fixed broken link to JavaScript-Console wiki in cmd line help

* cmd/geth: Added missing r in 'JavaScript'
---
 cmd/geth/consolecmd.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go
index 7eca4d59f6..2500a969cf 100644
--- a/cmd/geth/consolecmd.go
+++ b/cmd/geth/consolecmd.go
@@ -43,7 +43,7 @@ var (
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`,
+See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`,
 	}
 
 	attachCommand = cli.Command{
@@ -56,7 +56,7 @@ See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`,
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.
+See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.
 This command allows to open a console on a running geth node.`,
 	}
 
@@ -69,7 +69,7 @@ This command allows to open a console on a running geth node.`,
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The JavaScript VM exposes a node admin interface as well as the Ðapp
-JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console`,
+JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console`,
 	}
 )
 

From f83237573f0922dfc9fef17f79ccd06305ab6d16 Mon Sep 17 00:00:00 2001
From: Martin Holst Swende 
Date: Mon, 26 Feb 2018 10:53:10 +0100
Subject: [PATCH 090/110] core: make current*Block atomic, and accessor
 functions mutex-free (#16171)

* core: make current*Block atomic, and accessor functions mutex-free

* core: fix review concerns

* core: fix error in atomic assignment

* core/light: implement atomic getter/setter for headerchain
---
 core/blockchain.go  | 112 ++++++++++++++++++++++----------------------
 core/headerchain.go |  39 ++++++++-------
 light/lightchain.go |   6 ---
 3 files changed, 76 insertions(+), 81 deletions(-)

diff --git a/core/blockchain.go b/core/blockchain.go
index 53fe7ee2eb..6006e66740 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -107,8 +107,8 @@ type BlockChain struct {
 	procmu  sync.RWMutex // block processor lock
 
 	checkpoint       int          // checkpoint counts towards the new checkpoint
-	currentBlock     *types.Block // Current head of the block chain
-	currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!)
+	currentBlock     atomic.Value // Current head of the block chain
+	currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!)
 
 	stateCache   state.Database // State database to reuse between imports (contains state cache)
 	bodyCache    *lru.Cache     // Cache for the most recent block bodies
@@ -224,10 +224,10 @@ func (bc *BlockChain) loadLastState() error {
 		}
 	}
 	// Everything seems to be fine, set as the head block
-	bc.currentBlock = currentBlock
+	bc.currentBlock.Store(currentBlock)
 
 	// Restore the last known head header
-	currentHeader := bc.currentBlock.Header()
+	currentHeader := currentBlock.Header()
 	if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) {
 		if header := bc.GetHeaderByHash(head); header != nil {
 			currentHeader = header
@@ -236,21 +236,23 @@ func (bc *BlockChain) loadLastState() error {
 	bc.hc.SetCurrentHeader(currentHeader)
 
 	// Restore the last known head fast block
-	bc.currentFastBlock = bc.currentBlock
+	bc.currentFastBlock.Store(currentBlock)
 	if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) {
 		if block := bc.GetBlockByHash(head); block != nil {
-			bc.currentFastBlock = block
+			bc.currentFastBlock.Store(block)
 		}
 	}
 
 	// Issue a status log for the user
+	currentFastBlock := bc.CurrentFastBlock()
+
 	headerTd := bc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64())
-	blockTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
-	fastTd := bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64())
+	blockTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
+	fastTd := bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64())
 
 	log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd)
-	log.Info("Loaded most recent local full block", "number", bc.currentBlock.Number(), "hash", bc.currentBlock.Hash(), "td", blockTd)
-	log.Info("Loaded most recent local fast block", "number", bc.currentFastBlock.Number(), "hash", bc.currentFastBlock.Hash(), "td", fastTd)
+	log.Info("Loaded most recent local full block", "number", currentBlock.Number(), "hash", currentBlock.Hash(), "td", blockTd)
+	log.Info("Loaded most recent local fast block", "number", currentFastBlock.Number(), "hash", currentFastBlock.Hash(), "td", fastTd)
 
 	return nil
 }
@@ -279,30 +281,32 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	bc.futureBlocks.Purge()
 
 	// Rewind the block chain, ensuring we don't end up with a stateless head block
-	if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() {
-		bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())
+	if currentBlock := bc.CurrentBlock(); currentBlock != nil && currentHeader.Number.Uint64() < currentBlock.NumberU64() {
+		bc.currentBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()))
 	}
-	if bc.currentBlock != nil {
-		if _, err := state.New(bc.currentBlock.Root(), bc.stateCache); err != nil {
+	if currentBlock := bc.CurrentBlock(); currentBlock != nil {
+		if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
 			// Rewound state missing, rolled back to before pivot, reset to genesis
-			bc.currentBlock = nil
+			bc.currentBlock.Store(bc.genesisBlock)
 		}
 	}
 	// Rewind the fast block in a simpleton way to the target head
-	if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() {
-		bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())
+	if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock != nil && currentHeader.Number.Uint64() < currentFastBlock.NumberU64() {
+		bc.currentFastBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()))
 	}
 	// If either blocks reached nil, reset to the genesis state
-	if bc.currentBlock == nil {
-		bc.currentBlock = bc.genesisBlock
+	if currentBlock := bc.CurrentBlock(); currentBlock == nil {
+		bc.currentBlock.Store(bc.genesisBlock)
 	}
-	if bc.currentFastBlock == nil {
-		bc.currentFastBlock = bc.genesisBlock
+	if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock == nil {
+		bc.currentFastBlock.Store(bc.genesisBlock)
 	}
-	if err := WriteHeadBlockHash(bc.db, bc.currentBlock.Hash()); err != nil {
+	currentBlock := bc.CurrentBlock()
+	currentFastBlock := bc.CurrentFastBlock()
+	if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil {
 		log.Crit("Failed to reset head full block", "err", err)
 	}
-	if err := WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash()); err != nil {
+	if err := WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()); err != nil {
 		log.Crit("Failed to reset head fast block", "err", err)
 	}
 	return bc.loadLastState()
@@ -321,7 +325,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	}
 	// If all checks out, manually set the head block
 	bc.mu.Lock()
-	bc.currentBlock = block
+	bc.currentBlock.Store(block)
 	bc.mu.Unlock()
 
 	log.Info("Committed new head block", "number", block.Number(), "hash", hash)
@@ -330,28 +334,19 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 
 // GasLimit returns the gas limit of the current HEAD block.
 func (bc *BlockChain) GasLimit() uint64 {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentBlock.GasLimit()
+	return bc.CurrentBlock().GasLimit()
 }
 
 // CurrentBlock retrieves the current head block of the canonical chain. The
 // block is retrieved from the blockchain's internal cache.
 func (bc *BlockChain) CurrentBlock() *types.Block {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentBlock
+	return bc.currentBlock.Load().(*types.Block)
 }
 
 // CurrentFastBlock retrieves the current fast-sync head block of the canonical
 // chain. The block is retrieved from the blockchain's internal cache.
 func (bc *BlockChain) CurrentFastBlock() *types.Block {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentFastBlock
+	return bc.currentFastBlock.Load().(*types.Block)
 }
 
 // SetProcessor sets the processor required for making state modifications.
@@ -416,10 +411,10 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	}
 	bc.genesisBlock = genesis
 	bc.insert(bc.genesisBlock)
-	bc.currentBlock = bc.genesisBlock
+	bc.currentBlock.Store(bc.genesisBlock)
 	bc.hc.SetGenesis(bc.genesisBlock.Header())
 	bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
-	bc.currentFastBlock = bc.genesisBlock
+	bc.currentFastBlock.Store(bc.genesisBlock)
 
 	return nil
 }
@@ -444,7 +439,7 @@ func (bc *BlockChain) repair(head **types.Block) error {
 
 // Export writes the active chain to the given writer.
 func (bc *BlockChain) Export(w io.Writer) error {
-	return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64())
+	return bc.ExportN(w, uint64(0), bc.CurrentBlock().NumberU64())
 }
 
 // ExportN writes a subset of the active chain to the given writer.
@@ -488,7 +483,7 @@ func (bc *BlockChain) insert(block *types.Block) {
 	if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil {
 		log.Crit("Failed to insert head block hash", "err", err)
 	}
-	bc.currentBlock = block
+	bc.currentBlock.Store(block)
 
 	// If the block is better than our head or is on a different chain, force update heads
 	if updateHeads {
@@ -497,7 +492,7 @@ func (bc *BlockChain) insert(block *types.Block) {
 		if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil {
 			log.Crit("Failed to insert head fast block hash", "err", err)
 		}
-		bc.currentFastBlock = block
+		bc.currentFastBlock.Store(block)
 	}
 }
 
@@ -714,13 +709,15 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
 		if currentHeader.Hash() == hash {
 			bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1))
 		}
-		if bc.currentFastBlock.Hash() == hash {
-			bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1)
-			WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash())
+		if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash {
+			newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1)
+			bc.currentFastBlock.Store(newFastBlock)
+			WriteHeadFastBlockHash(bc.db, newFastBlock.Hash())
 		}
-		if bc.currentBlock.Hash() == hash {
-			bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1)
-			WriteHeadBlockHash(bc.db, bc.currentBlock.Hash())
+		if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash {
+			newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1)
+			bc.currentBlock.Store(newBlock)
+			WriteHeadBlockHash(bc.db, newBlock.Hash())
 		}
 	}
 }
@@ -829,11 +826,12 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	bc.mu.Lock()
 	head := blockChain[len(blockChain)-1]
 	if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case
-		if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 {
+		currentFastBlock := bc.CurrentFastBlock()
+		if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 {
 			if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil {
 				log.Crit("Failed to update head fast block hash", "err", err)
 			}
-			bc.currentFastBlock = head
+			bc.currentFastBlock.Store(head)
 		}
 	}
 	bc.mu.Unlock()
@@ -880,7 +878,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	bc.mu.Lock()
 	defer bc.mu.Unlock()
 
-	localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
+	currentBlock := bc.CurrentBlock()
+	localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
 	externTd := new(big.Int).Add(block.Difficulty(), ptd)
 
 	// Irrelevant of the canonical status, write the block itself to the database
@@ -955,14 +954,15 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	// Second clause in the if statement reduces the vulnerability to selfish mining.
 	// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
 	reorg := externTd.Cmp(localTd) > 0
+	currentBlock = bc.CurrentBlock()
 	if !reorg && externTd.Cmp(localTd) == 0 {
 		// Split same-difficulty blocks by number, then at random
-		reorg = block.NumberU64() < bc.currentBlock.NumberU64() || (block.NumberU64() == bc.currentBlock.NumberU64() && mrand.Float64() < 0.5)
+		reorg = block.NumberU64() < currentBlock.NumberU64() || (block.NumberU64() == currentBlock.NumberU64() && mrand.Float64() < 0.5)
 	}
 	if reorg {
 		// Reorganise the chain if the parent is not the head block
-		if block.ParentHash() != bc.currentBlock.Hash() {
-			if err := bc.reorg(bc.currentBlock, block); err != nil {
+		if block.ParentHash() != currentBlock.Hash() {
+			if err := bc.reorg(currentBlock, block); err != nil {
 				return NonStatTy, err
 			}
 		}
@@ -1091,7 +1091,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		case err == consensus.ErrPrunedAncestor:
 			// Block competing with the canonical chain, store in the db, but don't process
 			// until the competitor TD goes above the canonical TD
-			localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
+			currentBlock := bc.CurrentBlock()
+			localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
 			externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
 			if localTd.Cmp(externTd) > 0 {
 				if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
@@ -1480,9 +1481,6 @@ func (bc *BlockChain) writeHeader(header *types.Header) error {
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
 func (bc *BlockChain) CurrentHeader() *types.Header {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
 	return bc.hc.CurrentHeader()
 }
 
diff --git a/core/headerchain.go b/core/headerchain.go
index 0e5215293d..73cd5d2c41 100644
--- a/core/headerchain.go
+++ b/core/headerchain.go
@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/hashicorp/golang-lru"
+	"sync/atomic"
 )
 
 const (
@@ -51,8 +52,8 @@ type HeaderChain struct {
 	chainDb       ethdb.Database
 	genesisHeader *types.Header
 
-	currentHeader     *types.Header // Current head of the header chain (may be above the block chain!)
-	currentHeaderHash common.Hash   // Hash of the current head of the header chain (prevent recomputing all the time)
+	currentHeader     atomic.Value // Current head of the header chain (may be above the block chain!)
+	currentHeaderHash common.Hash  // Hash of the current head of the header chain (prevent recomputing all the time)
 
 	headerCache *lru.Cache // Cache for the most recent block headers
 	tdCache     *lru.Cache // Cache for the most recent block total difficulties
@@ -95,13 +96,13 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c
 		return nil, ErrNoGenesis
 	}
 
-	hc.currentHeader = hc.genesisHeader
+	hc.currentHeader.Store(hc.genesisHeader)
 	if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) {
 		if chead := hc.GetHeaderByHash(head); chead != nil {
-			hc.currentHeader = chead
+			hc.currentHeader.Store(chead)
 		}
 	}
-	hc.currentHeaderHash = hc.currentHeader.Hash()
+	hc.currentHeaderHash = hc.CurrentHeader().Hash()
 
 	return hc, nil
 }
@@ -139,7 +140,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 	if ptd == nil {
 		return NonStatTy, consensus.ErrUnknownAncestor
 	}
-	localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64())
+	localTd := hc.GetTd(hc.currentHeaderHash, hc.CurrentHeader().Number.Uint64())
 	externTd := new(big.Int).Add(header.Difficulty, ptd)
 
 	// Irrelevant of the canonical status, write the td and header to the database
@@ -181,7 +182,8 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 		if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil {
 			log.Crit("Failed to insert head header hash", "err", err)
 		}
-		hc.currentHeaderHash, hc.currentHeader = hash, types.CopyHeader(header)
+		hc.currentHeaderHash = hash
+		hc.currentHeader.Store(types.CopyHeader(header))
 
 		status = CanonStatTy
 	} else {
@@ -383,7 +385,7 @@ func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header {
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
 func (hc *HeaderChain) CurrentHeader() *types.Header {
-	return hc.currentHeader
+	return hc.currentHeader.Load().(*types.Header)
 }
 
 // SetCurrentHeader sets the current head header of the canonical chain.
@@ -391,7 +393,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) {
 	if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil {
 		log.Crit("Failed to insert head header hash", "err", err)
 	}
-	hc.currentHeader = head
+	hc.currentHeader.Store(head)
 	hc.currentHeaderHash = head.Hash()
 }
 
@@ -403,19 +405,20 @@ type DeleteCallback func(common.Hash, uint64)
 // will be deleted and the new one set.
 func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) {
 	height := uint64(0)
-	if hc.currentHeader != nil {
-		height = hc.currentHeader.Number.Uint64()
+
+	if hdr := hc.CurrentHeader(); hdr != nil {
+		height = hdr.Number.Uint64()
 	}
 
-	for hc.currentHeader != nil && hc.currentHeader.Number.Uint64() > head {
-		hash := hc.currentHeader.Hash()
-		num := hc.currentHeader.Number.Uint64()
+	for hdr := hc.CurrentHeader(); hdr != nil && hdr.Number.Uint64() > head; hdr = hc.CurrentHeader() {
+		hash := hdr.Hash()
+		num := hdr.Number.Uint64()
 		if delFn != nil {
 			delFn(hash, num)
 		}
 		DeleteHeader(hc.chainDb, hash, num)
 		DeleteTd(hc.chainDb, hash, num)
-		hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash, hc.currentHeader.Number.Uint64()-1)
+		hc.currentHeader.Store(hc.GetHeader(hdr.ParentHash, hdr.Number.Uint64()-1))
 	}
 	// Roll back the canonical chain numbering
 	for i := height; i > head; i-- {
@@ -426,10 +429,10 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) {
 	hc.tdCache.Purge()
 	hc.numberCache.Purge()
 
-	if hc.currentHeader == nil {
-		hc.currentHeader = hc.genesisHeader
+	if hc.CurrentHeader() == nil {
+		hc.currentHeader.Store(hc.genesisHeader)
 	}
-	hc.currentHeaderHash = hc.currentHeader.Hash()
+	hc.currentHeaderHash = hc.CurrentHeader().Hash()
 
 	if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil {
 		log.Crit("Failed to reset head header hash", "err", err)
diff --git a/light/lightchain.go b/light/lightchain.go
index 181a1c2a62..2784615d35 100644
--- a/light/lightchain.go
+++ b/light/lightchain.go
@@ -171,9 +171,6 @@ func (bc *LightChain) SetHead(head uint64) {
 
 // GasLimit returns the gas limit of the current HEAD block.
 func (self *LightChain) GasLimit() uint64 {
-	self.mu.RLock()
-	defer self.mu.RUnlock()
-
 	return self.hc.CurrentHeader().GasLimit
 }
 
@@ -387,9 +384,6 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int)
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
 func (self *LightChain) CurrentHeader() *types.Header {
-	self.mu.RLock()
-	defer self.mu.RUnlock()
-
 	return self.hc.CurrentHeader()
 }
 

From 6e0667fa06caa15a5333bc4c902315c9d56a3739 Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Mon, 26 Feb 2018 13:58:04 +0100
Subject: [PATCH 091/110] whisper: wnode updated - all messages are saved if
 savedir param is given

---
 cmd/wnode/main.go | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go
index 971b1c0ab8..8694219c57 100644
--- a/cmd/wnode/main.go
+++ b/cmd/wnode/main.go
@@ -76,14 +76,14 @@ var (
 
 // cmd arguments
 var (
-	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
-	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: does not initiate connection to peers, just waits for incoming connections")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forwards messages, neither encrypts nor decrypts messages")
 	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
 	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
 	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
 	generateKey    = flag.Bool("generatekey", false, "generate and show the private key")
 	fileExMode     = flag.Bool("fileexchange", false, "file exchange mode")
-	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics")
+	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics (password, etc.)")
 	echoMode       = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics")
 
 	argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level")
@@ -99,7 +99,7 @@ var (
 	argIDFile  = flag.String("idfile", "", "file name with node id (private key)")
 	argEnode   = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
 	argTopic   = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
-	argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files")
+	argSaveDir = flag.String("savedir", "", "directory where all incoming messages will be saved as files")
 )
 
 func main() {
@@ -548,20 +548,16 @@ func messageLoop() {
 	for {
 		select {
 		case <-ticker.C:
-			messages := sf.Retrieve()
+			m1 := sf.Retrieve()
+			m2 := af.Retrieve()
+			messages := append(m1, m2...)
 			for _, msg := range messages {
-				if *fileExMode || len(msg.Payload) > 2048 {
+				// NB: it is possible that *fileExMode == false && len(*argSaveDir) > 0
+				if len(*argSaveDir) > 0 {
 					writeMessageToFile(*argSaveDir, msg)
-				} else {
-					printMessageInfo(msg)
 				}
-			}
 
-			messages = af.Retrieve()
-			for _, msg := range messages {
-				if *fileExMode || len(msg.Payload) > 2048 {
-					writeMessageToFile(*argSaveDir, msg)
-				} else {
+				if !*fileExMode && len(msg.Payload) <= 2048 {
 					printMessageInfo(msg)
 				}
 			}

From 61c9730b2de2719059f0fb2ce8f965c0cde34527 Mon Sep 17 00:00:00 2001
From: JU HYEONG PARK 
Date: Tue, 27 Feb 2018 01:22:46 +0900
Subject: [PATCH 092/110] p2p: fix doEncHandshake documentation (#16184)

---
 p2p/rlpx.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/p2p/rlpx.go b/p2p/rlpx.go
index e65a0b6047..1889edac99 100644
--- a/p2p/rlpx.go
+++ b/p2p/rlpx.go
@@ -121,10 +121,6 @@ func (t *rlpx) close(err error) {
 	t.fd.Close()
 }
 
-// doEncHandshake runs the protocol handshake using authenticated
-// messages. the protocol handshake is the first authenticated message
-// and also verifies whether the encryption handshake 'worked' and the
-// remote side actually provided the right public key.
 func (t *rlpx) doProtoHandshake(our *protoHandshake) (their *protoHandshake, err error) {
 	// Writing our handshake happens concurrently, we prefer
 	// returning the handshake read error. If the remote side
@@ -175,6 +171,10 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake,
 	return &hs, nil
 }
 
+// doEncHandshake runs the protocol handshake using authenticated
+// messages. the protocol handshake is the first authenticated message
+// and also verifies whether the encryption handshake 'worked' and the
+// remote side actually provided the right public key.
 func (t *rlpx) doEncHandshake(prv *ecdsa.PrivateKey, dial *discover.Node) (discover.NodeID, error) {
 	var (
 		sec secrets

From f4e676cccdeaf781a449410482e75267672db61b Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Mon, 26 Feb 2018 19:26:36 +0100
Subject: [PATCH 093/110] whipser: comments updated

---
 cmd/wnode/main.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go
index 8694219c57..0f86adb81c 100644
--- a/cmd/wnode/main.go
+++ b/cmd/wnode/main.go
@@ -76,8 +76,8 @@ var (
 
 // cmd arguments
 var (
-	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: does not initiate connection to peers, just waits for incoming connections")
-	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forwards messages, neither encrypts nor decrypts messages")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't initiate connection to peers, just wait for incoming connections")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither encrypt nor decrypt messages")
 	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
 	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
 	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
@@ -552,7 +552,9 @@ func messageLoop() {
 			m2 := af.Retrieve()
 			messages := append(m1, m2...)
 			for _, msg := range messages {
-				// NB: it is possible that *fileExMode == false && len(*argSaveDir) > 0
+				// All messages are saved upon specifying argSaveDir.
+				// fileExMode only specifies how messages are displayed on the console after they are saved.
+				// if fileExMode == true, only the hashes are displayed, since messages might be too big.
 				if len(*argSaveDir) > 0 {
 					writeMessageToFile(*argSaveDir, msg)
 				}

From 2e9c8fd4fbd5d0de0ced03961d268c7492917860 Mon Sep 17 00:00:00 2001
From: Andrey Petrov 
Date: Tue, 27 Feb 2018 05:52:59 -0500
Subject: [PATCH 094/110] eth, les: allow exceeding maxPeers for trusted peers
 (#16189)

Fixes #3326, #14472
---
 eth/handler.go | 3 ++-
 les/handler.go | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/eth/handler.go b/eth/handler.go
index c2426544f6..3fae0cd00d 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -249,7 +249,8 @@ func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *p
 // handle is the callback invoked to manage the life cycle of an eth peer. When
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
-	if pm.peers.Len() >= pm.maxPeers {
+	// Ignore maxPeers if this is a trusted peer
+	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 	p.Log().Debug("Ethereum peer connected", "name", p.Name())
diff --git a/les/handler.go b/les/handler.go
index 864abe605a..9627f392be 100644
--- a/les/handler.go
+++ b/les/handler.go
@@ -260,7 +260,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea
 // handle is the callback invoked to manage the life cycle of a les peer. When
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
-	if pm.peers.Len() >= pm.maxPeers {
+	// Ignore maxPeers if this is a trusted peer
+	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 

From c41f1a3e23e603ffc01d787d7509adf1712633b3 Mon Sep 17 00:00:00 2001
From: Saulius Grigaitis 
Date: Tue, 27 Feb 2018 12:56:51 +0200
Subject: [PATCH 095/110] puppeth: fix Parity Chain Spec parameter
 GasLimitBoundDivision (#16188)

---
 cmd/puppeth/genesis.go | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go
index f747f47395..1974a94aa2 100644
--- a/cmd/puppeth/genesis.go
+++ b/cmd/puppeth/genesis.go
@@ -168,19 +168,18 @@ type parityChainSpec struct {
 	Engine struct {
 		Ethash struct {
 			Params struct {
-				MinimumDifficulty      *hexutil.Big   `json:"minimumDifficulty"`
-				DifficultyBoundDivisor *hexutil.Big   `json:"difficultyBoundDivisor"`
-				GasLimitBoundDivisor   hexutil.Uint64 `json:"gasLimitBoundDivisor"`
-				DurationLimit          *hexutil.Big   `json:"durationLimit"`
-				BlockReward            *hexutil.Big   `json:"blockReward"`
-				HomesteadTransition    uint64         `json:"homesteadTransition"`
-				EIP150Transition       uint64         `json:"eip150Transition"`
-				EIP160Transition       uint64         `json:"eip160Transition"`
-				EIP161abcTransition    uint64         `json:"eip161abcTransition"`
-				EIP161dTransition      uint64         `json:"eip161dTransition"`
-				EIP649Reward           *hexutil.Big   `json:"eip649Reward"`
-				EIP100bTransition      uint64         `json:"eip100bTransition"`
-				EIP649Transition       uint64         `json:"eip649Transition"`
+				MinimumDifficulty      *hexutil.Big `json:"minimumDifficulty"`
+				DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"`
+				DurationLimit          *hexutil.Big `json:"durationLimit"`
+				BlockReward            *hexutil.Big `json:"blockReward"`
+				HomesteadTransition    uint64       `json:"homesteadTransition"`
+				EIP150Transition       uint64       `json:"eip150Transition"`
+				EIP160Transition       uint64       `json:"eip160Transition"`
+				EIP161abcTransition    uint64       `json:"eip161abcTransition"`
+				EIP161dTransition      uint64       `json:"eip161dTransition"`
+				EIP649Reward           *hexutil.Big `json:"eip649Reward"`
+				EIP100bTransition      uint64       `json:"eip100bTransition"`
+				EIP649Transition       uint64       `json:"eip649Transition"`
 			} `json:"params"`
 		} `json:"Ethash"`
 	} `json:"engine"`
@@ -188,6 +187,7 @@ type parityChainSpec struct {
 	Params struct {
 		MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"`
 		MinGasLimit          hexutil.Uint64 `json:"minGasLimit"`
+		GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"`
 		NetworkID            hexutil.Uint64 `json:"networkID"`
 		MaxCodeSize          uint64         `json:"maxCodeSize"`
 		EIP155Transition     uint64         `json:"eip155Transition"`
@@ -270,7 +270,6 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	}
 	spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
 	spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
-	spec.Engine.Ethash.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
 	spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
 	spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
 	spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64()
@@ -284,6 +283,7 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
+	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64())
 	spec.Params.MaxCodeSize = params.MaxCodeSize
 	spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64()

From dd389e595f3fa1c9644478b4c21a5127e916ec04 Mon Sep 17 00:00:00 2001
From: Elad Nachmias 
Date: Tue, 27 Feb 2018 12:04:47 +0100
Subject: [PATCH 096/110] eth: added travis build badge (#16117)

* eth: added travis build status for master branch

* README: fix travis badge order, link to CI
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index aad14ed7d8..3d0d4d35d4 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,7 @@ Official golang implementation of the Ethereum protocol.
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/ethereum/go-ethereum)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
+[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 Automated builds are available for stable releases and the unstable master branch.

From 18bb3da55e4d2f6b5e9142b7f5d5bd76f2fb78b0 Mon Sep 17 00:00:00 2001
From: Anton Evangelatov 
Date: Tue, 27 Feb 2018 14:30:07 +0100
Subject: [PATCH 097/110] node: fill StandardCounters as part of
 debugapi/metrics (#16054)

---
 node/api.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/node/api.go b/node/api.go
index 992a7c4166..a3b8bc0bb5 100644
--- a/node/api.go
+++ b/node/api.go
@@ -308,6 +308,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) {
 		// Fill the counter with the metric details, formatting if requested
 		if raw {
 			switch metric := metric.(type) {
+			case metrics.Counter:
+				root[name] = map[string]interface{}{
+					"Overall": float64(metric.Count()),
+				}
+
 			case metrics.Meter:
 				root[name] = map[string]interface{}{
 					"AvgRate01Min": metric.Rate1(),
@@ -338,6 +343,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) {
 			}
 		} else {
 			switch metric := metric.(type) {
+			case metrics.Counter:
+				root[name] = map[string]interface{}{
+					"Overall": float64(metric.Count()),
+				}
+
 			case metrics.Meter:
 				root[name] = map[string]interface{}{
 					"Avg01Min": format(metric.Rate1()*60, metric.Rate1()),

From b574b5776695eb30e034fd8c7a468b3f03d4c6b9 Mon Sep 17 00:00:00 2001
From: Elad Nachmias 
Date: Tue, 27 Feb 2018 14:32:38 +0100
Subject: [PATCH 098/110] swarm: give correct error on 0x hash prefix (#16195)

- added a case error struct that contains information about certain error cases
in which we would like to output more information to the client
- added a validation method that iterates and adds the information that is
stored in the error cases
---
 .gitignore                        |  3 +++
 swarm/api/http/error.go           | 43 +++++++++++++++++++++++++++----
 swarm/api/http/error_templates.go | 11 ++++++++
 swarm/api/http/error_test.go      | 40 ++++++++++++++++++++++++----
 swarm/api/http/server.go          | 22 ++++++++--------
 5 files changed, 98 insertions(+), 21 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0763d8492c..b8d2929016 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,6 +34,9 @@ profile.cov
 # IdeaIDE
 .idea
 
+# VS Code
+.vscode
+
 # dashboard
 /dashboard/assets/flow-typed
 /dashboard/assets/node_modules
diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go
index 831cf23fe2..9a65412cf9 100644
--- a/swarm/api/http/error.go
+++ b/swarm/api/http/error.go
@@ -35,6 +35,7 @@ import (
 
 //templateMap holds a mapping of an HTTP error code to a template
 var templateMap map[int]*template.Template
+var caseErrors []CaseError
 
 //metrics variables
 var (
@@ -51,6 +52,13 @@ type ErrorParams struct {
 	Details   template.HTML
 }
 
+//a custom error case struct that would be used to store validators and
+//additional error info to display with client responses.
+type CaseError struct {
+	Validator func(*Request) bool
+	Msg       func(*Request) string
+}
+
 //we init the error handling right on boot time, so lookup and http response is fast
 func init() {
 	initErrHandling()
@@ -74,6 +82,29 @@ func initErrHandling() {
 		//assign formatted HTML to the code
 		templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname))
 	}
+
+	caseErrors = []CaseError{
+		{
+			Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") },
+			Msg: func(r *Request) string {
+				uriCopy := r.uri
+				uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x")
+				return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.

+			Please click here if your browser does not redirect you.`, "/"+uriCopy.String())
+			},
+		}}
+}
+
+//ValidateCaseErrors is a method that process the request object through certain validators
+//that assert if certain conditions are met for further information to log as an error
+func ValidateCaseErrors(r *Request) string {
+	for _, err := range caseErrors {
+		if err.Validator(r) {
+			return err.Msg(r)
+		}
+	}
+
+	return ""
 }
 
 //ShowMultipeChoices is used when a user requests a resource in a manifest which results
@@ -82,10 +113,10 @@ func initErrHandling() {
 //For example, if the user requests bzz://read and that manifest contains entries
 //"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
 //This only applies if the manifest has no default entry
-func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
+func ShowMultipleChoices(w http.ResponseWriter, r *Request, list api.ManifestList) {
 	msg := ""
 	if list.Entries == nil {
-		ShowError(w, r, "Internal Server Error", http.StatusInternalServerError)
+		ShowError(w, r, "Could not resolve", http.StatusInternalServerError)
 		return
 	}
 	//make links relative
@@ -102,7 +133,7 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife
 		//create clickable link for each entry
 		msg += "" + e.Path + "
"
 	}
-	respond(w, r, &ErrorParams{
+	respond(w, &r.Request, &ErrorParams{
 		Code:      http.StatusMultipleChoices,
 		Details:   template.HTML(msg),
 		Timestamp: time.Now().Format(time.RFC1123),
@@ -115,13 +146,15 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife
 //The function just takes a string message which will be displayed in the error page.
 //The code is used to evaluate which template will be displayed
 //(and return the correct HTTP status code)
-func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) {
+func ShowError(w http.ResponseWriter, r *Request, msg string, code int) {
+	additionalMessage := ValidateCaseErrors(r)
 	if code == http.StatusInternalServerError {
 		log.Error(msg)
 	}
-	respond(w, r, &ErrorParams{
+	respond(w, &r.Request, &ErrorParams{
 		Code:      code,
 		Msg:       msg,
+		Details:   template.HTML(additionalMessage),
 		Timestamp: time.Now().Format(time.RFC1123),
 		template:  getTemplate(code),
 	})
diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go
index 0457cb8a70..cc9b996ba4 100644
--- a/swarm/api/http/error_templates.go
+++ b/swarm/api/http/error_templates.go
@@ -168,6 +168,11 @@ func GetGenericErrorPage() string {
                   {{.Msg}}
                 
               
+              
+                
+                  {{.Details}}
+                
+              
 
               
                 
@@ -342,6 +347,12 @@ func GetNotFoundErrorPage() string {
                   {{.Msg}}
                 
               
+              
+                
+                  {{.Details}}
+                
+              
+
 
               
                 
diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go
index c2c8b908b8..dc545722e1 100644
--- a/swarm/api/http/error_test.go
+++ b/swarm/api/http/error_test.go
@@ -18,12 +18,13 @@ package http_test
 
 import (
 	"encoding/json"
-	"golang.org/x/net/html"
 	"io/ioutil"
 	"net/http"
 	"strings"
 	"testing"
 
+	"golang.org/x/net/html"
+
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -96,8 +97,37 @@ func Test500Page(t *testing.T) {
 	defer resp.Body.Close()
 	respbody, err = ioutil.ReadAll(resp.Body)
 
-	if resp.StatusCode != 500 || !strings.Contains(string(respbody), "500") {
-		t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode)
+	if resp.StatusCode != 404 {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
+	}
+
+	_, err = html.Parse(strings.NewReader(string(respbody)))
+	if err != nil {
+		t.Fatalf("HTML validation failed for error page returned!")
+	}
+}
+func Test500PageWith0xHashPrefix(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	var resp *http.Response
+	var respbody []byte
+
+	url := srv.URL + "/bzz:/0xthisShouldFailWith500CodeAndAHelpfulMessage"
+	resp, err := http.Get(url)
+
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+	defer resp.Body.Close()
+	respbody, err = ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != 404 {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
+	}
+
+	if !strings.Contains(string(respbody), "The requested hash seems to be prefixed with") {
+		t.Fatalf("Did not receive the expected error message")
 	}
 
 	_, err = html.Parse(strings.NewReader(string(respbody)))
@@ -127,8 +157,8 @@ func TestJsonResponse(t *testing.T) {
 	defer resp.Body.Close()
 	respbody, err = ioutil.ReadAll(resp.Body)
 
-	if resp.StatusCode != 500 {
-		t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode)
+	if resp.StatusCode != 404 {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
 	}
 
 	if !isJSON(string(respbody)) {
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index df90fd04f9..b8e7436cf0 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -336,7 +336,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
 	if err != nil {
 		getFail.Inc(1)
-		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
 		return
 	}
 
@@ -421,7 +421,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
 	if err != nil {
 		getFilesFail.Inc(1)
-		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
 		return
 	}
 
@@ -494,7 +494,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
 	if err != nil {
 		getListFail.Inc(1)
-		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
 		return
 	}
 
@@ -598,7 +598,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
 	if err != nil {
 		getFileFail.Inc(1)
-		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
 		return
 	}
 
@@ -628,7 +628,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 
 		s.logDebug(fmt.Sprintf("Multiple choices! -->  %v", list))
 		//show a nice page links to available entries
-		ShowMultipleChoices(w, &r.Request, list)
+		ShowMultipleChoices(w, r, list)
 		return
 	}
 
@@ -693,7 +693,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		//   strictly a traditional PUT request which replaces content
 		//   at a URI, and POST is more ubiquitous)
 		if uri.Raw() || uri.DeprecatedRaw() {
-			ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
+			ShowError(w, req, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
 		} else {
 			s.HandlePostFiles(w, req)
@@ -701,7 +701,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	case "DELETE":
 		if uri.Raw() || uri.DeprecatedRaw() {
-			ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
+			ShowError(w, req, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
 			return
 		}
 		s.HandleDelete(w, req)
@@ -725,7 +725,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		s.HandleGetFile(w, req)
 
 	default:
-		ShowError(w, r, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed)
+		ShowError(w, req, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed)
 
 	}
 }
@@ -757,13 +757,13 @@ func (s *Server) logError(format string, v ...interface{}) {
 }
 
 func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
-	ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest)
+	ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Request.Method, r.uri, reason), http.StatusBadRequest)
 }
 
 func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
-	ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
+	ShowError(w, r, fmt.Sprintf("Error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusInternalServerError)
 }
 
 func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) {
-	ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound)
+	ShowError(w, r, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusNotFound)
 }

From dadf4d53abd8be13f59fb09820712d74ce8f89a7 Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Tue, 27 Feb 2018 15:45:00 +0100
Subject: [PATCH 099/110] whisper: mailserver no longer supports the signature
 vaidation

---
 whisper/mailserver/mailserver.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go
index 6555fd5c0b..a889c2bd23 100644
--- a/whisper/mailserver/mailserver.go
+++ b/whisper/mailserver/mailserver.go
@@ -17,7 +17,6 @@
 package mailserver
 
 import (
-	"bytes"
 	"encoding/binary"
 	"fmt"
 
@@ -175,7 +174,10 @@ func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope)
 	if len(src)-len(peerID) == 1 {
 		src = src[1:]
 	}
-	if !bytes.Equal(peerID, src) {
+
+	// if you want to check the signature, you can do it here. e.g.:
+	// if !bytes.Equal(peerID, src) {
+	if src == nil {
 		log.Warn(fmt.Sprintf("Wrong signature of p2p request"))
 		return false, 0, 0, topic
 	}

From 5e30a5f66e126d35c00276abffe32d99182a1bc7 Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Tue, 27 Feb 2018 15:52:10 +0100
Subject: [PATCH 100/110] whisper: test fixed

---
 whisper/mailserver/server_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go
index c8e0a553a0..aeb317e597 100644
--- a/whisper/mailserver/server_test.go
+++ b/whisper/mailserver/server_test.go
@@ -167,7 +167,8 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *
 
 	src[0]++
 	ok, lower, upper, topic = server.validateRequest(src, request)
-	if ok {
+	if !ok {
+		// request should be valid regardless of signature
 		t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper)
 	}
 }

From 17b0e226d3d6b3829a82ee1c142bd125cc9a0109 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= 
Date: Tue, 27 Feb 2018 18:25:56 +0200
Subject: [PATCH 101/110] travis, build, consensus: drop support for Go 1.7

---
 .travis.yml                              | 11 -----
 build/ci.go                              |  6 +--
 consensus/ethash/algorithm.go            | 43 ++++++++++++++++
 consensus/ethash/algorithm_go1.7.go      | 47 ------------------
 consensus/ethash/algorithm_go1.8.go      | 63 ------------------------
 consensus/ethash/algorithm_go1.8_test.go | 37 --------------
 consensus/ethash/algorithm_test.go       | 16 ++++++
 consensus/ethash/consensus.go            | 10 +---
 8 files changed, 64 insertions(+), 169 deletions(-)
 delete mode 100644 consensus/ethash/algorithm_go1.7.go
 delete mode 100644 consensus/ethash/algorithm_go1.8.go
 delete mode 100644 consensus/ethash/algorithm_go1.8_test.go

diff --git a/.travis.yml b/.travis.yml
index 3941fa785b..a76a78954d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum
 sudo: false
 matrix:
   include:
-    - os: linux
-      dist: trusty
-      sudo: required
-      go: 1.7.x
-      script:
-        - sudo modprobe fuse
-        - sudo chmod 666 /dev/fuse
-        - sudo chown root:$USER /etc/fuse.conf
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage
-
     - os: linux
       dist: trusty
       sudo: required
diff --git a/build/ci.go b/build/ci.go
index 544483c42c..24b58c1ae4 100644
--- a/build/ci.go
+++ b/build/ci.go
@@ -182,13 +182,13 @@ func doInstall(cmdline []string) {
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
 	if !strings.Contains(runtime.Version(), "devel") {
-		// Figure out the minor version number since we can't textually compare (1.10 < 1.7)
+		// Figure out the minor version number since we can't textually compare (1.10 < 1.8)
 		var minor int
 		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
 
-		if minor < 7 {
+		if minor < 8 {
 			log.Println("You have Go version", runtime.Version())
-			log.Println("go-ethereum requires at least Go version 1.7 and cannot")
+			log.Println("go-ethereum requires at least Go version 1.8 and cannot")
 			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
 			os.Exit(1)
 		}
diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go
index 10767bb312..905a7b1ea7 100644
--- a/consensus/ethash/algorithm.go
+++ b/consensus/ethash/algorithm.go
@@ -19,6 +19,7 @@ package ethash
 import (
 	"encoding/binary"
 	"hash"
+	"math/big"
 	"reflect"
 	"runtime"
 	"sync"
@@ -47,6 +48,48 @@ const (
 	loopAccesses       = 64      // Number of accesses in hashimoto loop
 )
 
+// cacheSize returns the size of the ethash verification cache that belongs to a certain
+// block number.
+func cacheSize(block uint64) uint64 {
+	epoch := int(block / epochLength)
+	if epoch < maxEpoch {
+		return cacheSizes[epoch]
+	}
+	return calcCacheSize(epoch)
+}
+
+// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcCacheSize(epoch int) uint64 {
+	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
+	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
+		size -= 2 * hashBytes
+	}
+	return size
+}
+
+// datasetSize returns the size of the ethash mining dataset that belongs to a certain
+// block number.
+func datasetSize(block uint64) uint64 {
+	epoch := int(block / epochLength)
+	if epoch < maxEpoch {
+		return datasetSizes[epoch]
+	}
+	return calcDatasetSize(epoch)
+}
+
+// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcDatasetSize(epoch int) uint64 {
+	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
+	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
+		size -= 2 * mixBytes
+	}
+	return size
+}
+
 // hasher is a repetitive hasher allowing the same hash data structures to be
 // reused between hash runs instead of requiring new ones to be created.
 type hasher func(dest []byte, data []byte)
diff --git a/consensus/ethash/algorithm_go1.7.go b/consensus/ethash/algorithm_go1.7.go
deleted file mode 100644
index c7f7f48e41..0000000000
--- a/consensus/ethash/algorithm_go1.7.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// +build !go1.8
-
-package ethash
-
-// cacheSize calculates and returns the size of the ethash verification cache that
-// belongs to a certain block number. The cache size grows linearly, however, we
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
-func cacheSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return cacheSizes[epoch]
-	}
-	// We don't have a way to verify primes fast before Go 1.8
-	panic("fast prime testing unsupported in Go < 1.8")
-}
-
-// datasetSize calculates and returns the size of the ethash mining dataset that
-// belongs to a certain block number. The dataset size grows linearly, however, we
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
-func datasetSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return datasetSizes[epoch]
-	}
-	// We don't have a way to verify primes fast before Go 1.8
-	panic("fast prime testing unsupported in Go < 1.8")
-}
diff --git a/consensus/ethash/algorithm_go1.8.go b/consensus/ethash/algorithm_go1.8.go
deleted file mode 100644
index 975fdffe51..0000000000
--- a/consensus/ethash/algorithm_go1.8.go
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// +build go1.8
-
-package ethash
-
-import "math/big"
-
-// cacheSize returns the size of the ethash verification cache that belongs to a certain
-// block number.
-func cacheSize(block uint64) uint64 {
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return cacheSizes[epoch]
-	}
-	return calcCacheSize(epoch)
-}
-
-// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
-// however, we always take the highest prime below the linearly growing threshold in order
-// to reduce the risk of accidental regularities leading to cyclic behavior.
-func calcCacheSize(epoch int) uint64 {
-	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
-	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
-		size -= 2 * hashBytes
-	}
-	return size
-}
-
-// datasetSize returns the size of the ethash mining dataset that belongs to a certain
-// block number.
-func datasetSize(block uint64) uint64 {
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return datasetSizes[epoch]
-	}
-	return calcDatasetSize(epoch)
-}
-
-// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
-// however, we always take the highest prime below the linearly growing threshold in order
-// to reduce the risk of accidental regularities leading to cyclic behavior.
-func calcDatasetSize(epoch int) uint64 {
-	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
-	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
-		size -= 2 * mixBytes
-	}
-	return size
-}
diff --git a/consensus/ethash/algorithm_go1.8_test.go b/consensus/ethash/algorithm_go1.8_test.go
deleted file mode 100644
index 6648bd6a97..0000000000
--- a/consensus/ethash/algorithm_go1.8_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// +build go1.8
-
-package ethash
-
-import "testing"
-
-// Tests whether the dataset size calculator works correctly by cross checking the
-// hard coded lookup table with the value generated by it.
-func TestSizeCalculations(t *testing.T) {
-	// Verify all the cache and dataset sizes from the lookup table.
-	for epoch, want := range cacheSizes {
-		if size := calcCacheSize(epoch); size != want {
-			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
-		}
-	}
-	for epoch, want := range datasetSizes {
-		if size := calcDatasetSize(epoch); size != want {
-			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
-		}
-	}
-}
diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go
index a54f3b582c..841e39233f 100644
--- a/consensus/ethash/algorithm_test.go
+++ b/consensus/ethash/algorithm_test.go
@@ -30,6 +30,22 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
+// Tests whether the dataset size calculator works correctly by cross checking the
+// hard coded lookup table with the value generated by it.
+func TestSizeCalculations(t *testing.T) {
+	// Verify all the cache and dataset sizes from the lookup table.
+	for epoch, want := range cacheSizes {
+		if size := calcCacheSize(epoch); size != want {
+			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
+		}
+	}
+	for epoch, want := range datasetSizes {
+		if size := calcDatasetSize(epoch); size != want {
+			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
+		}
+	}
+}
+
 // Tests that verification caches can be correctly generated.
 func TestCacheGeneration(t *testing.T) {
 	tests := []struct {
diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go
index 92a23d4a4d..3f860bf473 100644
--- a/consensus/ethash/consensus.go
+++ b/consensus/ethash/consensus.go
@@ -53,7 +53,6 @@ var (
 	errDuplicateUncle    = errors.New("duplicate uncle")
 	errUncleIsAncestor   = errors.New("uncle is ancestor")
 	errDanglingUncle     = errors.New("uncle's parent is not ancestor")
-	errNonceOutOfRange   = errors.New("nonce out of range")
 	errInvalidDifficulty = errors.New("non-positive difficulty")
 	errInvalidMixDigest  = errors.New("invalid mix digest")
 	errInvalidPoW        = errors.New("invalid proof-of-work")
@@ -474,18 +473,13 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	if ethash.shared != nil {
 		return ethash.shared.VerifySeal(chain, header)
 	}
-	// Sanity check that the block number is below the lookup table size (60M blocks)
-	number := header.Number.Uint64()
-	if number/epochLength >= maxEpoch {
-		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
-		return errNonceOutOfRange
-	}
 	// Ensure that we have a valid difficulty for the block
 	if header.Difficulty.Sign() <= 0 {
 		return errInvalidDifficulty
 	}
-
 	// Recompute the digest and PoW value and verify against the header
+	number := header.Number.Uint64()
+
 	cache := ethash.cache(number)
 	size := datasetSize(number)
 	if ethash.config.PowMode == ModeTest {

From 014d8d98370abf770b89f7987a92999133e3d4ea Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Tue, 27 Feb 2018 21:16:15 +0100
Subject: [PATCH 102/110] whisper: message filtering optimized

---
 whisper/whisperv6/filter.go | 66 ++++++++++++++++++++++++++++++-------
 1 file changed, 55 insertions(+), 11 deletions(-)

diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go
index eb0c65fa3b..75c31209c9 100644
--- a/whisper/whisperv6/filter.go
+++ b/whisper/whisperv6/filter.go
@@ -35,6 +35,7 @@ type Filter struct {
 	PoW        float64           // Proof of work as described in the Whisper spec
 	AllowP2P   bool              // Indicates whether this filter is interested in direct peer-to-peer messages
 	SymKeyHash common.Hash       // The Keccak256Hash of the symmetric key, needed for optimization
+	id         string            // unique identifier
 
 	Messages map[common.Hash]*ReceivedMessage
 	mutex    sync.RWMutex
@@ -42,16 +43,21 @@ type Filter struct {
 
 // Filters represents a collection of filters
 type Filters struct {
-	watchers map[string]*Filter
-	whisper  *Whisper
-	mutex    sync.RWMutex
+	watchers         map[string]*Filter
+	topicMatcher     map[TopicType]map[*Filter]struct{}
+	allTopicsMatcher map[*Filter]struct{}
+
+	whisper *Whisper
+	mutex   sync.RWMutex
 }
 
 // NewFilters returns a newly created filter collection
 func NewFilters(w *Whisper) *Filters {
 	return &Filters{
-		watchers: make(map[string]*Filter),
-		whisper:  w,
+		watchers:         make(map[string]*Filter),
+		topicMatcher:     make(map[TopicType]map[*Filter]struct{}),
+		allTopicsMatcher: make(map[*Filter]struct{}),
+		whisper:          w,
 	}
 }
 
@@ -81,7 +87,9 @@ func (fs *Filters) Install(watcher *Filter) (string, error) {
 		watcher.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym)
 	}
 
+	watcher.id = id
 	fs.watchers[id] = watcher
+	fs.addTopicMatcher(watcher)
 	return id, err
 }
 
@@ -91,12 +99,49 @@ func (fs *Filters) Uninstall(id string) bool {
 	fs.mutex.Lock()
 	defer fs.mutex.Unlock()
 	if fs.watchers[id] != nil {
+		fs.removeFromTopicMatchers(fs.watchers[id])
 		delete(fs.watchers, id)
 		return true
 	}
 	return false
 }
 
+// addTopicMatcher adds a filter to the topic matchers
+func (fs *Filters) addTopicMatcher(watcher *Filter) {
+	if len(watcher.Topics) == 0 {
+		fs.allTopicsMatcher[watcher] = struct{}{}
+	} else {
+		for _, t := range watcher.Topics {
+			topic := BytesToTopic(t)
+			if fs.topicMatcher[topic] == nil {
+				fs.topicMatcher[topic] = make(map[*Filter]struct{})
+			}
+			fs.topicMatcher[topic][watcher] = struct{}{}
+		}
+	}
+}
+
+// removeFromTopicMatchers removes a filter from the topic matchers
+func (fs *Filters) removeFromTopicMatchers(watcher *Filter) {
+	delete(fs.allTopicsMatcher, watcher)
+	for _, topic := range watcher.Topics {
+		delete(fs.topicMatcher[BytesToTopic(topic)], watcher)
+	}
+}
+
+// getWatchersByTopic returns a slice containing the filters that
+// match a specific topic
+func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter {
+	res := make([]*Filter, 0, len(fs.allTopicsMatcher))
+	for watcher, _ := range fs.allTopicsMatcher {
+		res = append(res, watcher)
+	}
+	for watcher, _ := range fs.topicMatcher[topic] {
+		res = append(res, watcher)
+	}
+	return res
+}
+
 // Get returns a filter from the collection with a specific ID
 func (fs *Filters) Get(id string) *Filter {
 	fs.mutex.RLock()
@@ -112,11 +157,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) {
 	fs.mutex.RLock()
 	defer fs.mutex.RUnlock()
 
-	i := -1 // only used for logging info
-	for _, watcher := range fs.watchers {
-		i++
+	candidates := fs.getWatchersByTopic(env.Topic)
+	for _, watcher := range candidates {
 		if p2pMessage && !watcher.AllowP2P {
-			log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i))
+			log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcher.id))
 			continue
 		}
 
@@ -128,10 +172,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) {
 			if match {
 				msg = env.Open(watcher)
 				if msg == nil {
-					log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i)
+					log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcher.id)
 				}
 			} else {
-				log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i)
+				log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcher.id)
 			}
 		}
 

From c733792be4b49ec47c5307b1a3b8fa116ea16933 Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Tue, 27 Feb 2018 23:38:20 +0100
Subject: [PATCH 103/110] whsiper: refactoring

---
 whisper/whisperv6/filter.go       | 29 ++++++-----------------------
 whisper/whisperv6/filter_test.go  | 25 +++++++++++++++----------
 whisper/whisperv6/whisper.go      | 18 ------------------
 whisper/whisperv6/whisper_test.go | 11 +----------
 4 files changed, 22 insertions(+), 61 deletions(-)

diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go
index 75c31209c9..d7b71795de 100644
--- a/whisper/whisperv6/filter.go
+++ b/whisper/whisperv6/filter.go
@@ -238,16 +238,17 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) {
 
 // MatchMessage checks if the filter matches an already decrypted
 // message (i.e. a Message that has already been handled by
-// MatchEnvelope when checked by a previous filter)
+// MatchEnvelope when checked by a previous filter).
+// Topics are not checked here, since this is done by topic matchers.
 func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
 	if f.PoW > 0 && msg.PoW < f.PoW {
 		return false
 	}
 
 	if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() {
-		return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic)
+		return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst)
 	} else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() {
-		return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic)
+		return f.SymKeyHash == msg.SymKeyHash
 	}
 	return false
 }
@@ -255,27 +256,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
 // MatchEnvelope checks if it's worth decrypting the message. If
 // it returns `true`, client code is expected to attempt decrypting
 // the message and subsequently call MatchMessage.
+// Topics are not checked here, since this is done by topic matchers.
 func (f *Filter) MatchEnvelope(envelope *Envelope) bool {
-	if f.PoW > 0 && envelope.pow < f.PoW {
-		return false
-	}
-
-	return f.MatchTopic(envelope.Topic)
-}
-
-// MatchTopic checks that the filter captures a given topic.
-func (f *Filter) MatchTopic(topic TopicType) bool {
-	if len(f.Topics) == 0 {
-		// any topic matches
-		return true
-	}
-
-	for _, bt := range f.Topics {
-		if matchSingleTopic(topic, bt) {
-			return true
-		}
-	}
-	return false
+	return f.PoW <= 0 || envelope.pow >= f.PoW
 }
 
 func matchSingleTopic(topic TopicType, bt []byte) bool {
diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go
index e7230ef388..491e137bdb 100644
--- a/whisper/whisperv6/filter_test.go
+++ b/whisper/whisperv6/filter_test.go
@@ -303,9 +303,8 @@ func TestMatchEnvelope(t *testing.T) {
 		t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err)
 	}
 
-	params.Topic[0] = 0xFF // ensure mismatch
+	params.Topic[0] = 0xFF // topic mismatch
 
-	// mismatch with pseudo-random data
 	msg, err := NewSentMessage(params)
 	if err != nil {
 		t.Fatalf("failed to create new message with seed %d: %s.", seed, err)
@@ -315,11 +314,13 @@ func TestMatchEnvelope(t *testing.T) {
 		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
 	}
 	match := fsym.MatchEnvelope(env)
-	if match {
+	if !match {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed)
 	}
 	match = fasym.MatchEnvelope(env)
-	if match {
+	if !match {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed)
 	}
 
@@ -396,7 +397,7 @@ func TestMatchEnvelope(t *testing.T) {
 	// asymmetric + matching topic: match
 	fasym.Topics[i] = fasym.Topics[i+1]
 	match = fasym.MatchEnvelope(env)
-	if match {
+	if !match {
 		t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed)
 	}
 
@@ -431,7 +432,8 @@ func TestMatchEnvelope(t *testing.T) {
 	// filter with topic + envelope without topic: mismatch
 	fasym.Topics = fsym.Topics
 	match = fasym.MatchEnvelope(env)
-	if match {
+	if !match {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed)
 	}
 }
@@ -487,7 +489,8 @@ func TestMatchMessageSym(t *testing.T) {
 
 	// topic mismatch
 	f.Topics[index][0]++
-	if f.MatchMessage(msg) {
+	if !f.MatchMessage(msg) {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed)
 	}
 	f.Topics[index][0]--
@@ -580,7 +583,8 @@ func TestMatchMessageAsym(t *testing.T) {
 
 	// topic mismatch
 	f.Topics[index][0]++
-	if f.MatchMessage(msg) {
+	if !f.MatchMessage(msg) {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed)
 	}
 	f.Topics[index][0]--
@@ -829,8 +833,9 @@ func TestVariableTopics(t *testing.T) {
 
 		f.Topics[i][lastTopicByte]++
 		match = f.MatchEnvelope(env)
-		if match {
-			t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i)
+		if !match {
+			// topic mismatch should have no affect, as topics are handled by topic matchers
+			t.Fatalf("MatchEnvelope symmetric with seed %d, step %d.", seed, i)
 		}
 	}
 }
diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go
index b0b601f032..81f59b3cc7 100644
--- a/whisper/whisperv6/whisper.go
+++ b/whisper/whisperv6/whisper.go
@@ -928,24 +928,6 @@ func (whisper *Whisper) Envelopes() []*Envelope {
 	return all
 }
 
-// Messages iterates through all currently floating envelopes
-// and retrieves all the messages, that this filter could decrypt.
-func (whisper *Whisper) Messages(id string) []*ReceivedMessage {
-	result := make([]*ReceivedMessage, 0)
-	whisper.poolMu.RLock()
-	defer whisper.poolMu.RUnlock()
-
-	if filter := whisper.filters.Get(id); filter != nil {
-		for _, env := range whisper.envelopes {
-			msg := filter.processEnvelope(env)
-			if msg != nil {
-				result = append(result, msg)
-			}
-		}
-	}
-	return result
-}
-
 // isEnvelopeCached checks if envelope with specific hash has already been received and cached.
 func (whisper *Whisper) isEnvelopeCached(hash common.Hash) bool {
 	whisper.poolMu.Lock()
diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go
index 99e5f0bbb4..3a219cd133 100644
--- a/whisper/whisperv6/whisper_test.go
+++ b/whisper/whisperv6/whisper_test.go
@@ -75,10 +75,6 @@ func TestWhisperBasic(t *testing.T) {
 	if len(mail) != 0 {
 		t.Fatalf("failed w.Envelopes().")
 	}
-	m := w.Messages("non-existent")
-	if len(m) != 0 {
-		t.Fatalf("failed w.Messages.")
-	}
 
 	derived := pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New)
 	if !validateDataIntegrity(derived, aesKeyLength) {
@@ -593,7 +589,7 @@ func TestCustomization(t *testing.T) {
 	}
 
 	// check w.messages()
-	id, err := w.Subscribe(f)
+	_, err = w.Subscribe(f)
 	if err != nil {
 		t.Fatalf("failed subscribe with seed %d: %s.", seed, err)
 	}
@@ -602,11 +598,6 @@ func TestCustomization(t *testing.T) {
 	if len(mail) > 0 {
 		t.Fatalf("received premature mail")
 	}
-
-	mail = w.Messages(id)
-	if len(mail) != 2 {
-		t.Fatalf("failed to get whisper messages")
-	}
 }
 
 func TestSymmetricSendCycle(t *testing.T) {

From a69cb3b4ff55c6ea2bafb7bd0ab0ff4afafb148f Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Wed, 28 Feb 2018 00:39:38 +0100
Subject: [PATCH 104/110] whisper: comment updated

---
 whisper/mailserver/server_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go
index aeb317e597..63864c1e6b 100644
--- a/whisper/mailserver/server_test.go
+++ b/whisper/mailserver/server_test.go
@@ -169,7 +169,7 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *
 	ok, lower, upper, topic = server.validateRequest(src, request)
 	if !ok {
 		// request should be valid regardless of signature
-		t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper)
+		t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper)
 	}
 }
 

From cf52d5c91f4f9c583eecca63bce52c7df6dd2169 Mon Sep 17 00:00:00 2001
From: b00ris 
Date: Wed, 28 Feb 2018 09:50:36 +0300
Subject: [PATCH 105/110] whisper: fixed datarace

---
 whisper/whisperv6/peer_test.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go
index 9ce5eed8bc..3d3dbd88aa 100644
--- a/whisper/whisperv6/peer_test.go
+++ b/whisper/whisperv6/peer_test.go
@@ -23,6 +23,7 @@ import (
 	mrand "math/rand"
 	"net"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -71,7 +72,7 @@ var keys = []string{
 }
 
 type TestData struct {
-	started int
+	started int64
 	counter [NumNodes]int
 	mutex   sync.RWMutex
 }
@@ -240,9 +241,7 @@ func startServer(t *testing.T, s *p2p.Server) {
 		t.Fatalf("failed to start the fisrt server.")
 	}
 
-	result.mutex.Lock()
-	defer result.mutex.Unlock()
-	result.started++
+	atomic.AddInt64(&result.started, 1)
 }
 
 func stopServers() {
@@ -472,7 +471,10 @@ func checkPowExchange(t *testing.T) {
 func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool {
 	for i, node := range nodes {
 		for peer := range node.shh.peers {
-			if !bytes.Equal(peer.bloomFilter, masterBloomFilter) {
+			peer.bloomMu.Lock()
+			eqals := bytes.Equal(peer.bloomFilter, masterBloomFilter)
+			peer.bloomMu.Unlock()
+			if !eqals {
 				if mustPass {
 					t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got",
 						i, round, masterBloomFilter, peer.bloomFilter)
@@ -500,11 +502,13 @@ func checkBloomFilterExchange(t *testing.T) {
 
 func waitForServersToStart(t *testing.T) {
 	const iterations = 200
+	var started int64
 	for j := 0; j < iterations; j++ {
 		time.Sleep(50 * time.Millisecond)
-		if result.started == NumNodes {
+		started = atomic.LoadInt64(&result.started)
+		if started == NumNodes {
 			return
 		}
 	}
-	t.Fatalf("Failed to start all the servers, running: %d", result.started)
+	t.Fatalf("Failed to start all the servers, running: %d", started)
 }

From 98ec5e50115b47670c1f3ffb2cc890ce4838c4d6 Mon Sep 17 00:00:00 2001
From: Mark Rushakoff 
Date: Wed, 28 Feb 2018 02:20:07 -0800
Subject: [PATCH 106/110] core/asm: rename isAlphaNumeric to isLetter (#16212)

The function would return false for numbers, so isLetter is a more
accurate description of the behavior.
---
 core/asm/lexer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/asm/lexer.go b/core/asm/lexer.go
index a34b2cbd8f..4054999504 100644
--- a/core/asm/lexer.go
+++ b/core/asm/lexer.go
@@ -206,7 +206,7 @@ func lexLine(l *lexer) stateFn {
 			return lexComment
 		case isSpace(r):
 			l.ignore()
-		case isAlphaNumeric(r) || r == '_':
+		case isLetter(r) || r == '_':
 			return lexElement
 		case isNumber(r):
 			return lexNumber
@@ -278,7 +278,7 @@ func lexElement(l *lexer) stateFn {
 	return lexLine
 }
 
-func isAlphaNumeric(t rune) bool {
+func isLetter(t rune) bool {
 	return unicode.IsLetter(t)
 }
 

From ba7b384019de17a72a8c24e3aaac8a98ff30dcc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= 
Date: Wed, 28 Feb 2018 12:40:15 +0200
Subject: [PATCH 107/110] internal/ethapi: fix getTransactionReceipt

---
 internal/ethapi/api.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go
index d021b127c5..20a060e727 100644
--- a/internal/ethapi/api.go
+++ b/internal/ethapi/api.go
@@ -1035,14 +1035,14 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context,
 func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
 	tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash)
 	if tx == nil {
-		return nil, errors.New("unknown transaction")
+		return nil, nil
 	}
 	receipts, err := s.b.GetReceipts(ctx, blockHash)
 	if err != nil {
 		return nil, err
 	}
 	if len(receipts) <= int(index) {
-		return nil, errors.New("unknown receipt")
+		return nil, nil
 	}
 	receipt := receipts[index]
 

From 62c239f608ec2ec3721c33d5755ca2287c71f085 Mon Sep 17 00:00:00 2001
From: b00ris 
Date: Wed, 28 Feb 2018 14:38:42 +0300
Subject: [PATCH 108/110] whisper: fix typo

---
 whisper/whisperv6/peer_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go
index 3d3dbd88aa..6d7754d790 100644
--- a/whisper/whisperv6/peer_test.go
+++ b/whisper/whisperv6/peer_test.go
@@ -472,9 +472,9 @@ func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool {
 	for i, node := range nodes {
 		for peer := range node.shh.peers {
 			peer.bloomMu.Lock()
-			eqals := bytes.Equal(peer.bloomFilter, masterBloomFilter)
+			equals := bytes.Equal(peer.bloomFilter, masterBloomFilter)
 			peer.bloomMu.Unlock()
-			if !eqals {
+			if !equals {
 				if mustPass {
 					t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got",
 						i, round, masterBloomFilter, peer.bloomFilter)

From d24d10a764457937f1bda72053c82d98ac95dd7a Mon Sep 17 00:00:00 2001
From: Vlad 
Date: Wed, 28 Feb 2018 15:05:35 +0100
Subject: [PATCH 109/110] whisper: style fixes

---
 whisper/whisperv6/filter.go      | 15 +++++++++------
 whisper/whisperv6/filter_test.go | 12 +-----------
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go
index d7b71795de..e4171f85c9 100644
--- a/whisper/whisperv6/filter.go
+++ b/whisper/whisperv6/filter.go
@@ -43,9 +43,10 @@ type Filter struct {
 
 // Filters represents a collection of filters
 type Filters struct {
-	watchers         map[string]*Filter
-	topicMatcher     map[TopicType]map[*Filter]struct{}
-	allTopicsMatcher map[*Filter]struct{}
+	watchers map[string]*Filter
+
+	topicMatcher     map[TopicType]map[*Filter]struct{} // map a topic to the filters that are interested in being notified when a message matches that topic
+	allTopicsMatcher map[*Filter]struct{}               // list all the filters that will be notified of a new message, no matter what its topic is
 
 	whisper *Whisper
 	mutex   sync.RWMutex
@@ -106,7 +107,9 @@ func (fs *Filters) Uninstall(id string) bool {
 	return false
 }
 
-// addTopicMatcher adds a filter to the topic matchers
+// addTopicMatcher adds a filter to the topic matchers.
+// If the filter's Topics array is empty, it will be tried on every topic.
+// Otherwise, it will be tried on the topics specified.
 func (fs *Filters) addTopicMatcher(watcher *Filter) {
 	if len(watcher.Topics) == 0 {
 		fs.allTopicsMatcher[watcher] = struct{}{}
@@ -133,10 +136,10 @@ func (fs *Filters) removeFromTopicMatchers(watcher *Filter) {
 // match a specific topic
 func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter {
 	res := make([]*Filter, 0, len(fs.allTopicsMatcher))
-	for watcher, _ := range fs.allTopicsMatcher {
+	for watcher := range fs.allTopicsMatcher {
 		res = append(res, watcher)
 	}
-	for watcher, _ := range fs.topicMatcher[topic] {
+	for watcher := range fs.topicMatcher[topic] {
 		res = append(res, watcher)
 	}
 	return res
diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go
index 491e137bdb..0bb7986c39 100644
--- a/whisper/whisperv6/filter_test.go
+++ b/whisper/whisperv6/filter_test.go
@@ -313,16 +313,6 @@ func TestMatchEnvelope(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
 	}
-	match := fsym.MatchEnvelope(env)
-	if !match {
-		// topic mismatch should have no affect, as topics are handled by topic matchers
-		t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed)
-	}
-	match = fasym.MatchEnvelope(env)
-	if !match {
-		// topic mismatch should have no affect, as topics are handled by topic matchers
-		t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed)
-	}
 
 	// encrypt symmetrically
 	i := mrand.Int() % 4
@@ -338,7 +328,7 @@ func TestMatchEnvelope(t *testing.T) {
 	}
 
 	// symmetric + matching topic: match
-	match = fsym.MatchEnvelope(env)
+	match := fsym.MatchEnvelope(env)
 	if !match {
 		t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed)
 	}

From 5a150e1b7724c91009a237ab0879cd64844b390d Mon Sep 17 00:00:00 2001
From: gluk256 
Date: Thu, 1 Mar 2018 09:34:46 +0100
Subject: [PATCH 110/110] whisper: serious security issue fixed (#16219)

The diagnostic tool was saving the unencrypted version of the messages, which is an obvious
security flaw. As of this commit:
  * encrypted messages saved instead of plain text.
  * all messages are stored, even that created by the user of wnode.
---
 cmd/wnode/main.go | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go
index 0f86adb81c..f8606bf824 100644
--- a/cmd/wnode/main.go
+++ b/cmd/wnode/main.go
@@ -594,19 +594,22 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
 		address = crypto.PubkeyToAddress(*msg.Src)
 	}
 
-	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
-		// message from myself: don't save, only report
-		fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name)
-	} else if len(dir) > 0 {
+	// this is a sample code; uncomment if you don't want to save your own messages.
+	//if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+	//	fmt.Printf("\n%s <%x>: message from myself received, not saved: '%s'\n", timestamp, address, name)
+	//	return
+	//}
+
+	if len(dir) > 0 {
 		fullpath := filepath.Join(dir, name)
-		err := ioutil.WriteFile(fullpath, msg.Payload, 0644)
+		err := ioutil.WriteFile(fullpath, msg.Raw, 0644)
 		if err != nil {
 			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
 		} else {
-			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload))
+			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Raw))
 		}
 	} else {
-		fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name)
+		fmt.Printf("\n%s {%x}: message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Raw), name)
 	}
 }