Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -68,8 +68,11 @@ matrix:
             - debhelper
             - dput
             - fakeroot
+            - python-bzrlib
+            - python-paramiko
       script:
-        - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - if: type = push
@@ -156,7 +159,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.4.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.5.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go

README.md

@@ -1,6 +1,6 @@
 ## Go Ethereum
 
-Official golang implementation of the Ethereum protocol.
+Official Golang implementation of the Ethereum protocol.
 
 [![API Reference](
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
@@ -35,12 +35,12 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 | Command    | Description |
 |:----------:|-------------|
 | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
-| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
+| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
 | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
-| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
+| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
-| `swarm`    | Swarm daemon and tools. This is the entrypoint for the Swarm network. `swarm --help` for command line options and subcommands. See [Swarm README](https://github.com/ethereum/go-ethereum/tree/master/swarm) for more information. |
+| `swarm`    | Swarm daemon and tools. This is the entry point for the Swarm network. `swarm --help` for command line options and subcommands. See [Swarm README](https://github.com/ethereum/go-ethereum/tree/master/swarm) for more information. |
 | `puppeth`    | a CLI wizard that aids in creating a new Ethereum network. |
 
 ## Running geth
@@ -72,7 +72,7 @@ This command will:
    This tool is optional and if you leave it out you can always attach to an already running Geth instance
    with `geth attach`.
 
-### Full node on the Ethereum test network
+### A Full node on the Ethereum test network
 
 Transitioning towards developers, if you'd like to play around with creating Ethereum contracts, you
 almost certainly would like to do that without any real money involved until you get the hang of the
@@ -83,10 +83,10 @@ network with your node, which is fully equivalent to the main network, but with
 $ geth --testnet console
 ```
 
-The `console` subcommand have the exact same meaning as above and they are equally useful on the
+The `console` subcommand has the exact same meaning as above and they are equally useful on the
-testnet too. Please see above for their explanations if you've skipped to here.
+testnet too. Please see above for their explanations if you've skipped here.
 
-Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
+Specifying the `--testnet` flag, however, will reconfigure your Geth instance a bit:
 
  * Instead of using the default data directory (`~/.ethereum` on Linux for example), Geth will nest
    itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux). Note, on OSX
@@ -103,7 +103,7 @@ separate the two networks and will not make any accounts available between them.
 
 ### Full node on the Rinkeby test network
 
-The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
+The above test network is a cross-client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty/security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
 
 ```
 $ geth --rinkeby console
@@ -139,13 +139,13 @@ This will start geth in fast-sync mode with a DB memory allowance of 1GB just as
 
 Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
 
-### Programatically interfacing Geth nodes
+### Programmatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
 network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
-exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).
+exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows).
 
 The IPC interface is enabled by default and exposes all the APIs supported by Geth, whereas the HTTP
 and WS interfaces need to manually be enabled and only expose a subset of APIs due to security reasons.
@@ -173,7 +173,7 @@ on all transports. You can reuse the same connection for multiple requests!
 
 **Note: Please understand the security implications of opening up an HTTP/WS based transport before
 doing so! Hackers on the internet are actively trying to subvert Ethereum nodes with exposed APIs!
-Further, all browser tabs can access locally running webservers, so malicious webpages could try to
+Further, all browser tabs can access locally running web servers, so malicious web pages could try to
 subvert locally available APIs!**
 
 ### Operating a private network
@@ -241,7 +241,7 @@ that other nodes can use to connect to it and exchange peer information. Make su
 displayed IP address information (most probably `[::]`) with your externally accessible IP to get the
 actual `enode` URL.
 
-*Note: You could also use a full fledged Geth node as a bootnode, but it's the less recommended way.*
+*Note: You could also use a full-fledged Geth node as a bootnode, but it's the less recommended way.*
 
 #### Starting up your member nodes
 
@@ -264,7 +264,7 @@ an OpenCL or CUDA enabled `ethminer` instance. For information on such a setup,
 [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) and the [Genoil miner](https://github.com/Genoil/cpp-ethereum)
 repository.
 
-In a private network setting however, a single CPU miner instance is more than enough for practical
+In a private network setting, however a single CPU miner instance is more than enough for practical
 purposes as it can produce a stable stream of blocks at the correct intervals without needing heavy
 resources (consider running on a single thread, no need for multiple ones either). To start a Geth
 instance for mining, run it with all your usual flags, extended by:
@@ -298,7 +298,7 @@ Please make sure your contributions adhere to our coding guidelines:
    * E.g. "eth, rpc: make trace configs optional"
 
 Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, managing project dependencies and testing procedures.
+for more details on configuring your environment, managing project dependencies, and testing procedures.
 
 ## License
 

accounts/abi/bind/backends/simulated.go

@@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
 	return receipt, nil
 }
 
+// TransactionByHash checks the pool of pending transactions in addition to the
+// blockchain. The isPending return value indicates whether the transaction has been
+// mined yet. Note that the transaction may not be part of the canonical chain even if
+// it's not pending.
+func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) {
+
+	tx = b.pendingBlock.Transaction(txHash)
+	if tx != nil {
+		return tx, true, nil
+	}
+
+	tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
+	if tx != nil {
+		return tx, false, nil
+	}
+
+	return nil, false, ethereum.NotFound
+}
+
 // PendingCodeAt returns the code associated with an account in the pending state.
 func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
 	b.mu.Lock()

accounts/abi/bind/backends/simulated_test.go

@@ -0,0 +1,66 @@
+package backends_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+func TestSimulatedBackend(t *testing.T) {
+	var gasLimit uint64 = 8000029
+	key, _ := crypto.GenerateKey() // nolint: gosec
+	auth := bind.NewKeyedTransactor(key)
+	genAlloc := make(core.GenesisAlloc)
+	genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
+
+	sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
+
+	// should return an error if the tx is not found
+	txHash := common.HexToHash("2")
+	_, isPending, err := sim.TransactionByHash(context.Background(), txHash)
+
+	if isPending {
+		t.Fatal("transaction should not be pending")
+	}
+	if err != ethereum.NotFound {
+		t.Fatalf("err should be `ethereum.NotFound` but received %v", err)
+	}
+
+	// generate a transaction and confirm you can retrieve it
+	code := `6060604052600a8060106000396000f360606040526008565b00`
+	var gas uint64 = 3000000
+	tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code))
+	tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key)
+
+	err = sim.SendTransaction(context.Background(), tx)
+	if err != nil {
+		t.Fatal("error sending transaction")
+	}
+
+	txHash = tx.Hash()
+	_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if !isPending {
+		t.Fatal("transaction should have pending status")
+	}
+
+	sim.Commit()
+	tx, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if isPending {
+		t.Fatal("transaction should not have pending status")
+	}
+
+}

accounts/accounts.go

@@ -18,12 +18,14 @@
 package accounts
 
 import (
+	"fmt"
 	"math/big"
 
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/event"
+	"golang.org/x/crypto/sha3"
 )
 
 // Account represents an Ethereum account located at a specific location defined
@@ -33,6 +35,13 @@ type Account struct {
 	URL     URL            `json:"url"`     // Optional resource locator within a backend
 }
 
+const (
+	MimetypeTextWithValidator = "text/validator"
+	MimetypeTypedData         = "data/typed"
+	MimetypeClique            = "application/x-clique-header"
+	MimetypeTextPlain         = "text/plain"
+)
+
 // Wallet represents a software or hardware wallet that might contain one or more
 // accounts (derived from the same seed).
 type Wallet interface {
@@ -87,8 +96,7 @@ type Wallet interface {
 	// chain state reader.
 	SelfDerive(base DerivationPath, chain ethereum.ChainStateReader)
 
-	// SignHash requests the wallet to sign the given hash.
+	// SignData requests the wallet to sign the hash of the given data
-	//
 	// It looks up the account specified either solely via its address contained within,
 	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
@@ -98,7 +106,29 @@ type Wallet interface {
 	// about which fields or actions are needed. The user may retry by providing
 	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
 	// the account in a keystore).
-	SignHash(account Account, hash []byte) ([]byte, error)
+	SignData(account Account, mimeType string, data []byte) ([]byte, error)
+
+	// SignDataWithPassphrase is identical to SignData, but also takes a password
+	// NOTE: there's an chance that an erroneous call might mistake the two strings, and
+	// supply password in the mimetype field, or vice versa. Thus, an implementation
+	// should never echo the mimetype or return the mimetype in the error-response
+	SignDataWithPassphrase(account Account, passphrase, mimeType string, data []byte) ([]byte, error)
+
+	// Signtext requests the wallet to sign the hash of a given piece of data, prefixed
+	// by the Ethereum prefix scheme
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignText(account Account, text []byte) ([]byte, error)
+
+	// SignTextWithPassphrase is identical to Signtext, but also takes a password
+	SignTextWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
 
 	// SignTx requests the wallet to sign the given transaction.
 	//
@@ -113,18 +143,7 @@ type Wallet interface {
 	// the account in a keystore).
 	SignTx(account Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
 
-	// SignHashWithPassphrase requests the wallet to sign the given hash with the
+	// SignTxWithPassphrase is identical to SignTx, but also takes a password
-	// given passphrase as extra authentication information.
-	//
-	// It looks up the account specified either solely via its address contained within,
-	// or optionally with the aid of any location metadata from the embedded URL field.
-	SignHashWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
-
-	// SignTxWithPassphrase requests the wallet to sign the given transaction, with the
-	// given passphrase as extra authentication information.
-	//
-	// It looks up the account specified either solely via its address contained within,
-	// or optionally with the aid of any location metadata from the embedded URL field.
 	SignTxWithPassphrase(account Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
 }
 
@@ -148,6 +167,32 @@ type Backend interface {
 	Subscribe(sink chan<- WalletEvent) event.Subscription
 }
 
+// TextHash is a helper function that calculates a hash for the given message that can be
+// safely used to calculate a signature from.
+//
+// The hash is calulcated as
+//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+// This gives context to the signed message and prevents signing of transactions.
+func TextHash(data []byte) []byte {
+	hash, _ := TextAndHash(data)
+	return hash
+}
+
+// TextAndHash is a helper function that calculates a hash for the given message that can be
+// safely used to calculate a signature from.
+//
+// The hash is calulcated as
+//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+// This gives context to the signed message and prevents signing of transactions.
+func TextAndHash(data []byte) ([]byte, string) {
+	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data))
+	hasher := sha3.NewLegacyKeccak256()
+	hasher.Write([]byte(msg))
+	return hasher.Sum(nil), msg
+}
+
 // WalletEventType represents the different event types that can be fired by
 // the wallet subscription subsystem.
 type WalletEventType int

accounts/accounts_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,30 +14,19 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package rpc
+package accounts
 
 import (
-	"strings"
+	"bytes"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
-func TestNewID(t *testing.T) {
+func TestTextHash(t *testing.T) {
-	hexchars := "0123456789ABCDEFabcdef"
+	hash := TextHash([]byte("Hello Joe"))
-	for i := 0; i < 100; i++ {
+	want := hexutil.MustDecode("0xa080337ae51c4e064c189e113edd0ba391df9206e2f49db658bb32cf2911730b")
-		id := string(NewID())
+	if !bytes.Equal(hash, want) {
-		if !strings.HasPrefix(id, "0x") {
+		t.Fatalf("wrong hash: %x", hash)
-			t.Fatalf("invalid ID prefix, want '0x...', got %s", id)
-		}
-
-		id = id[2:]
-		if len(id) == 0 || len(id) > 32 {
-			t.Fatalf("invalid ID length, want len(id) > 0 && len(id) <= 32), got %d", len(id))
-		}
-
-		for i := 0; i < len(id); i++ {
-			if strings.IndexByte(hexchars, id[i]) == -1 {
-				t.Fatalf("unexpected byte, want any valid hex char, got %c", id[i])
-			}
-		}
 	}
 }

accounts/external/backend.go

@@ -0,0 +1,228 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package external
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+)
+
+type ExternalBackend struct {
+	signers []accounts.Wallet
+}
+
+func (eb *ExternalBackend) Wallets() []accounts.Wallet {
+	return eb.signers
+}
+
+func NewExternalBackend(endpoint string) (*ExternalBackend, error) {
+	signer, err := NewExternalSigner(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &ExternalBackend{
+		signers: []accounts.Wallet{signer},
+	}, nil
+}
+
+func (eb *ExternalBackend) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		<-quit
+		return nil
+	})
+}
+
+// ExternalSigner provides an API to interact with an external signer (clef)
+// It proxies request to the external signer while forwarding relevant
+// request headers
+type ExternalSigner struct {
+	client   *rpc.Client
+	endpoint string
+	status   string
+	cacheMu  sync.RWMutex
+	cache    []accounts.Account
+}
+
+func NewExternalSigner(endpoint string) (*ExternalSigner, error) {
+	client, err := rpc.Dial(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	extsigner := &ExternalSigner{
+		client:   client,
+		endpoint: endpoint,
+	}
+	// Check if reachable
+	version, err := extsigner.pingVersion()
+	if err != nil {
+		return nil, err
+	}
+	extsigner.status = fmt.Sprintf("ok [version=%v]", version)
+	return extsigner, nil
+}
+
+func (api *ExternalSigner) URL() accounts.URL {
+	return accounts.URL{
+		Scheme: "extapi",
+		Path:   api.endpoint,
+	}
+}
+
+func (api *ExternalSigner) Status() (string, error) {
+	return api.status, nil
+}
+
+func (api *ExternalSigner) Open(passphrase string) error {
+	return fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) Close() error {
+	return fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) Accounts() []accounts.Account {
+	var accnts []accounts.Account
+	res, err := api.listAccounts()
+	if err != nil {
+		log.Error("account listing failed", "error", err)
+		return accnts
+	}
+	for _, addr := range res {
+		accnts = append(accnts, accounts.Account{
+			URL: accounts.URL{
+				Scheme: "extapi",
+				Path:   api.endpoint,
+			},
+			Address: addr,
+		})
+	}
+	api.cacheMu.Lock()
+	api.cache = accnts
+	api.cacheMu.Unlock()
+	return accnts
+}
+
+func (api *ExternalSigner) Contains(account accounts.Account) bool {
+	api.cacheMu.RLock()
+	defer api.cacheMu.RUnlock()
+	for _, a := range api.cache {
+		if a.Address == account.Address && (account.URL == (accounts.URL{}) || account.URL == api.URL()) {
+			return true
+		}
+	}
+	return false
+}
+
+func (api *ExternalSigner) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	return accounts.Account{}, fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
+	log.Error("operation SelfDerive not supported on external signers")
+}
+
+func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) {
+	return []byte{}, fmt.Errorf("operation not supported on external signers")
+}
+
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	var res hexutil.Bytes
+	var signAddress = common.NewMixedcaseAddress(account.Address)
+	if err := api.client.Call(&res, "account_signData",
+		mimeType,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(data)); err != nil {
+		return nil, err
+	}
+	// If V is on 27/28-form, convert to to 0/1 for Clique
+	if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) {
+		res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	var res hexutil.Bytes
+	var signAddress = common.NewMixedcaseAddress(account.Address)
+	if err := api.client.Call(&res, "account_signData",
+		accounts.MimetypeTextPlain,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(text)); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	res := ethapi.SignTransactionResult{}
+	to := common.NewMixedcaseAddress(*tx.To())
+	data := hexutil.Bytes(tx.Data())
+	args := &core.SendTxArgs{
+		Data:     &data,
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		Value:    hexutil.Big(*tx.Value()),
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: hexutil.Big(*tx.GasPrice()),
+		To:       &to,
+		From:     common.NewMixedcaseAddress(account.Address),
+	}
+
+	if err := api.client.Call(&res, "account_signTransaction", args); err != nil {
+		return nil, err
+	}
+	return res.Tx, nil
+}
+
+func (api *ExternalSigner) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	return []byte{}, fmt.Errorf("passphrase-operations not supported on external signers")
+}
+
+func (api *ExternalSigner) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	return nil, fmt.Errorf("passphrase-operations not supported on external signers")
+}
+func (api *ExternalSigner) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	return nil, fmt.Errorf("passphrase-operations not supported on external signers")
+}
+
+func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
+	var res []common.Address
+	if err := api.client.Call(&res, "account_list"); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) pingVersion() (string, error) {
+	var v string
+	if err := api.client.Call(&v, "account_version"); err != nil {
+		return "", err
+	}
+	return v, nil
+}

accounts/keystore/wallet.go

@@ -22,6 +22,7 @@ import (
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 )
 
 // keystoreWallet implements the accounts.Wallet interface for the original
@@ -78,11 +79,11 @@ func (w *keystoreWallet) Derive(path accounts.DerivationPath, pin bool) (account
 // there is no notion of hierarchical account derivation for plain keystore accounts.
 func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {}
 
-// SignHash implements accounts.Wallet, attempting to sign the given hash with
+// signHash attempts to sign the given hash with
 // the given account. If the wallet does not wrap this particular account, an
 // error is returned to avoid account leakage (even though in theory we may be
 // able to sign via our shared keystore backend).
-func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+func (w *keystoreWallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
 	// Make sure the requested account is contained within
 	if !w.Contains(account) {
 		return nil, accounts.ErrUnknownAccount
@@ -91,6 +92,36 @@ func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte
 	return w.keystore.SignHash(account, hash)
 }
 
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+// SignDataWithPassphrase signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, crypto.Keccak256(data))
+}
+
+func (w *keystoreWallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
+// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
+// given hash with the given account using passphrase as extra authentication.
+func (w *keystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, accounts.TextHash(text))
+}
+
 // SignTx implements accounts.Wallet, attempting to sign the given transaction
 // with the given account. If the wallet does not wrap this particular account,
 // an error is returned to avoid account leakage (even though in theory we may
@@ -104,17 +135,6 @@ func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction,
 	return w.keystore.SignTx(account, tx, chainID)
 }
 
-// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
-// given hash with the given account using passphrase as extra authentication.
-func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
-	// Make sure the requested account is contained within
-	if !w.Contains(account) {
-		return nil, accounts.ErrUnknownAccount
-	}
-	// Account seems valid, request the keystore to sign
-	return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
-}
-
 // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
 // transaction with the given account using passphrase as extra authentication.
 func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {

accounts/usbwallet/trezor.go

@@ -28,7 +28,7 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/accounts"
-	"github.com/ethereum/go-ethereum/accounts/usbwallet/internal/trezor"
+	"github.com/ethereum/go-ethereum/accounts/usbwallet/trezor"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"

accounts/usbwallet/wallet.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/karalabe/hid"
 )
@@ -495,12 +496,28 @@ func (w *wallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainSt
 	w.deriveChain = chain
 }
 
-// SignHash implements accounts.Wallet, however signing arbitrary data is not
+// signHash implements accounts.Wallet, however signing arbitrary data is not
 // supported for hardware wallets, so this method will always return an error.
-func (w *wallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
 	return nil, accounts.ErrNotSupported
 }
 
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+// SignDataWithPassphrase implements accounts.Wallet, attempting to sign the given
+// data with the given account using passphrase as extra authentication.
+// Since USB wallets don't rely on passphrases, these are silently ignored.
+func (w *wallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	return w.SignData(account, mimeType, data)
+}
+
+func (w *wallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
 // SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
 // wallet to request a confirmation from the user. It returns either the signed
 // transaction or a failure if the user denied the transaction.
@@ -550,8 +567,8 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID
 // SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
 // data is not supported for Ledger wallets, so this method will always return
 // an error.
-func (w *wallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
-	return w.SignHash(account, hash)
+	return w.SignText(account, accounts.TextHash(text))
 }
 
 // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.4.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.5.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.11.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci-notes.md

@@ -7,11 +7,18 @@ Canonical.
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
 
+## Launchpad
+
 The packages are built and served by launchpad.net. We generate a Debian source package
 for each distribution and upload it. Their builder picks up the source package, builds it
 and installs the new version into the PPA repository. Launchpad requires a valid signature
-by a team member for source package uploads. The signing key is stored in an environment
+by a team member for source package uploads.
-variable which Travis CI makes available to certain builds.
+
+The signing key is stored in an environment variable which Travis CI makes available to
+certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
+packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
+and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
+variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
@@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 

build/ci.go

@@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
 func archiveUpload(archive string, blobstore string, signer string) error {
 	// If signing was requested, generate the signature files
 	if signer != "" {
-		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
+		key := getenvBase64(signer)
-		if err != nil {
+		if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
-			return fmt.Errorf("invalid base64 %s", signer)
-		}
-		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
 			return err
 		}
 	}
@@ -488,7 +485,8 @@ func maybeSkipArchive(env build.Environment) {
 func doDebianSource(cmdline []string) {
 	var (
 		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+		upload  = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+		sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
 		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
 		now     = time.Now()
 	)
@@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
 	maybeSkipArchive(env)
 
 	// Import the signing key.
-	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
+	if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatal("invalid base64 PPA_SIGNING_KEY")
-		}
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
@@ -513,22 +507,58 @@ func doDebianSource(cmdline []string) {
 		for _, distro := range debDistros {
 			meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
-			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
+			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
 			debuild.Dir = pkgdir
 			build.MustRun(debuild)
 
-			changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
+			var (
-			changes = filepath.Join(*workdir, changes)
+				basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString())
+				source   = filepath.Join(*workdir, basename+".tar.xz")
+				dsc      = filepath.Join(*workdir, basename+".dsc")
+				changes  = filepath.Join(*workdir, basename+"_source.changes")
+			)
 			if *signer != "" {
 				build.MustRunCommand("debsign", changes)
 			}
 			if *upload != "" {
-				build.MustRunCommand("dput", *upload, changes)
+				ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes})
 			}
 		}
 	}
 }
 
+func ppaUpload(workdir, ppa, sshUser string, files []string) {
+	p := strings.Split(ppa, "/")
+	if len(p) != 2 {
+		log.Fatal("-upload PPA name must contain single /")
+	}
+	if sshUser == "" {
+		sshUser = p[0]
+	}
+	incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1])
+	// Create the SSH identity file if it doesn't exist.
+	var idfile string
+	if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
+		idfile = filepath.Join(workdir, "sshkey")
+		if _, err := os.Stat(idfile); os.IsNotExist(err) {
+			ioutil.WriteFile(idfile, sshkey, 0600)
+		}
+	}
+	// Upload
+	dest := sshUser + "@ppa.launchpad.net"
+	if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getenvBase64(variable string) []byte {
+	dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
+	if err != nil {
+		log.Fatal("invalid base64 " + variable)
+	}
+	return []byte(dec)
+}
+
 func makeWorkdir(wdflag string) string {
 	var err error
 	if wdflag != "" {
@@ -800,15 +830,10 @@ func doAndroidArchive(cmdline []string) {
 	os.Rename(archive, meta.Package+".aar")
 	if *signer != "" && *deploy != "" {
 		// Import the signing key into the local GPG instance
-		b64key := os.Getenv(*signer)
+		key := getenvBase64(*signer)
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatalf("invalid base64 %s", *signer)
-		}
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
-
 		keyID, err := build.PGPKeyID(string(key))
 		if err != nil {
 			log.Fatal(err)

cmd/abigen/main.go

@@ -124,7 +124,7 @@ func main() {
 		}
 		abis = append(abis, string(abi))
 
-		bin := []byte{}
+		var bin []byte
 		if *binFlag != "" {
 			if bin, err = ioutil.ReadFile(*binFlag); err != nil {
 				fmt.Printf("Failed to read input bytecode: %v\n", err)

cmd/bootnode/main.go

@@ -112,12 +112,13 @@ func main() {
 		if !realaddr.IP.IsLoopback() {
 			go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
 		}
-		// TODO: react to external IP changes over time.
 		if ext, err := natm.ExternalIP(); err == nil {
 			realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
 		}
 	}
 
+	printNotice(&nodeKey.PublicKey, *realaddr)
+
 	if *runv5 {
 		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
@@ -136,3 +137,13 @@ func main() {
 
 	select {}
 }
+
+func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) {
+	if addr.IP.IsUnspecified() {
+		addr.IP = net.IP{127, 0, 0, 1}
+	}
+	n := enode.NewV4(nodeKey, addr.IP, 0, addr.Port)
+	fmt.Println(n.String())
+	fmt.Println("Note: you're using cmd/bootnode, a developer tool.")
+	fmt.Println("We recommend using a regular node as bootstrap node for production deployments.")
+}

cmd/clef/README.md

@@ -16,7 +16,8 @@ Check out
 
 * the [tutorial](tutorial.md) for some concrete examples on how the signer works.
 * the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. 
-
+* the [data types](datatypes.md) for detailed information on the json types used in the communication between
+  clef and an external UI 
 
 ## Command line flags
 Clef accepts the following command line options:
@@ -24,25 +25,31 @@ Clef accepts the following command line options:
 COMMANDS:
    init    Initialize the signer, generate secret storage
    attest  Attest that a js-file is to be used
-   addpw   Store a credential for a keystore file
+   setpw   Store a credential for a keystore file
+   gendoc  Generate documentation about json-rpc format
    help    Shows a list of commands or help for one command
    
 GLOBAL OPTIONS:
    --loglevel value        log level to emit to the screen (default: 4)
    --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
-   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
+   --configdir value       Directory for Clef configuration (default: "$HOME/.clef")
-   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --chainid value         Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli) (default: 1)
    --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
    --nousb                 Disables monitoring for and managing USB hardware wallets
    --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcvhosts value       Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
+   --ipcdisable            Disable the IPC-RPC server
+   --ipcpath               Filename for IPC socket/pipe within the datadir (explicit paths escape it)
+   --rpc                   Enable the HTTP-RPC server
    --rpcport value         HTTP-RPC server listening port (default: 8550)
-   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --signersecret value    A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
    --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
    --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
    --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
    --rules value           Enable rule-engine (default: "rules.json")
-   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
-   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --stdio-ui-test         Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
+   --advanced              If enabled, issues warnings instead of rejections for suspicious requests. Default off
    --help, -h              show help
    --version, -v           print the version
    
@@ -189,7 +196,9 @@ None
   "method": "account_new",
   "params": []
 }
-
+```
+Response
+```
 {
   "id": 0,
   "jsonrpc": "2.0",
@@ -222,7 +231,9 @@ None
   "jsonrpc": "2.0",
   "method": "account_list"
 }
-
+```
+Response
+```
 {
   "id": 1,
   "jsonrpc": "2.0",
@@ -285,8 +296,8 @@ Response
 
 ```json
 {
+  "id": 2,
   "jsonrpc": "2.0",
-  "id": 67,
   "error": {
     "code": -32000,
     "message": "Request denied"
@@ -298,6 +309,7 @@ Response
 
 ```json
 {
+  "id": 67,
   "jsonrpc": "2.0",
   "method": "account_signTransaction",
   "params": [
@@ -311,8 +323,7 @@ Response
       "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
     },
     "safeSend(address)"
-  ],
+  ]
-  "id": 67
 }
 ```
 Response
@@ -346,15 +357,18 @@ Bash example:
 {"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
 ```
 
-
+### account_signData
-### account_sign
 
 #### Sign data
    Signs a chunk of data and returns the calculated signature.
 
 #### Arguments
+  - content type [string]: type of signed data
+     - `text/validator`: hex data with custom validator defined in a contract
+     - `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
+     - `text/plain`: simple hex data validated by `account_ecRecover`
   - account [address]: account to sign with
-  - data [data]: data to sign
+  - data [object]: data to sign
 
 #### Result
   - calculated signature [data]
@@ -364,8 +378,9 @@ Bash example:
 {
   "id": 3,
   "jsonrpc": "2.0",
-  "method": "account_sign",
+  "method": "account_signData",
   "params": [
+    "data/plain",
     "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
     "0xaabbccdd"
   ]
@@ -381,10 +396,108 @@ Response
 }
 ```
 
+### account_signTypedData
+
+#### Sign data
+   Signs a chunk of structured data conformant to [EIP712]([EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md)) and returns the calculated signature.
+
+#### Arguments
+  - account [address]: account to sign with
+  - data [object]: data to sign
+
+#### Result
+  - calculated signature [data]
+  
+#### Sample call
+```json
+{
+  "id": 68,
+  "jsonrpc": "2.0",
+  "method": "account_signTypedData",
+  "params": [
+    "0xcd2a3d9f938e13cd947ec05abc7fe734df8dd826",
+    {
+      "types": {
+        "EIP712Domain": [
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "version",
+            "type": "string"
+          },
+          {
+            "name": "chainId",
+            "type": "uint256"
+          },
+          {
+            "name": "verifyingContract",
+            "type": "address"
+          }
+        ],
+        "Person": [
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "wallet",
+            "type": "address"
+          }
+        ],
+        "Mail": [
+          {
+            "name": "from",
+            "type": "Person"
+          },
+          {
+            "name": "to",
+            "type": "Person"
+          },
+          {
+            "name": "contents",
+            "type": "string"
+          }
+        ]
+      },
+      "primaryType": "Mail",
+      "domain": {
+        "name": "Ether Mail",
+        "version": "1",
+        "chainId": 1,
+        "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC"
+      },
+      "message": {
+        "from": {
+          "name": "Cow",
+          "wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826"
+        },
+        "to": {
+          "name": "Bob",
+          "wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB"
+        },
+        "contents": "Hello, Bob!"
+      }
+    }
+  ]
+}
+```
+Response
+
+```json
+{
+    "id": 1,
+    "jsonrpc": "2.0",
+    "result": "0x4355c47d63924e8a72e509b65029052eb6c299d53a04e167c5775fd466751c9d07299936d304c153f6443dfa05f40ff007d72911b6f72307f996231605b915621c"
+}
+```
+
 ### account_ecRecover
 
-#### Recover address
+#### Sign data
-   Derive the address from the account that was used to sign data from the data and signature.
+
+Derive the address from the account that was used to sign data with content type `text/plain` and the signature.
 
 #### Arguments
   - data [data]: data that was signed
@@ -400,6 +513,7 @@ Response
   "jsonrpc": "2.0",
   "method": "account_ecRecover",
   "params": [
+    "data/plain",
     "0xaabbccdd",
     "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
   ]
@@ -413,7 +527,6 @@ Response
   "jsonrpc": "2.0",
   "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
 }
-
 ```
 
 ### account_import
@@ -458,7 +571,7 @@ Response
       },
       "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
       "version": 3
-    },
+    }
   ]
 }
 ```

cmd/clef/datatypes.md

@@ -0,0 +1,219 @@
+## UI Client interface
+
+These data types are defined in the channel between clef and the UI
+### SignDataRequest
+
+SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to presentthe user with the contents of the `message`
+
+Example:
+```json
+{
+  "content_type": "text/plain",
+  "address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+  "raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
+  "message": [
+    {
+      "name": "message",
+      "value": "\u0019Ethereum Signed Message:\n11hello world",
+      "type": "text/plain"
+    }
+  ],
+  "hash": "0xd9eba16ed0ecae432b71fe008c98cc872bb4cc214d3220a36f365326cf807d68",
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### SignDataResponse - approve
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "approved": true,
+  "Password": "apassword"
+}
+```
+### SignDataResponse - deny
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "approved": false,
+  "Password": ""
+}
+```
+### SignTxRequest
+
+SignTxRequest contains information about a pending request to sign a transaction. Aside from the transaction itself, there is also a `call_info`-struct. That struct contains messages of various types, that the user should be informed of.
+
+As in any request, it's important to consider that the `meta` info also contains untrusted data.
+
+The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, they must be identical, otherwise an error is generated. However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+    "to": null,
+    "gas": "0x3e8",
+    "gasPrice": "0x5",
+    "value": "0x6",
+    "nonce": "0x1",
+    "data": "0x01020304"
+  },
+  "call_info": [
+    {
+      "type": "Warning",
+      "message": "Something looks odd, show this message as a warning"
+    },
+    {
+      "type": "Info",
+      "message": "User should see this aswell"
+    }
+  ],
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### SignDataResponse - approve
+
+Response to SignDataRequest. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+    "to": null,
+    "gas": "0x3e8",
+    "gasPrice": "0x5",
+    "value": "0x6",
+    "nonce": "0x4",
+    "data": "0x04030201"
+  },
+  "approved": true,
+  "password": "apassword"
+}
+```
+### SignDataResponse - deny
+
+Response to SignDataRequest. When denying a request, there's no need to provide the transaction in return
+
+Example:
+```json
+{
+  "approved": false,
+  "Password": ""
+}
+```
+### OnApproved - SignTransactionResult
+
+SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`
+
+This occurs _after_ successful completion of the entire signing procedure, but right before the signed transaction is passed to the external caller. This method (and data) can be used by the UI to signal to the user that the transaction was signed, but it is primarily useful for ruleset implementations.
+
+A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
+
+**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. 
+
+The `OnApproved` method cannot be responded to, it's purely informative
+
+Example:
+```json
+{
+  "raw": "0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
+  "tx": {
+    "nonce": "0x64",
+    "gasPrice": "0x1",
+    "gas": "0x1",
+    "to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
+    "value": "0x1",
+    "input": "0x",
+    "v": "0x26",
+    "r": "0x716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293",
+    "s": "0x4e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
+    "hash": "0x662f6d772692dd692f1b5e8baa77a9ff95bbd909362df3fc3d301aafebde5441"
+  }
+}
+```
+### UserInputRequest
+
+Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)
+
+Example:
+```json
+{
+  "prompt": "The question to ask the user",
+  "title": "The title here",
+  "isPassword": true
+}
+```
+### UserInputResponse
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "text": "The textual response from user"
+}
+```
+### ListRequest
+
+Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es. 
+
+Example:
+```json
+{
+  "accounts": [
+    {
+      "address": "0xdeadbeef000000000000000000000000deadbeef",
+      "url": "keystore:///path/to/keyfile/a"
+    },
+    {
+      "address": "0x1111111122222222222233333333334444444444",
+      "url": "keystore:///path/to/keyfile/b"
+    }
+  ],
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### UserInputResponse
+
+Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
+
+Example:
+```json
+{
+  "accounts": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "url": ".. ignored .."
+    },
+    {
+      "address": "0xffffffffffffffffffffffffffffffffffffffff",
+      "url": ""
+    }
+  ]
+}
+```

cmd/clef/extapi_changelog.md

@@ -1,5 +1,20 @@
 ### Changelog for external API
 
+### 6.0.0
+
+* `New` was changed to deliver only an address, not the full `Account` data
+* `Export` was moved from External API to the UI Server API
+
+#### 5.0.0
+
+* The external `account_EcRecover`-method was reimplemented.
+* The external method `account_sign(address, data)` was replaced with `account_signData(contentType, address, data)`.
+The addition of `contentType` makes it possible to use the method for different types of objects, such as:
+  * signing data with an intended validator (not yet implemented)
+  * signing clique headers,
+  * signing plain personal messages,
+* The external method `account_signTypedData` implements [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md) and makes it possible to sign typed data.
+  
 #### 4.0.0
 
 * The external `account_Ecrecover`-method was removed. 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,40 @@
 ### Changelog for internal API (ui-api)
 
+### 4.0.0
+
+* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
+  - `clef_listWallets` 
+  - `clef_listAccounts`
+  - `clef_listWallets`
+  - `clef_deriveAccount`
+  - `clef_importRawKey`
+  - `clef_openWallet`
+  - `clef_chainId`
+  - `clef_setChainId`
+  - `clef_export`
+  - `clef_import`
+ 
+* The type `Account` was modified (the json-field `type` was removed), to consist of 
+
+```golang
+type Account struct {
+	Address common.Address `json:"address"` // Ethereum account address derived from the key
+	URL     URL            `json:"url"`     // Optional resource locator within a backend
+}
+```
+
+
+### 3.2.0
+
+* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
+
+> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
+> 
+>  Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
+### 3.1.0
+
+* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
+
 ### 3.0.0
 
 * Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup

cmd/clef/main.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/big"
 	"os"
 	"os/signal"
 	"os/user"
@@ -35,13 +36,19 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/console"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/signer/core"
 	"github.com/ethereum/go-ethereum/signer/rules"
@@ -49,12 +56,6 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-// ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "4.0.0"
-
-// InternalAPIVersion -- see intapi_changelog.md
-const InternalAPIVersion = "3.0.0"
-
 const legalWarning = `
 WARNING! 
 
@@ -86,6 +87,11 @@ var (
 		Value: DefaultConfigDir(),
 		Usage: "Directory for Clef configuration",
 	}
+	chainIdFlag = cli.Int64Flag{
+		Name:  "chainid",
+		Value: params.MainnetChainConfig.ChainID.Int64(),
+		Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
+	}
 	rpcPortFlag = cli.IntFlag{
 		Name:  "rpcport",
 		Usage: "HTTP-RPC server listening port",
@@ -168,10 +174,16 @@ Clef that the file is 'safe' to execute.`,
 			signerSecretFlag,
 		},
 		Description: `
-		The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
+The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
-`,
+`}
-	}
+	gendocCommand = cli.Command{
+		Action: GenDoc,
+		Name:   "gendoc",
+		Usage:  "Generate documentation about json-rpc format",
+		Description: `
+The gendoc generates example structures of the json-rpc communication types.
+`}
 )
 
 func init() {
@@ -181,7 +193,7 @@ func init() {
 		logLevelFlag,
 		keystoreFlag,
 		configdirFlag,
-		utils.NetworkIdFlag,
+		chainIdFlag,
 		utils.LightKDFFlag,
 		utils.NoUSBFlag,
 		utils.RPCListenAddrFlag,
@@ -200,7 +212,7 @@ func init() {
 		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, gendocCommand}
 
 }
 func main() {
@@ -341,7 +353,7 @@ func signer(c *cli.Context) error {
 		return err
 	}
 	var (
-		ui core.SignerUI
+		ui core.UIClientAPI
 	)
 	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
@@ -405,14 +417,21 @@ func signer(c *cli.Context) error {
 			}
 		}
 	}
+	var (
+		chainId  = c.GlobalInt64(chainIdFlag.Name)
+		ksLoc    = c.GlobalString(keystoreFlag.Name)
+		lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
+		advanced = c.GlobalBool(advancedMode.Name)
+		nousb    = c.GlobalBool(utils.NoUSBFlag.Name)
+	)
+	log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
+		"light-kdf", lightKdf, "advanced", advanced)
+	am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
+	apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced)
 
-	apiImpl := core.NewSignerAPI(
+	// Establish the bidirectional communication, by creating a new UI backend and registering
-		c.GlobalInt64(utils.NetworkIdFlag.Name),
+	// it with the UI.
-		c.GlobalString(keystoreFlag.Name),
+	ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
-		c.GlobalBool(utils.NoUSBFlag.Name),
-		ui, db,
-		c.GlobalBool(utils.LightKDFFlag.Name),
-		c.GlobalBool(advancedMode.Name))
 	api = apiImpl
 	// Audit logging
 	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
@@ -479,8 +498,8 @@ func signer(c *cli.Context) error {
 	}
 	ui.OnSignerStartup(core.StartupInfo{
 		Info: map[string]interface{}{
-			"extapi_version": ExternalAPIVersion,
+			"extapi_version": core.ExternalAPIVersion,
-			"intapi_version": InternalAPIVersion,
+			"intapi_version": core.InternalAPIVersion,
 			"extapi_http":    extapiURL,
 			"extapi_ipc":     ipcapiURL,
 		},
@@ -532,7 +551,7 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	var (
 		file      string
 		configDir = ctx.GlobalString(configdirFlag.Name)
@@ -629,18 +648,44 @@ func testExternalUI(api *core.SignerAPI) {
 	}
 	var err error
 
+	cliqueHeader := types.Header{
+		common.HexToHash("0000H45H"),
+		common.HexToHash("0000H45H"),
+		common.HexToAddress("0000H45H"),
+		common.HexToHash("0000H00H"),
+		common.HexToHash("0000H45H"),
+		common.HexToHash("0000H45H"),
+		types.Bloom{},
+		big.NewInt(1337),
+		big.NewInt(1337),
+		1338,
+		1338,
+		big.NewInt(1338),
+		[]byte("Extra data Extra data Extra data  Extra data  Extra data  Extra data  Extra data Extra data"),
+		common.HexToHash("0x0000H45H"),
+		types.BlockNonce{},
+	}
+	cliqueRlp, err := rlp.EncodeToBytes(cliqueHeader)
+	if err != nil {
+		utils.Fatalf("Should not error: %v", err)
+	}
+	addr, err := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+	if err != nil {
+		utils.Fatalf("Should not error: %v", err)
+	}
+	_, err = api.SignData(ctx, "application/clique", *addr, cliqueRlp)
+	checkErr("SignData", err)
+
 	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
 	checkErr("SignTransaction", err)
-	_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
+	_, err = api.SignData(ctx, "text/plain", common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
-	checkErr("Sign", err)
+	checkErr("SignData", err)
+	//_, err = api.SignTypedData(ctx, common.MixedcaseAddress{}, core.TypedData{})
+	//checkErr("SignTypedData", err)
 	_, err = api.List(ctx)
 	checkErr("List", err)
 	_, err = api.New(ctx)
 	checkErr("New", err)
-	_, err = api.Export(ctx, common.Address{})
-	checkErr("Export", err)
-	_, err = api.Import(ctx, json.RawMessage{})
-	checkErr("Import", err)
 
 	api.UI.ShowInfo("Tests completed")
 
@@ -652,7 +697,6 @@ func testExternalUI(api *core.SignerAPI) {
 	} else {
 		log.Info("No errors")
 	}
-
 }
 
 // getPassPhrase retrieves the password associated with clef, either fetched
@@ -708,6 +752,154 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
 	return seed, err
 }
 
+// GenDoc outputs examples of all structures used in json-rpc communication
+func GenDoc(ctx *cli.Context) {
+
+	var (
+		a    = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
+		b    = common.HexToAddress("0x1111111122222222222233333333334444444444")
+		meta = core.Metadata{
+			Scheme:    "http",
+			Local:     "localhost:8545",
+			Origin:    "www.malicious.ru",
+			Remote:    "localhost:9999",
+			UserAgent: "Firefox 3.2",
+		}
+		output []string
+		add    = func(name, desc string, v interface{}) {
+			if data, err := json.MarshalIndent(v, "", "  "); err == nil {
+				output = append(output, fmt.Sprintf("### %s\n\n%s\n\nExample:\n```json\n%s\n```", name, desc, data))
+			} else {
+				log.Error("Error generating output", err)
+			}
+		}
+	)
+
+	{ // Sign plain text request
+		desc := "SignDataRequest contains information about a pending request to sign some data. " +
+			"The data to be signed can be of various types, defined by content-type. Clef has done most " +
+			"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
+			"the user with the contents of the `message`"
+		sighash, msg := accounts.TextAndHash([]byte("hello world"))
+		message := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
+
+		add("SignDataRequest", desc, &core.SignDataRequest{
+			Address:     common.NewMixedcaseAddress(a),
+			Meta:        meta,
+			ContentType: accounts.MimetypeTextPlain,
+			Rawdata:     []byte(msg),
+			Message:     message,
+			Hash:        sighash})
+	}
+	{ // Sign plain text response
+		add("SignDataResponse - approve", "Response to SignDataRequest",
+			&core.SignDataResponse{Password: "apassword", Approved: true})
+		add("SignDataResponse - deny", "Response to SignDataRequest",
+			&core.SignDataResponse{})
+	}
+	{ // Sign transaction request
+		desc := "SignTxRequest contains information about a pending request to sign a transaction. " +
+			"Aside from the transaction itself, there is also a `call_info`-struct. That struct contains " +
+			"messages of various types, that the user should be informed of." +
+			"\n\n" +
+			"As in any request, it's important to consider that the `meta` info also contains untrusted data." +
+			"\n\n" +
+			"The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, " +
+			"they must be identical, otherwise an error is generated. " +
+			"However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)"
+
+		data := hexutil.Bytes([]byte{0x01, 0x02, 0x03, 0x04})
+		add("SignTxRequest", desc, &core.SignTxRequest{
+			Meta: meta,
+			Callinfo: []core.ValidationInfo{
+				{"Warning", "Something looks odd, show this message as a warning"},
+				{"Info", "User should see this aswell"},
+			},
+			Transaction: core.SendTxArgs{
+				Data:     &data,
+				Nonce:    0x1,
+				Value:    hexutil.Big(*big.NewInt(6)),
+				From:     common.NewMixedcaseAddress(a),
+				To:       nil,
+				GasPrice: hexutil.Big(*big.NewInt(5)),
+				Gas:      1000,
+				Input:    nil,
+			}})
+	}
+	{ // Sign tx response
+		data := hexutil.Bytes([]byte{0x04, 0x03, 0x02, 0x01})
+		add("SignDataResponse - approve", "Response to SignDataRequest. This response needs to contain the `transaction`"+
+			", because the UI is free to make modifications to the transaction.",
+			&core.SignTxResponse{Password: "apassword", Approved: true,
+				Transaction: core.SendTxArgs{
+					Data:     &data,
+					Nonce:    0x4,
+					Value:    hexutil.Big(*big.NewInt(6)),
+					From:     common.NewMixedcaseAddress(a),
+					To:       nil,
+					GasPrice: hexutil.Big(*big.NewInt(5)),
+					Gas:      1000,
+					Input:    nil,
+				}})
+		add("SignDataResponse - deny", "Response to SignDataRequest. When denying a request, there's no need to "+
+			"provide the transaction in return",
+			&core.SignDataResponse{})
+	}
+	{ // WHen a signed tx is ready to go out
+		desc := "SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`" +
+			"\n\n" +
+			"This occurs _after_ successful completion of the entire signing procedure, but right before the signed " +
+			"transaction is passed to the external caller. This method (and data) can be used by the UI to signal " +
+			"to the user that the transaction was signed, but it is primarily useful for ruleset implementations." +
+			"\n\n" +
+			"A ruleset that implements a rate limitation needs to know what transactions are sent out to the external " +
+			"interface. By hooking into this methods, the ruleset can maintain track of that count." +
+			"\n\n" +
+			"**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time" +
+			" (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. " +
+			"\n\n" +
+			"The `OnApproved` method cannot be responded to, it's purely informative"
+
+		rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed")
+		var tx types.Transaction
+		rlp.DecodeBytes(rlpdata, &tx)
+		add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
+
+	}
+	{ // User input
+		add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
+			&core.UserInputRequest{IsPassword: true, Title: "The title here", Prompt: "The question to ask the user"})
+		add("UserInputResponse", "Response to SignDataRequest",
+			&core.UserInputResponse{Text: "The textual response from user"})
+	}
+	{ // List request
+		add("ListRequest", "Sent when a request has been made to list addresses. The UI is provided with the "+
+			"full `account`s, including local directory names. Note: this information is not passed back to the external caller, "+
+			"who only sees the `address`es. ",
+			&core.ListRequest{
+				Meta: meta,
+				Accounts: []accounts.Account{
+					{a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
+					{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
+			})
+
+		add("UserInputResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
+			"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
+			&core.ListResponse{
+				Accounts: []accounts.Account{
+					{common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}},
+					{common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}},
+				}})
+	}
+
+	fmt.Println(`## UI Client interface
+
+These data types are defined in the channel between clef and the UI`)
+	for _, elem := range output {
+		fmt.Println(elem)
+	}
+}
+
 /**
 //Create Account
 

cmd/clef/tests/testsigner.js

@@ -0,0 +1,73 @@
+// This file is a test-utility for testing clef-functionality
+//
+// Start clef with
+//
+// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
+//
+// Start geth with
+//
+// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
+//
+// and in the console simply invoke
+//
+// > test()
+//
+// You can reload the file via `reload()`
+
+function reload(){
+	loadScript("./cmd/clef/tests/testsigner.js");
+}
+
+function init(){
+    if (typeof accts == 'undefined' || accts.length == 0){
+        accts = eth.accounts
+        console.log("Got accounts ", accts);
+    }
+}
+init()
+function testTx(){
+    if( accts && accts.length > 0) {
+        var a = accts[0]
+        var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
+        var v = parseInt(txdata.tx.v)
+        console.log("V value: ", v)
+        if (v == 37 || v == 38){
+            console.log("Mainnet 155-protected chainid was used")
+        }
+        if (v == 27 || v == 28){
+            throw new Error("Mainnet chainid was used, but without replay protection!")
+        }
+    }
+}
+function testSignText(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
+        console.log("signing response",  r)
+    }
+}
+function testClique(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
+        console.log("signing response",  r)
+        if( a != r){
+            throw new Error("Requested signing by "+a+ " but got sealer "+r)
+        }
+    }
+}
+
+function test(){
+    var tests = [
+        testTx,
+        testSignText,
+        testClique,
+    ]
+    for( i in tests){
+        try{
+            tests[i]()
+        }catch(err){
+            console.log(err)
+        }
+    }
+ }

cmd/faucet/faucet.go

@@ -282,7 +282,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 
 // close terminates the Ethereum connection and tears down the faucet.
 func (f *faucet) close() error {
-	return f.stack.Stop()
+	return f.stack.Close()
 }
 
 // listenAndServe registers the HTTP handlers for the faucet and boots it up

cmd/geth/chaincmd.go

@@ -190,6 +190,8 @@ func initGenesis(ctx *cli.Context) error {
 	}
 	// Open an initialise both full and light databases
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	for _, name := range []string{"chaindata", "lightchaindata"} {
 		chaindb, err := stack.OpenDatabase(name, 0, 0)
 		if err != nil {
@@ -209,6 +211,8 @@ func importChain(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	chain, chainDb := utils.MakeChain(ctx, stack)
 	defer chainDb.Close()
 
@@ -303,6 +307,8 @@ func exportChain(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	chain, _ := utils.MakeChain(ctx, stack)
 	start := time.Now()
 
@@ -336,9 +342,11 @@ func importPreimages(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+	defer stack.Close()
 
+	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
 	start := time.Now()
+
 	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
 		utils.Fatalf("Import error: %v\n", err)
 	}
@@ -352,9 +360,11 @@ func exportPreimages(ctx *cli.Context) error {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
-	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+	defer stack.Close()
 
+	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
 	start := time.Now()
+
 	if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil {
 		utils.Fatalf("Export error: %v\n", err)
 	}
@@ -369,8 +379,9 @@ func copyDb(ctx *cli.Context) error {
 	}
 	// Initialize a new chain for the running node to sync into
 	stack := makeFullNode(ctx)
-	chain, chainDb := utils.MakeChain(ctx, stack)
+	defer stack.Close()
 
+	chain, chainDb := utils.MakeChain(ctx, stack)
 	syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
 	dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
 
@@ -441,6 +452,8 @@ func removeDB(ctx *cli.Context) error {
 
 func dump(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
+	defer stack.Close()
+
 	chain, chainDb := utils.MakeChain(ctx, stack)
 	for _, arg := range ctx.Args() {
 		var block *types.Block

cmd/geth/consolecmd.go

@@ -79,7 +79,7 @@ func localConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
 	node := makeFullNode(ctx)
 	startNode(ctx, node)
-	defer node.Stop()
+	defer node.Close()
 
 	// Attach to the newly started node and start the JavaScript console
 	client, err := node.Attach()
@@ -180,7 +180,7 @@ func ephemeralConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
 	node := makeFullNode(ctx)
 	startNode(ctx, node)
-	defer node.Stop()
+	defer node.Close()
 
 	// Attach to the newly started node and start the JavaScript console
 	client, err := node.Attach()

cmd/geth/main.go

@@ -40,7 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 const (
@@ -62,6 +62,7 @@ var (
 		utils.BootnodesV5Flag,
 		utils.DataDirFlag,
 		utils.KeyStoreDirFlag,
+		utils.ExternalSignerFlag,
 		utils.NoUSBFlag,
 		utils.DashboardEnabledFlag,
 		utils.DashboardAddrFlag,
@@ -128,6 +129,7 @@ var (
 		utils.DeveloperPeriodFlag,
 		utils.TestnetFlag,
 		utils.RinkebyFlag,
+		utils.GoerliFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.ConstantinopleOverrideFlag,
@@ -176,7 +178,7 @@ var (
 		utils.MetricsInfluxDBDatabaseFlag,
 		utils.MetricsInfluxDBUsernameFlag,
 		utils.MetricsInfluxDBPasswordFlag,
-		utils.MetricsInfluxDBHostTagFlag,
+		utils.MetricsInfluxDBTagsFlag,
 	}
 )
 
@@ -277,6 +279,7 @@ func geth(ctx *cli.Context) error {
 		return fmt.Errorf("invalid command: %q", args[0])
 	}
 	node := makeFullNode(ctx)
+	defer node.Close()
 	startNode(ctx, node)
 	node.Wait()
 	return nil
@@ -292,13 +295,14 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	utils.StartNode(stack)
 
 	// Unlock any account specifically requested
-	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
-
+		ks := keystores[0].(*keystore.KeyStore)
-	passwords := utils.MakePasswordList(ctx)
+		passwords := utils.MakePasswordList(ctx)
-	unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
+		unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
-	for i, account := range unlocks {
+		for i, account := range unlocks {
-		if trimmed := strings.TrimSpace(account); trimmed != "" {
+			if trimmed := strings.TrimSpace(account); trimmed != "" {
-			unlockAccount(ctx, ks, trimmed, i, passwords)
+				unlockAccount(ctx, ks, trimmed, i, passwords)
+			}
 		}
 	}
 	// Register wallet event handlers to open and auto-derive wallets

cmd/geth/monitorcmd.go

@@ -169,7 +169,7 @@ func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) {
 // resolveMetrics takes a list of input metric patterns, and resolves each to one
 // or more canonical metric names.
 func resolveMetrics(metrics map[string]interface{}, patterns []string) []string {
-	res := []string{}
+	var res []string
 	for _, pattern := range patterns {
 		res = append(res, resolveMetric(metrics, pattern, "")...)
 	}
@@ -179,7 +179,7 @@ func resolveMetrics(metrics map[string]interface{}, patterns []string) []string
 // resolveMetrics takes a single of input metric pattern, and resolves it to one
 // or more canonical metric names.
 func resolveMetric(metrics map[string]interface{}, pattern string, path string) []string {
-	results := []string{}
+	var results []string
 
 	// If a nested metric was requested, recurse optionally branching (via comma)
 	parts := strings.SplitN(pattern, "/", 2)
@@ -215,7 +215,7 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
 // expandMetrics expands the entire tree of metrics into a flat list of paths.
 func expandMetrics(metrics map[string]interface{}, path string) []string {
 	// Iterate over all fields and expand individually
-	list := []string{}
+	var list []string
 	for name, metric := range metrics {
 		switch metric := metric.(type) {
 		case float64:

cmd/geth/usage.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/internal/debug"
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 // AppHelpTemplate is the test template for the default, global app help topic.
@@ -74,6 +74,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NetworkIdFlag,
 			utils.TestnetFlag,
 			utils.RinkebyFlag,
+			utils.GoerliFlag,
 			utils.SyncModeFlag,
 			utils.ExitWhenSyncedFlag,
 			utils.GCModeFlag,
@@ -144,6 +145,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.UnlockedAccountFlag,
 			utils.PasswordFileFlag,
+			utils.ExternalSignerFlag,
 		},
 	},
 	{
@@ -230,7 +232,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.MetricsInfluxDBDatabaseFlag,
 			utils.MetricsInfluxDBUsernameFlag,
 			utils.MetricsInfluxDBPasswordFlag,
-			utils.MetricsInfluxDBHostTagFlag,
+			utils.MetricsInfluxDBTagsFlag,
 		},
 	},
 	{
@@ -306,7 +308,7 @@ func init() {
 					categorized[flag.String()] = struct{}{}
 				}
 			}
-			uncategorized := []cli.Flag{}
+			var uncategorized []cli.Flag
 			for _, flag := range data.(*cli.App).Flags {
 				if _, ok := categorized[flag.String()]; !ok {
 					if strings.HasPrefix(flag.GetName(), "dashboard") {

cmd/puppeth/wizard_genesis.go

@@ -222,14 +222,18 @@ func (w *wizard) manageGenesis() {
 		fmt.Println()
 		fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock)
 		w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock)
-
+		if w.conf.Genesis.Config.PetersburgBlock == nil {
+			w.conf.Genesis.Config.PetersburgBlock = w.conf.Genesis.Config.ConstantinopleBlock
+		}
 		fmt.Println()
-		fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock)
+		fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.PetersburgBlock)
-		w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock)
+		w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.PetersburgBlock)
 
 		out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", "  ")
 		fmt.Printf("Chain configuration updated:\n\n%s\n", out)
 
+		w.conf.flush()
+
 	case "2":
 		// Save whatever genesis configuration we currently have
 		fmt.Println()

cmd/swarm/access.go

@@ -178,8 +178,8 @@ func accessNewACT(ctx *cli.Context) {
 		accessKey            []byte
 		err                  error
 		ref                  = args[0]
-		pkGrantees           = []string{}
+		pkGrantees           []string
-		passGrantees         = []string{}
+		passGrantees         []string
 		pkGranteesFilename   = ctx.String(SwarmAccessGrantKeysFlag.Name)
 		passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
 		privateKey           = getPrivKey(ctx)

cmd/swarm/access_test.go

@@ -397,7 +397,7 @@ func testACT(t *testing.T, bogusEntries int) {
 	}
 
 	ref := matches[0]
-	grantees := []string{}
+	var grantees []string
 	for i, v := range cluster.Nodes {
 		if i == nodeToSkip {
 			continue
@@ -408,7 +408,7 @@ func testACT(t *testing.T, bogusEntries int) {
 	}
 
 	if bogusEntries > 0 {
-		bogusGrantees := []string{}
+		var bogusGrantees []string
 
 		for i := 0; i < bogusEntries; i++ {
 			prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)

cmd/swarm/config.go

@@ -82,6 +82,7 @@ const (
 	SWARM_ENV_BOOTNODE_MODE           = "SWARM_BOOTNODE_MODE"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
 	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
+	SWARM_GLOBALSTORE_API             = "SWARM_GLOBALSTORE_API"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 
@@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name)
 	}
 
+	if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) {
+		currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name)
+	}
+
 	return currentConfig
 
 }
@@ -375,6 +380,10 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		currentConfig.BootnodeMode = bootnodeMode
 	}
 
+	if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" {
+		currentConfig.GlobalStoreAPI = api
+	}
+
 	return currentConfig
 }
 

cmd/swarm/explore.go

@@ -0,0 +1,59 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command bzzhash computes a swarm tree hash.
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var hashesCommand = cli.Command{
+	Action:             hashes,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "hashes",
+	Usage:              "print all hashes of a file to STDOUT",
+	ArgsUsage:          "<file>",
+	Description:        "Prints all hashes of a file to STDOUT",
+}
+
+func hashes(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) < 1 {
+		utils.Fatalf("Usage: swarm hashes <file name>")
+	}
+	f, err := os.Open(args[0])
+	if err != nil {
+		utils.Fatalf("Error opening file " + args[1])
+	}
+	defer f.Close()
+
+	fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams())
+	refs, err := fileStore.GetAllReferences(context.TODO(), f, false)
+	if err != nil {
+		utils.Fatalf("%v\n", err)
+	} else {
+		for _, r := range refs {
+			fmt.Println(r.String())
+		}
+	}
+}

cmd/swarm/flags.go

@@ -176,4 +176,9 @@ var (
 		Name:  "user",
 		Usage: "Indicates the user who updates the feed",
 	}
+	SwarmGlobalStoreAPIFlag = cli.StringFlag{
+		Name:   "globalstore-api",
+		Usage:  "URL of the Global Store API provider (only for testing)",
+		EnvVar: SWARM_GLOBALSTORE_API,
+	}
 )

cmd/swarm/fs.go

@@ -123,7 +123,7 @@ func listMounts(cliContext *cli.Context) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	mf := []fuse.MountInfo{}
+	var mf []fuse.MountInfo
 	err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
 	if err != nil {
 		utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)

cmd/swarm/global-store/global_store.go

@@ -0,0 +1,100 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+// startHTTP starts a global store with HTTP RPC server.
+// It is used for "http" cli command.
+func startHTTP(ctx *cli.Context) (err error) {
+	server, cleanup, err := newServer(ctx)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	listener, err := net.Listen("tcp", ctx.String("addr"))
+	if err != nil {
+		return err
+	}
+	log.Info("http", "address", listener.Addr().String())
+
+	return http.Serve(listener, server)
+}
+
+// startWS starts a global store with WebSocket RPC server.
+// It is used for "websocket" cli command.
+func startWS(ctx *cli.Context) (err error) {
+	server, cleanup, err := newServer(ctx)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	listener, err := net.Listen("tcp", ctx.String("addr"))
+	if err != nil {
+		return err
+	}
+	origins := ctx.StringSlice("origins")
+	log.Info("websocket", "address", listener.Addr().String(), "origins", origins)
+
+	return http.Serve(listener, server.WebsocketHandler(origins))
+}
+
+// newServer creates a global store and returns its RPC server.
+// Returned cleanup function should be called only if err is nil.
+func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
+
+	cleanup = func() {}
+	var globalStore mock.GlobalStorer
+	dir := ctx.String("dir")
+	if dir != "" {
+		dbStore, err := db.NewGlobalStore(dir)
+		if err != nil {
+			return nil, nil, err
+		}
+		cleanup = func() {
+			dbStore.Close()
+		}
+		globalStore = dbStore
+		log.Info("database global store", "dir", dir)
+	} else {
+		globalStore = mem.NewGlobalStore()
+		log.Info("in-memory global store")
+	}
+
+	server = rpc.NewServer()
+	if err := server.RegisterName("mockStore", globalStore); err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+
+	return server, cleanup, nil
+}

cmd/swarm/global-store/global_store_test.go

@@ -0,0 +1,191 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
+)
+
+// TestHTTP_InMemory tests in-memory global store that exposes
+// HTTP server.
+func TestHTTP_InMemory(t *testing.T) {
+	testHTTP(t, true)
+}
+
+// TestHTTP_Database tests global store with persisted database
+// that exposes HTTP server.
+func TestHTTP_Database(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-global-store-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// create a fresh global store
+	testHTTP(t, true, "--dir", dir)
+
+	// check if data saved by the previous global store instance
+	testHTTP(t, false, "--dir", dir)
+}
+
+// testWebsocket starts global store binary with HTTP server
+// and validates that it can store and retrieve data.
+// If put is false, no data will be stored, only retrieved,
+// giving the possibility to check if data is present in the
+// storage directory.
+func testHTTP(t *testing.T, put bool, args ...string) {
+	addr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...)
+	defer testCmd.Interrupt()
+
+	client, err := rpc.DialHTTP("http://" + addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// wait until global store process is started as
+	// rpc.DialHTTP is actually not connecting
+	for i := 0; i < 1000; i++ {
+		_, err = http.DefaultClient.Get("http://" + addr)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	node := store.NewNodeStore(common.HexToAddress("123abc"))
+
+	wantKey := "key"
+	wantValue := "value"
+
+	if put {
+		err = node.Put([]byte(wantKey), []byte(wantValue))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	gotValue, err := node.Get([]byte(wantKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(gotValue) != wantValue {
+		t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
+	}
+}
+
+// TestWebsocket_InMemory tests in-memory global store that exposes
+// WebSocket server.
+func TestWebsocket_InMemory(t *testing.T) {
+	testWebsocket(t, true)
+}
+
+// TestWebsocket_Database tests global store with persisted database
+// that exposes HTTP server.
+func TestWebsocket_Database(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-global-store-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// create a fresh global store
+	testWebsocket(t, true, "--dir", dir)
+
+	// check if data saved by the previous global store instance
+	testWebsocket(t, false, "--dir", dir)
+}
+
+// testWebsocket starts global store binary with WebSocket server
+// and validates that it can store and retrieve data.
+// If put is false, no data will be stored, only retrieved,
+// giving the possibility to check if data is present in the
+// storage directory.
+func testWebsocket(t *testing.T, put bool, args ...string) {
+	addr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
+	defer testCmd.Interrupt()
+
+	var client *rpc.Client
+	var err error
+	// wait until global store process is started
+	for i := 0; i < 1000; i++ {
+		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	node := store.NewNodeStore(common.HexToAddress("123abc"))
+
+	wantKey := "key"
+	wantValue := "value"
+
+	if put {
+		err = node.Put([]byte(wantKey), []byte(wantValue))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	gotValue, err := node.Get([]byte(wantKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(gotValue) != wantValue {
+		t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
+	}
+}
+
+// findFreeTCPAddress returns a local address (IP:Port) to which
+// global store can listen on.
+func findFreeTCPAddress(t *testing.T) (addr string) {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer listener.Close()
+
+	return listener.Addr().String()
+}

cmd/swarm/global-store/main.go

@@ -0,0 +1,104 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/log"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+
+func main() {
+	err := newApp().Run(os.Args)
+	if err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+// newApp construct a new instance of Swarm Global Store.
+// Method Run is called on it in the main function and in tests.
+func newApp() (app *cli.App) {
+	app = utils.NewApp(gitCommit, "Swarm Global Store")
+
+	app.Name = "global-store"
+
+	// app flags (for all commands)
+	app.Flags = []cli.Flag{
+		cli.IntFlag{
+			Name:  "verbosity",
+			Value: 3,
+			Usage: "verbosity level",
+		},
+	}
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "http",
+			Aliases: []string{"h"},
+			Usage:   "start swarm global store with http server",
+			Action:  startHTTP,
+			// Flags only for "start" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.StringFlag{
+					Name:  "dir",
+					Value: "",
+					Usage: "data directory",
+				},
+				cli.StringFlag{
+					Name:  "addr",
+					Value: "0.0.0.0:3033",
+					Usage: "address to listen for http connection",
+				},
+			),
+		},
+		{
+			Name:    "websocket",
+			Aliases: []string{"ws"},
+			Usage:   "start swarm global store with websocket server",
+			Action:  startWS,
+			// Flags only for "start" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.StringFlag{
+					Name:  "dir",
+					Value: "",
+					Usage: "data directory",
+				},
+				cli.StringFlag{
+					Name:  "addr",
+					Value: "0.0.0.0:3033",
+					Usage: "address to listen for websocket connection",
+				},
+				cli.StringSliceFlag{
+					Name:  "origins",
+					Value: &cli.StringSlice{"*"},
+					Usage: "websocket origins",
+				},
+			),
+		},
+	}
+
+	return app
+}

cmd/swarm/global-store/run_test.go

@@ -0,0 +1,49 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
+)
+
+func init() {
+	reexec.Register("swarm-global-store", func() {
+		if err := newApp().Run(os.Args); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	})
+}
+
+func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd {
+	tt := cmdtest.NewTestCmd(t, nil)
+	tt.Run("swarm-global-store", args...)
+	return tt
+}
+
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
+}

cmd/swarm/main.go

@@ -39,13 +39,16 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
 	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
 	"github.com/ethereum/go-ethereum/swarm/tracing"
 	sv "github.com/ethereum/go-ethereum/swarm/version"
 
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 const clientIdentifier = "swarm"
@@ -142,6 +145,8 @@ func init() {
 		dbCommand,
 		// See config.go
 		DumpConfigCommand,
+		// hashesCommand
+		hashesCommand,
 	}
 
 	// append a hidden help subcommand to all commands that have subcommands
@@ -194,6 +199,7 @@ func init() {
 		SwarmStorePath,
 		SwarmStoreCapacity,
 		SwarmStoreCacheCapacity,
+		SwarmGlobalStoreAPIFlag,
 	}
 	rpcFlags := []cli.Flag{
 		utils.WSEnabledFlag,
@@ -288,6 +294,7 @@ func bzzd(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("can't create node: %v", err)
 	}
+	defer stack.Close()
 
 	//a few steps need to be done after the config phase is completed,
 	//due to overriding behavior
@@ -322,8 +329,18 @@ func bzzd(ctx *cli.Context) error {
 func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
 	//define the swarm service boot function
 	boot := func(_ *node.ServiceContext) (node.Service, error) {
-		// In production, mockStore must be always nil.
+		var nodeStore *mock.NodeStore
-		return swarm.NewSwarm(bzzconfig, nil)
+		if bzzconfig.GlobalStoreAPI != "" {
+			// connect to global store
+			client, err := rpc.Dial(bzzconfig.GlobalStoreAPI)
+			if err != nil {
+				return nil, fmt.Errorf("global store: %v", err)
+			}
+			globalStore := mockrpc.NewGlobalStore(client)
+			// create a node store for this swarm key on global store
+			nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey))
+		}
+		return swarm.NewSwarm(bzzconfig, nodeStore)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {
@@ -365,6 +382,8 @@ func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey {
 	if err != nil {
 		utils.Fatalf("can't create node: %v", err)
 	}
+	defer stack.Close()
+
 	return getAccount(bzzconfig.BzzAccount, ctx, stack)
 }
 
@@ -449,5 +468,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
 		}
 		cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node)
 	}
-	log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes))
+
 }

cmd/swarm/swarm-smoke/feed_upload_and_sync.go

@@ -2,13 +2,10 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"crypto/md5"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
-	"net/http/httptrace"
 	"os"
 	"os/exec"
 	"strings"
@@ -19,12 +16,8 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
-	colorable "github.com/mattn/go-colorable"
-	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pborman/uuid"
 	cli "gopkg.in/urfave/cli.v1"
 )
@@ -33,34 +26,28 @@ const (
 	feedRandomDataLength = 8
 )
 
-func cliFeedUploadAndSync(c *cli.Context) error {
+func feedUploadAndSyncCmd(ctx *cli.Context, tuid string) error {
-	metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1)
-	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))))
-
 	errc := make(chan error)
+
 	go func() {
-		errc <- feedUploadAndSync(c)
+		errc <- feedUploadAndSync(ctx, tuid)
 	}()
 
 	select {
 	case err := <-errc:
 		if err != nil {
-			metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1)
+			metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
 		}
 		return err
 	case <-time.After(time.Duration(timeout) * time.Second):
-		metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1)
+		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
+
 		return fmt.Errorf("timeout after %v sec", timeout)
 	}
 }
 
-// TODO: retrieve with manifest + extract repeating code
+func feedUploadAndSync(c *cli.Context, tuid string) error {
-func feedUploadAndSync(c *cli.Context) error {
+	log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing")
-	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
-
-	generateEndpoints(scheme, cluster, appName, from, to)
-
-	log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing")
 
 	// create a random private key to sign updates with and derive the address
 	pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
@@ -114,7 +101,7 @@ func feedUploadAndSync(c *cli.Context) error {
 
 	// create feed manifest, topic only
 	var out bytes.Buffer
-	cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex)
+	cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex)
 	cmd.Stdout = &out
 	log.Debug("create feed manifest topic cmd", "cmd", cmd)
 	err = cmd.Run()
@@ -129,7 +116,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	out.Reset()
 
 	// create feed manifest, subtopic only
-	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex)
+	cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex)
 	cmd.Stdout = &out
 	log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
 	err = cmd.Run()
@@ -144,7 +131,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	out.Reset()
 
 	// create feed manifest, merged topic
-	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
+	cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
 	cmd.Stdout = &out
 	log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
 	err = cmd.Run()
@@ -170,7 +157,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	dataHex := hexutil.Encode(data)
 
 	// update with topic
-	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex)
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex)
 	cmd.Stdout = &out
 	log.Debug("update feed manifest topic cmd", "cmd", cmd)
 	err = cmd.Run()
@@ -181,7 +168,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	out.Reset()
 
 	// update with subtopic
-	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex)
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex)
 	cmd.Stdout = &out
 	log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
 	err = cmd.Run()
@@ -192,7 +179,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	out.Reset()
 
 	// update with merged topic
-	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
 	cmd.Stdout = &out
 	log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
 	err = cmd.Run()
@@ -206,14 +193,14 @@ func feedUploadAndSync(c *cli.Context) error {
 
 	// retrieve the data
 	wg := sync.WaitGroup{}
-	for _, endpoint := range endpoints {
+	for _, host := range hosts {
 		// raw retrieve, topic only
 		for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
 			wg.Add(1)
 			ruid := uuid.New()[:8]
 			go func(hex string, endpoint string, ruid string) {
 				for {
-					err := fetchFeed(hex, userHex, endpoint, dataHash, ruid)
+					err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid)
 					if err != nil {
 						continue
 					}
@@ -221,20 +208,18 @@ func feedUploadAndSync(c *cli.Context) error {
 					wg.Done()
 					return
 				}
-			}(hex, endpoint, ruid)
+			}(hex, httpEndpoint(host), ruid)
-
 		}
 	}
 	wg.Wait()
 	log.Info("all endpoints synced random data successfully")
 
 	// upload test file
-	seed := int(time.Now().UnixNano() / 1e6)
+	log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
-	log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed)
 
 	randomBytes := testutil.RandomBytes(seed, filesize*1000)
 
-	hash, err := upload(&randomBytes, endpoints[0])
+	hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
 	if err != nil {
 		return err
 	}
@@ -243,15 +228,12 @@ func feedUploadAndSync(c *cli.Context) error {
 		return err
 	}
 	multihashHex := hexutil.Encode(hashBytes)
-	fileHash, err := digest(bytes.NewReader(randomBytes))
+	fileHash := h.Sum(nil)
-	if err != nil {
-		return err
-	}
 
 	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
 
 	// update file with topic
-	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex)
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex)
 	cmd.Stdout = &out
 	err = cmd.Run()
 	if err != nil {
@@ -261,7 +243,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	out.Reset()
 
 	// update file with subtopic
-	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex)
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex)
 	cmd.Stdout = &out
 	err = cmd.Run()
 	if err != nil {
@@ -271,7 +253,7 @@ func feedUploadAndSync(c *cli.Context) error {
 	out.Reset()
 
 	// update file with merged topic
-	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
 	cmd.Stdout = &out
 	err = cmd.Run()
 	if err != nil {
@@ -282,7 +264,7 @@ func feedUploadAndSync(c *cli.Context) error {
 
 	time.Sleep(3 * time.Second)
 
-	for _, endpoint := range endpoints {
+	for _, host := range hosts {
 
 		// manifest retrieve, topic only
 		for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
@@ -290,7 +272,7 @@ func feedUploadAndSync(c *cli.Context) error {
 			ruid := uuid.New()[:8]
 			go func(url string, endpoint string, ruid string) {
 				for {
-					err := fetch(url, endpoint, fileHash, ruid)
+					err := fetch(url, endpoint, fileHash, ruid, "")
 					if err != nil {
 						continue
 					}
@@ -298,7 +280,7 @@ func feedUploadAndSync(c *cli.Context) error {
 					wg.Done()
 					return
 				}
-			}(url, endpoint, ruid)
+			}(url, httpEndpoint(host), ruid)
 		}
 
 	}
@@ -307,60 +289,3 @@ func feedUploadAndSync(c *cli.Context) error {
 
 	return nil
 }
-
-func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
-	ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
-	defer sp.Finish()
-
-	log.Trace("sleeping", "ruid", ruid)
-	time.Sleep(3 * time.Second)
-
-	log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
-
-	var tn time.Time
-	reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
-	req, _ := http.NewRequest("GET", reqUri, nil)
-
-	opentracing.GlobalTracer().Inject(
-		sp.Context(),
-		opentracing.HTTPHeaders,
-		opentracing.HTTPHeadersCarrier(req.Header))
-
-	trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
-
-	req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
-	transport := http.DefaultTransport
-
-	//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-
-	tn = time.Now()
-	res, err := transport.RoundTrip(req)
-	if err != nil {
-		log.Error(err.Error(), "ruid", ruid)
-		return err
-	}
-
-	log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
-
-	if res.StatusCode != 200 {
-		return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
-	}
-
-	defer res.Body.Close()
-
-	rdigest, err := digest(res.Body)
-	if err != nil {
-		log.Warn(err.Error(), "ruid", ruid)
-		return err
-	}
-
-	if !bytes.Equal(rdigest, original) {
-		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
-		log.Warn(err.Error(), "ruid", ruid)
-		return err
-	}
-
-	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
-
-	return nil
-}

cmd/swarm/swarm-smoke/main.go

@@ -37,18 +37,16 @@ var (
 )
 
 var (
-	endpoints        []string
+	allhosts     string
-	includeLocalhost bool
+	hosts        []string
-	cluster          string
+	filesize     int
-	appName          string
+	syncDelay    int
-	scheme           string
+	httpPort     int
-	filesize         int
+	wsPort       int
-	syncDelay        int
+	verbosity    int
-	from             int
+	timeout      int
-	to               int
+	single       bool
-	verbosity        int
+	trackTimeout int
-	timeout          int
-	single           bool
 )
 
 func main() {
@@ -59,39 +57,22 @@ func main() {
 
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
-			Name:        "cluster-endpoint",
+			Name:        "hosts",
-			Value:       "prod",
+			Value:       "",
-			Usage:       "cluster to point to (prod or a given namespace)",
+			Usage:       "comma-separated list of swarm hosts",
-			Destination: &cluster,
+			Destination: &allhosts,
-		},
-		cli.StringFlag{
-			Name:        "app",
-			Value:       "swarm",
-			Usage:       "application to point to (swarm or swarm-private)",
-			Destination: &appName,
 		},
 		cli.IntFlag{
-			Name:        "cluster-from",
+			Name:        "http-port",
-			Value:       8501,
+			Value:       80,
-			Usage:       "swarm node (from)",
+			Usage:       "http port",
-			Destination: &from,
+			Destination: &httpPort,
 		},
 		cli.IntFlag{
-			Name:        "cluster-to",
+			Name:        "ws-port",
-			Value:       8512,
+			Value:       8546,
-			Usage:       "swarm node (to)",
+			Usage:       "ws port",
-			Destination: &to,
+			Destination: &wsPort,
-		},
-		cli.StringFlag{
-			Name:        "cluster-scheme",
-			Value:       "http",
-			Usage:       "http or https",
-			Destination: &scheme,
-		},
-		cli.BoolFlag{
-			Name:        "include-localhost",
-			Usage:       "whether to include localhost:8500 as an endpoint",
-			Destination: &includeLocalhost,
 		},
 		cli.IntFlag{
 			Name:        "filesize",
@@ -122,6 +103,12 @@ func main() {
 			Usage:       "whether to fetch content from a single node or from all nodes",
 			Destination: &single,
 		},
+		cli.IntFlag{
+			Name:        "track-timeout",
+			Value:       5,
+			Usage:       "timeout in seconds to wait for GetAllReferences to return",
+			Destination: &trackTimeout,
+		},
 	}
 
 	app.Flags = append(app.Flags, []cli.Flag{
@@ -130,7 +117,7 @@ func main() {
 		swarmmetrics.MetricsInfluxDBDatabaseFlag,
 		swarmmetrics.MetricsInfluxDBUsernameFlag,
 		swarmmetrics.MetricsInfluxDBPasswordFlag,
-		swarmmetrics.MetricsInfluxDBHostTagFlag,
+		swarmmetrics.MetricsInfluxDBTagsFlag,
 	}...)
 
 	app.Flags = append(app.Flags, tracing.Flags...)
@@ -140,19 +127,25 @@ func main() {
 			Name:    "upload_and_sync",
 			Aliases: []string{"c"},
 			Usage:   "upload and sync",
-			Action:  cliUploadAndSync,
+			Action:  wrapCliCommand("upload-and-sync", uploadAndSyncCmd),
 		},
 		{
 			Name:    "feed_sync",
 			Aliases: []string{"f"},
 			Usage:   "feed update generate, upload and sync",
-			Action:  cliFeedUploadAndSync,
+			Action:  wrapCliCommand("feed-and-sync", feedUploadAndSyncCmd),
 		},
 		{
 			Name:    "upload_speed",
 			Aliases: []string{"u"},
 			Usage:   "measure upload speed",
-			Action:  cliUploadSpeed,
+			Action:  wrapCliCommand("upload-speed", uploadSpeedCmd),
+		},
+		{
+			Name:    "sliding_window",
+			Aliases: []string{"s"},
+			Usage:   "measure network aggregate capacity",
+			Action:  wrapCliCommand("sliding-window", slidingWindowCmd),
 		},
 	}
 
@@ -183,13 +176,14 @@ func emitMetrics(ctx *cli.Context) error {
 			database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name)
 			username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name)
 			password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name)
-			hosttag  = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name)
+			tags     = ctx.GlobalString(swarmmetrics.MetricsInfluxDBTagsFlag.Name)
 		)
-		return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{
+
-			"host":     hosttag,
+		tagsMap := utils.SplitTagsFlag(tags)
-			"version":  gitCommit,
+		tagsMap["version"] = gitCommit
-			"filesize": fmt.Sprintf("%v", filesize),
+		tagsMap["filesize"] = fmt.Sprintf("%v", filesize)
-		})
+
+		return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", tagsMap)
 	}
 
 	return nil

cmd/swarm/swarm-smoke/sliding_window.go

@@ -0,0 +1,131 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+	"github.com/pborman/uuid"
+
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+type uploadResult struct {
+	hash   string
+	digest []byte
+}
+
+func slidingWindowCmd(ctx *cli.Context, tuid string) error {
+	errc := make(chan error)
+
+	go func() {
+		errc <- slidingWindow(ctx, tuid)
+	}()
+
+	select {
+	case err := <-errc:
+		if err != nil {
+			metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
+		}
+		return err
+	case <-time.After(time.Duration(timeout) * time.Second):
+		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
+
+		return fmt.Errorf("timeout after %v sec", timeout)
+	}
+}
+
+func slidingWindow(ctx *cli.Context, tuid string) error {
+	var hashes []uploadResult //swarm hashes of the uploads
+	nodes := len(hosts)
+	const iterationTimeout = 30 * time.Second
+	log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)
+	uploadedBytes := 0
+	networkDepth := 0
+	errored := false
+
+outer:
+	for {
+		log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
+
+		t1 := time.Now()
+
+		randomBytes := testutil.RandomBytes(seed, filesize*1000)
+
+		hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
+		if err != nil {
+			log.Error(err.Error())
+			return err
+		}
+
+		metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1)
+
+		fhash, err := digest(bytes.NewReader(randomBytes))
+		if err != nil {
+			log.Error(err.Error())
+			return err
+		}
+
+		log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay)
+		hashes = append(hashes, uploadResult{hash: hash, digest: fhash})
+		time.Sleep(time.Duration(syncDelay) * time.Second)
+		uploadedBytes += filesize * 1000
+
+		for i, v := range hashes {
+			timeout := time.After(time.Duration(timeout) * time.Second)
+			errored = false
+
+		inner:
+			for {
+				select {
+				case <-timeout:
+					errored = true
+					log.Error("error retrieving hash. timeout", "hash idx", i, "err", err)
+					metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
+					break inner
+				default:
+					idx := 1 + rand.Intn(len(hosts)-1)
+					ruid := uuid.New()[:8]
+					start := time.Now()
+					err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid, "")
+					if err != nil {
+						continue inner
+					}
+					metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start)
+					break inner
+				}
+			}
+
+			if errored {
+				break outer
+			}
+			networkDepth = i
+			metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth))
+		}
+	}
+
+	log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize)
+	log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize)
+
+	return nil
+}

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -19,94 +19,122 @@ package main
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
-	crand "crypto/rand"
-	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"math/rand"
-	"net/http"
-	"net/http/httptrace"
 	"os"
 	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/storage"
-	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
-	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pborman/uuid"
 
 	cli "gopkg.in/urfave/cli.v1"
 )
 
-func generateEndpoints(scheme string, cluster string, app string, from int, to int) {
+func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
-	if cluster == "prod" {
+	randomBytes := testutil.RandomBytes(seed, filesize*1000)
-		for port := from; port < to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
-		}
-	} else if cluster == "private-internal" {
-		for port := from; port < to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port))
-		}
-	} else {
-		for port := from; port < to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster))
-		}
-	}
-
-	if includeLocalhost {
-		endpoints = append(endpoints, "http://localhost:8500")
-	}
-}
-
-func cliUploadAndSync(c *cli.Context) error {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
-
-	metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1)
 
 	errc := make(chan error)
+
 	go func() {
-		errc <- uploadAndSync(c)
+		errc <- uplaodAndSync(ctx, randomBytes, tuid)
 	}()
 
 	select {
 	case err := <-errc:
 		if err != nil {
-			metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1)
+			metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
 		}
 		return err
 	case <-time.After(time.Duration(timeout) * time.Second):
-		metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1)
+		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
-		return fmt.Errorf("timeout after %v sec", timeout)
+
+		e := fmt.Errorf("timeout after %v sec", timeout)
+		// trigger debug functionality on randomBytes
+		err := trackChunks(randomBytes[:])
+		if err != nil {
+			e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err)
+		}
+
+		return e
 	}
 }
 
-func uploadAndSync(c *cli.Context) error {
+func trackChunks(testData []byte) error {
-	defer func(now time.Time) {
+	log.Warn("Test timed out; running chunk debug sequence")
-		totalTime := time.Since(now)
-		log.Info("total time", "time", totalTime, "kb", filesize)
-		metrics.GetOrRegisterResettingTimer("upload-and-sync.total-time", nil).Update(totalTime)
-	}(time.Now())
 
-	generateEndpoints(scheme, cluster, appName, from, to)
+	addrs, err := getAllRefs(testData)
-	seed := int(time.Now().UnixNano() / 1e6)
+	if err != nil {
-	log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed)
+		return err
+	}
+	log.Trace("All references retrieved")
 
-	randomBytes := testutil.RandomBytes(seed, filesize*1000)
+	// has-chunks
+	for _, host := range hosts {
+		httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
+		log.Trace("Calling `Has` on host", "httpHost", httpHost)
+		rpcClient, err := rpc.Dial(httpHost)
+		if err != nil {
+			log.Trace("Error dialing host", "err", err)
+			return err
+		}
+		log.Trace("rpc dial ok")
+		var hasInfo []api.HasInfo
+		err = rpcClient.Call(&hasInfo, "bzz_has", addrs)
+		if err != nil {
+			log.Trace("Error calling host", "err", err)
+			return err
+		}
+		log.Trace("rpc call ok")
+		count := 0
+		for _, info := range hasInfo {
+			if !info.Has {
+				count++
+				log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr)
+			}
+		}
+		if count == 0 {
+			log.Info("Host reported to have all chunks", "host", httpHost)
+		}
+	}
+	return nil
+}
+
+func getAllRefs(testData []byte) (storage.AddressCollection, error) {
+	log.Trace("Getting all references for given root hash")
+	datadir, err := ioutil.TempDir("", "chunk-debug")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(datadir)
+	fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second)
+	defer cancel()
+
+	reader := bytes.NewReader(testData)
+	return fileStore.GetAllReferences(ctx, reader, false)
+}
+
+func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error {
+	log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed)
 
 	t1 := time.Now()
-	hash, err := upload(&randomBytes, endpoints[0])
+	hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
 	if err != nil {
 		log.Error(err.Error())
 		return err
 	}
-	metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1)
+	t2 := time.Since(t1)
+	metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).Update(t2)
 
 	fhash, err := digest(bytes.NewReader(randomBytes))
 	if err != nil {
@@ -114,143 +142,53 @@ func uploadAndSync(c *cli.Context) error {
 		return err
 	}
 
-	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
+	log.Info("uploaded successfully", "tuid", tuid, "hash", hash, "took", t2, "digest", fmt.Sprintf("%x", fhash))
 
 	time.Sleep(time.Duration(syncDelay) * time.Second)
 
 	wg := sync.WaitGroup{}
 	if single {
-		rand.Seed(time.Now().UTC().UnixNano())
+		randIndex := 1 + rand.Intn(len(hosts)-1)
-		randIndex := 1 + rand.Intn(len(endpoints)-1)
 		ruid := uuid.New()[:8]
 		wg.Add(1)
 		go func(endpoint string, ruid string) {
 			for {
 				start := time.Now()
-				err := fetch(hash, endpoint, fhash, ruid)
+				err := fetch(hash, endpoint, fhash, ruid, tuid)
 				if err != nil {
 					continue
 				}
+				ended := time.Since(start)
 
-				metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).UpdateSince(start)
+				metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).Update(ended)
+				log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint)
 				wg.Done()
 				return
 			}
-		}(endpoints[randIndex], ruid)
+		}(httpEndpoint(hosts[randIndex]), ruid)
 	} else {
-		for _, endpoint := range endpoints[1:] {
+		for _, endpoint := range hosts[1:] {
 			ruid := uuid.New()[:8]
 			wg.Add(1)
 			go func(endpoint string, ruid string) {
 				for {
 					start := time.Now()
-					err := fetch(hash, endpoint, fhash, ruid)
+					err := fetch(hash, endpoint, fhash, ruid, tuid)
 					if err != nil {
 						continue
 					}
+					ended := time.Since(start)
 
-					metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).UpdateSince(start)
+					metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).Update(ended)
+					log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint)
 					wg.Done()
 					return
 				}
-			}(endpoint, ruid)
+			}(httpEndpoint(endpoint), ruid)
 		}
 	}
 	wg.Wait()
-	log.Info("all endpoints synced random file successfully")
+	log.Info("all hosts synced random file successfully")
 
 	return nil
 }
-
-// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
-func fetch(hash string, endpoint string, original []byte, ruid string) error {
-	ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
-	defer sp.Finish()
-
-	log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
-
-	var tn time.Time
-	reqUri := endpoint + "/bzz:/" + hash + "/"
-	req, _ := http.NewRequest("GET", reqUri, nil)
-
-	opentracing.GlobalTracer().Inject(
-		sp.Context(),
-		opentracing.HTTPHeaders,
-		opentracing.HTTPHeadersCarrier(req.Header))
-
-	trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn)
-
-	req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
-	transport := http.DefaultTransport
-
-	//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-
-	tn = time.Now()
-	res, err := transport.RoundTrip(req)
-	if err != nil {
-		log.Error(err.Error(), "ruid", ruid)
-		return err
-	}
-	log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
-
-	if res.StatusCode != 200 {
-		err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
-		log.Warn(err.Error(), "ruid", ruid)
-		return err
-	}
-
-	defer res.Body.Close()
-
-	rdigest, err := digest(res.Body)
-	if err != nil {
-		log.Warn(err.Error(), "ruid", ruid)
-		return err
-	}
-
-	if !bytes.Equal(rdigest, original) {
-		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
-		log.Warn(err.Error(), "ruid", ruid)
-		return err
-	}
-
-	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
-
-	return nil
-}
-
-// upload is uploading a file `f` to `endpoint` via the `swarm up` cmd
-func upload(dataBytes *[]byte, endpoint string) (string, error) {
-	swarm := client.NewClient(endpoint)
-	f := &client.File{
-		ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)),
-		ManifestEntry: api.ManifestEntry{
-			ContentType: "text/plain",
-			Mode:        0660,
-			Size:        int64(len(*dataBytes)),
-		},
-	}
-
-	// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
-	return swarm.Upload(f, "", false)
-}
-
-func digest(r io.Reader) ([]byte, error) {
-	h := md5.New()
-	_, err := io.Copy(h, r)
-	if err != nil {
-		return nil, err
-	}
-	return h.Sum(nil), nil
-}
-
-// generates random data in heap buffer
-func generateRandomData(datasize int) ([]byte, error) {
-	b := make([]byte, datasize)
-	c, err := crand.Read(b)
-	if err != nil {
-		return nil, err
-	} else if c != datasize {
-		return nil, errors.New("short read")
-	}
-	return b, nil
-}

cmd/swarm/swarm-smoke/upload_speed.go

@@ -19,7 +19,6 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -29,65 +28,41 @@ import (
 	cli "gopkg.in/urfave/cli.v1"
 )
 
-var endpoint string
+func uploadSpeedCmd(ctx *cli.Context, tuid string) error {
-
+	log.Info("uploading to "+hosts[0], "tuid", tuid, "seed", seed)
-//just use the first endpoint
+	randomBytes := testutil.RandomBytes(seed, filesize*1000)
-func generateEndpoint(scheme string, cluster string, app string, from int) {
-	if cluster == "prod" {
-		endpoint = fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from)
-	} else if cluster == "private-internal" {
-		endpoint = fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from)
-	} else {
-		endpoint = fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster)
-	}
-}
-
-func cliUploadSpeed(c *cli.Context) error {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
-
-	metrics.GetOrRegisterCounter("upload-speed", nil).Inc(1)
 
 	errc := make(chan error)
+
 	go func() {
-		errc <- uploadSpeed(c)
+		errc <- uploadSpeed(ctx, tuid, randomBytes)
 	}()
 
 	select {
 	case err := <-errc:
 		if err != nil {
-			metrics.GetOrRegisterCounter("upload-speed.fail", nil).Inc(1)
+			metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
 		}
 		return err
 	case <-time.After(time.Duration(timeout) * time.Second):
-		metrics.GetOrRegisterCounter("upload-speed.timeout", nil).Inc(1)
+		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
+
+		// trigger debug functionality on randomBytes
+
 		return fmt.Errorf("timeout after %v sec", timeout)
 	}
 }
 
-func uploadSpeed(c *cli.Context) error {
+func uploadSpeed(c *cli.Context, tuid string, data []byte) error {
-	defer func(now time.Time) {
-		totalTime := time.Since(now)
-
-		log.Info("total time", "time", totalTime, "kb", filesize)
-		metrics.GetOrRegisterCounter("upload-speed.total-time", nil).Inc(int64(totalTime))
-	}(time.Now())
-
-	generateEndpoint(scheme, cluster, appName, from)
-	seed := int(time.Now().UnixNano() / 1e6)
-	log.Info("uploading to "+endpoint, "seed", seed)
-
-	randomBytes := testutil.RandomBytes(seed, filesize*1000)
-
 	t1 := time.Now()
-	hash, err := upload(&randomBytes, endpoint)
+	hash, err := upload(data, hosts[0])
 	if err != nil {
 		log.Error(err.Error())
 		return err
 	}
 	metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1)))
 
-	fhash, err := digest(bytes.NewReader(randomBytes))
+	fhash, err := digest(bytes.NewReader(data))
 	if err != nil {
 		log.Error(err.Error())
 		return err

cmd/swarm/swarm-smoke/util.go

@@ -0,0 +1,235 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	crand "crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptrace"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/spancontext"
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/pborman/uuid"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var (
+	commandName = ""
+	seed        = int(time.Now().UTC().UnixNano())
+)
+
+func init() {
+	rand.Seed(int64(seed))
+}
+
+func httpEndpoint(host string) string {
+	return fmt.Sprintf("http://%s:%d", host, httpPort)
+}
+
+func wsEndpoint(host string) string {
+	return fmt.Sprintf("ws://%s:%d", host, wsPort)
+}
+
+func wrapCliCommand(name string, command func(*cli.Context, string) error) func(*cli.Context) error {
+	return func(ctx *cli.Context) error {
+		log.PrintOrigins(true)
+		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
+
+		// test uuid
+		tuid := uuid.New()[:8]
+
+		commandName = name
+
+		hosts = strings.Split(allhosts, ",")
+
+		defer func(now time.Time) {
+			totalTime := time.Since(now)
+			log.Info("total time", "tuid", tuid, "time", totalTime, "kb", filesize)
+			metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime)
+		}(time.Now())
+
+		log.Info("smoke test starting", "tuid", tuid, "task", name, "timeout", timeout)
+		metrics.GetOrRegisterCounter(name, nil).Inc(1)
+
+		return command(ctx, tuid)
+	}
+}
+
+func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
+	ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
+	defer sp.Finish()
+
+	log.Trace("sleeping", "ruid", ruid)
+	time.Sleep(3 * time.Second)
+
+	log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
+
+	var tn time.Time
+	reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
+	req, _ := http.NewRequest("GET", reqUri, nil)
+
+	opentracing.GlobalTracer().Inject(
+		sp.Context(),
+		opentracing.HTTPHeaders,
+		opentracing.HTTPHeadersCarrier(req.Header))
+
+	trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
+
+	req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
+	transport := http.DefaultTransport
+
+	//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	tn = time.Now()
+	res, err := transport.RoundTrip(req)
+	if err != nil {
+		log.Error(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
+
+	if res.StatusCode != 200 {
+		return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
+	}
+
+	defer res.Body.Close()
+
+	rdigest, err := digest(res.Body)
+	if err != nil {
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	if !bytes.Equal(rdigest, original) {
+		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
+
+	return nil
+}
+
+// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
+func fetch(hash string, endpoint string, original []byte, ruid string, tuid string) error {
+	ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
+	defer sp.Finish()
+
+	log.Info("http get request", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash)
+
+	var tn time.Time
+	reqUri := endpoint + "/bzz:/" + hash + "/"
+	req, _ := http.NewRequest("GET", reqUri, nil)
+
+	opentracing.GlobalTracer().Inject(
+		sp.Context(),
+		opentracing.HTTPHeaders,
+		opentracing.HTTPHeadersCarrier(req.Header))
+
+	trace := client.GetClientTrace(commandName+" - http get", commandName, ruid, &tn)
+
+	req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
+	transport := http.DefaultTransport
+
+	//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	tn = time.Now()
+	res, err := transport.RoundTrip(req)
+	if err != nil {
+		log.Error(err.Error(), "ruid", ruid)
+		return err
+	}
+	log.Info("http get response", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
+
+	if res.StatusCode != 200 {
+		err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	defer res.Body.Close()
+
+	rdigest, err := digest(res.Body)
+	if err != nil {
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	if !bytes.Equal(rdigest, original) {
+		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
+
+	return nil
+}
+
+// upload an arbitrary byte as a plaintext file  to `endpoint` using the api client
+func upload(data []byte, endpoint string) (string, error) {
+	swarm := client.NewClient(endpoint)
+	f := &client.File{
+		ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+		ManifestEntry: api.ManifestEntry{
+			ContentType: "text/plain",
+			Mode:        0660,
+			Size:        int64(len(data)),
+		},
+	}
+
+	// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
+	return swarm.Upload(f, "", false)
+}
+
+func digest(r io.Reader) ([]byte, error) {
+	h := md5.New()
+	_, err := io.Copy(h, r)
+	if err != nil {
+		return nil, err
+	}
+	return h.Sum(nil), nil
+}
+
+// generates random data in heap buffer
+func generateRandomData(datasize int) ([]byte, error) {
+	b := make([]byte, datasize)
+	c, err := crand.Read(b)
+	if err != nil {
+		return nil, err
+	} else if c != datasize {
+		return nil, errors.New("short read")
+	}
+	return b, nil
+}

cmd/utils/flags.go

@@ -58,7 +58,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 var (
@@ -141,6 +141,10 @@ var (
 		Name:  "rinkeby",
 		Usage: "Rinkeby network: pre-configured proof-of-authority test network",
 	}
+	GoerliFlag = cli.BoolFlag{
+		Name:  "goerli",
+		Usage: "Görli network: pre-configured proof-of-authority test network",
+	}
 	ConstantinopleOverrideFlag = cli.Uint64Flag{
 		Name:  "override.constantinople",
 		Usage: "Manually specify constantinople fork-block, overriding the bundled setting",
@@ -328,12 +332,12 @@ var (
 	}
 	CacheTrieFlag = cli.IntFlag{
 		Name:  "cache.trie",
-		Usage: "Percentage of cache memory allowance to use for trie caching",
+		Usage: "Percentage of cache memory allowance to use for trie caching (default = 25% full mode, 50% archive mode)",
 		Value: 25,
 	}
 	CacheGCFlag = cli.IntFlag{
 		Name:  "cache.gc",
-		Usage: "Percentage of cache memory allowance to use for trie pruning",
+		Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)",
 		Value: 25,
 	}
 	TrieCacheGenFlag = cli.IntFlag{
@@ -423,7 +427,11 @@ var (
 		Usage: "Password file to use for non-interactive password input",
 		Value: "",
 	}
-
+	ExternalSignerFlag = cli.StringFlag{
+		Name:  "signer",
+		Usage: "External signer (url or path to ipc file)",
+		Value: "",
+	}
 	VMEnableDebugFlag = cli.BoolFlag{
 		Name:  "vmdebug",
 		Usage: "Record information useful for VM and contract debugging",
@@ -659,14 +667,14 @@ var (
 		Usage: "Password to authorize access to the database",
 		Value: "test",
 	}
-	// The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB.
+	// Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB.
-	// It is used so that we can group all nodes and average a measurement across all of them, but also so
+	// For example `host` tag could be used so that we can group all nodes and average a measurement
-	// that we can select a specific node and inspect its measurements.
+	// across all of them, but also so that we can select a specific node and inspect its measurements.
 	// https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key
-	MetricsInfluxDBHostTagFlag = cli.StringFlag{
+	MetricsInfluxDBTagsFlag = cli.StringFlag{
-		Name:  "metrics.influxdb.host.tag",
+		Name:  "metrics.influxdb.tags",
-		Usage: "InfluxDB `host` tag attached to all measurements",
+		Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements",
-		Value: "localhost",
+		Value: "host=localhost",
 	}
 
 	EWASMInterpreterFlag = cli.StringFlag{
@@ -692,6 +700,9 @@ func MakeDataDir(ctx *cli.Context) string {
 		if ctx.GlobalBool(RinkebyFlag.Name) {
 			return filepath.Join(path, "rinkeby")
 		}
+		if ctx.GlobalBool(GoerliFlag.Name) {
+			return filepath.Join(path, "goerli")
+		}
 		return path
 	}
 	Fatalf("Cannot determine default data directory, please set manually (--datadir)")
@@ -746,6 +757,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 		urls = params.TestnetBootnodes
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		urls = params.RinkebyBootnodes
+	case ctx.GlobalBool(GoerliFlag.Name):
+		urls = params.GoerliBootnodes
 	case cfg.BootstrapNodes != nil:
 		return // already set, don't apply defaults.
 	}
@@ -775,6 +788,8 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
 		}
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		urls = params.RinkebyBootnodes
+	case ctx.GlobalBool(GoerliFlag.Name):
+		urls = params.GoerliBootnodes
 	case cfg.BootstrapNodesV5 != nil:
 		return // already set, don't apply defaults.
 	}
@@ -935,10 +950,11 @@ func makeDatabaseHandles() int {
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if err := fdlimit.Raise(uint64(limit)); err != nil {
+	raised, err := fdlimit.Raise(uint64(limit))
+	if err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
-	return limit / 2 // Leave half for networking and other stuff
+	return int(raised / 2) // Leave half for networking and other stuff
 }
 
 // MakeAddress converts an account specified directly as a hex encoded string or
@@ -979,11 +995,15 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) {
 	}
 	// Convert the etherbase into an address and configure it
 	if etherbase != "" {
-		account, err := MakeAddress(ks, etherbase)
+		if ks != nil {
-		if err != nil {
+			account, err := MakeAddress(ks, etherbase)
-			Fatalf("Invalid miner etherbase: %v", err)
+			if err != nil {
+				Fatalf("Invalid miner etherbase: %v", err)
+			}
+			cfg.Etherbase = account.Address
+		} else {
+			Fatalf("No etherbase configured")
 		}
-		cfg.Etherbase = account.Address
 	}
 }
 
@@ -1080,9 +1100,12 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setGraphQL(ctx, cfg)
 	setWS(ctx, cfg)
 	setNodeUserIdent(ctx, cfg)
-
 	setDataDir(ctx, cfg)
 
+	if ctx.GlobalIsSet(ExternalSignerFlag.Name) {
+		cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name)
+	}
+
 	if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
 		cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
 	}
@@ -1104,6 +1127,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
+	case ctx.GlobalBool(GoerliFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli")
 	}
 }
 
@@ -1260,10 +1285,14 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	// Avoid conflicting network flags
-	checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag)
+	checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag)
 	checkExclusive(ctx, LightServFlag, SyncModeFlag, "light")
-
+	// Can't use both ephemeral unlocked and external signer
-	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	checkExclusive(ctx, DeveloperFlag, ExternalSignerFlag)
+	var ks *keystore.KeyStore
+	if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
+		ks = keystores[0].(*keystore.KeyStore)
+	}
 	setEtherbase(ctx, ks, cfg)
 	setGPO(ctx, &cfg.GPO)
 	setTxPool(ctx, &cfg.TxPool)
@@ -1359,6 +1388,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 			cfg.NetworkId = 4
 		}
 		cfg.Genesis = core.DefaultRinkebyGenesisBlock()
+	case ctx.GlobalBool(GoerliFlag.Name):
+		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
+			cfg.NetworkId = 5
+		}
+		cfg.Genesis = core.DefaultGoerliGenesisBlock()
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 1337
@@ -1463,18 +1497,35 @@ func SetupMetrics(ctx *cli.Context) {
 			database     = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name)
 			username     = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name)
 			password     = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name)
-			hosttag      = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name)
 		)
 
 		if enableExport {
+			tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name))
+
 			log.Info("Enabling metrics export to InfluxDB")
-			go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", map[string]string{
+
-				"host": hosttag,
+			go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap)
-			})
 		}
 	}
 }
 
+func SplitTagsFlag(tagsFlag string) map[string]string {
+	tags := strings.Split(tagsFlag, ",")
+	tagsMap := map[string]string{}
+
+	for _, t := range tags {
+		if t != "" {
+			kv := strings.Split(t, "=")
+
+			if len(kv) == 2 {
+				tagsMap[kv[0]] = kv[1]
+			}
+		}
+	}
+
+	return tagsMap
+}
+
 // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
 func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	var (
@@ -1499,6 +1550,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 		genesis = core.DefaultTestnetGenesisBlock()
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		genesis = core.DefaultRinkebyGenesisBlock()
+	case ctx.GlobalBool(GoerliFlag.Name):
+		genesis = core.DefaultGoerliGenesisBlock()
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		Fatalf("Developer chains are ephemeral")
 	}
@@ -1560,7 +1613,7 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
 		return nil
 	}
 	// Otherwise resolve absolute paths and return them
-	preloads := []string{}
+	var preloads []string
 
 	assets := ctx.GlobalString(JSpathFlag.Name)
 	for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") {

cmd/utils/flags_test.go

@@ -0,0 +1,64 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Package utils contains internal helper functions for go-ethereum commands.
+package utils
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_SplitTagsFlag(t *testing.T) {
+	tests := []struct {
+		name string
+		args string
+		want map[string]string
+	}{
+		{
+			"2 tags case",
+			"host=localhost,bzzkey=123",
+			map[string]string{
+				"host":   "localhost",
+				"bzzkey": "123",
+			},
+		},
+		{
+			"1 tag case",
+			"host=localhost123",
+			map[string]string{
+				"host": "localhost123",
+			},
+		},
+		{
+			"empty case",
+			"",
+			map[string]string{},
+		},
+		{
+			"garbage",
+			"smth=smthelse=123",
+			map[string]string{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := SplitTagsFlag(tt.args); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("splitTagsFlag() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

common/compiler/solidity_test.go

@@ -23,9 +23,10 @@ import (
 
 const (
 	testSource = `
+pragma solidity >0.0.0;
 contract test {
    /// @notice Will multiply ` + "`a`" + ` by 7.
-   function multiply(uint a) returns(uint d) {
+   function multiply(uint a) public returns(uint d) {
        return a * 7;
    }
 }

common/fdlimit/fdlimit_freebsd.go

@@ -26,11 +26,11 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
@@ -38,9 +38,12 @@ func Raise(max uint64) error {
 		limit.Cur = int64(max)
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
-	return nil
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_test.go

@@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
 	if limit, err := Current(); err != nil || limit <= 0 {
 		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
 	}
-	if err := Raise(uint64(target)); err != nil {
+	if _, err := Raise(uint64(target)); err != nil {
 		t.Fatalf("failed to raise file allowance")
 	}
 	if limit, err := Current(); err != nil || limit < target {

common/fdlimit/fdlimit_unix.go

@@ -22,11 +22,12 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
@@ -34,9 +35,13 @@ func Raise(max uint64) error {
 		limit.Cur = max
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
-	return nil
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_windows.go

@@ -16,28 +16,30 @@
 
 package fdlimit
 
-import "errors"
+import "fmt"
+
+const hardlimit = 16384
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// This method is NOP by design:
 	//  * Linux/Darwin counterparts need to manually increase per process limits
 	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
 	//    changeable from within a running process
 	// This way we can always "request" raising the limits, which will either have
 	// or not have effect based on the platform we're running on.
-	if max > 16384 {
+	if max > hardlimit {
-		return errors.New("file descriptor limit (16384) reached")
+		return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
 	}
-	return nil
+	return max, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this
 // process.
 func Current() (int, error) {
 	// Please see Raise for the reason why we use hard coded 16K as the limit
-	return 16384, nil
+	return hardlimit, nil
 }
 
 // Maximum retrieves the maximum number of file descriptors this process is

common/size.go

@@ -26,10 +26,10 @@ type StorageSize float64
 
 // String implements the stringer interface.
 func (s StorageSize) String() string {
-	if s > 1000000 {
+	if s > 1048576 {
-		return fmt.Sprintf("%.2f mB", s/1000000)
+		return fmt.Sprintf("%.2f MiB", s/1048576)
-	} else if s > 1000 {
+	} else if s > 1024 {
-		return fmt.Sprintf("%.2f kB", s/1000)
+		return fmt.Sprintf("%.2f KiB", s/1024)
 	} else {
 		return fmt.Sprintf("%.2f B", s)
 	}
@@ -38,10 +38,10 @@ func (s StorageSize) String() string {
 // TerminalString implements log.TerminalStringer, formatting a string for console
 // output during logging.
 func (s StorageSize) TerminalString() string {
-	if s > 1000000 {
+	if s > 1048576 {
-		return fmt.Sprintf("%.2fmB", s/1000000)
+		return fmt.Sprintf("%.2fMiB", s/1048576)
-	} else if s > 1000 {
+	} else if s > 1024 {
-		return fmt.Sprintf("%.2fkB", s/1000)
+		return fmt.Sprintf("%.2fKiB", s/1024)
 	} else {
 		return fmt.Sprintf("%.2fB", s)
 	}

common/size_test.go

@@ -25,8 +25,8 @@ func TestStorageSizeString(t *testing.T) {
 		size StorageSize
 		str  string
 	}{
-		{2381273, "2.38 mB"},
+		{2381273, "2.27 MiB"},
-		{2192, "2.19 kB"},
+		{2192, "2.14 KiB"},
 		{12, "12.00 B"},
 	}
 

consensus/clique/clique.go

@@ -20,6 +20,7 @@ package clique
 import (
 	"bytes"
 	"errors"
+	"io"
 	"math/big"
 	"math/rand"
 	"sync"
@@ -136,40 +137,9 @@ var (
 	errRecentlySigned = errors.New("recently signed")
 )
 
-// SignerFn is a signer callback function to request a hash to be signed by a
+// SignerFn is a signer callback function to request a header to be signed by a
 // backing account.
-type SignerFn func(accounts.Account, []byte) ([]byte, error)
+type SignerFn func(accounts.Account, string, []byte) ([]byte, error)
-
-// sigHash returns the hash which is used as input for the proof-of-authority
-// signing. It is the hash of the entire header apart from the 65 byte signature
-// contained at the end of the extra data.
-//
-// Note, the method requires the extra data to be at least 65 bytes, otherwise it
-// panics. This is done to avoid accidentally using both forms (signature present
-// or not), which could be abused to produce different hashes for the same header.
-func sigHash(header *types.Header) (hash common.Hash) {
-	hasher := sha3.NewLegacyKeccak256()
-
-	rlp.Encode(hasher, []interface{}{
-		header.ParentHash,
-		header.UncleHash,
-		header.Coinbase,
-		header.Root,
-		header.TxHash,
-		header.ReceiptHash,
-		header.Bloom,
-		header.Difficulty,
-		header.Number,
-		header.GasLimit,
-		header.GasUsed,
-		header.Time,
-		header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
-		header.MixDigest,
-		header.Nonce,
-	})
-	hasher.Sum(hash[:0])
-	return hash
-}
 
 // ecrecover extracts the Ethereum account address from a signed header.
 func ecrecover(header *types.Header, sigcache *lru.ARCCache) (common.Address, error) {
@@ -185,7 +155,7 @@ func ecrecover(header *types.Header, sigcache *lru.ARCCache) (common.Address, er
 	signature := header.Extra[len(header.Extra)-extraSeal:]
 
 	// Recover the public key and the Ethereum address
-	pubkey, err := crypto.Ecrecover(sigHash(header).Bytes(), signature)
+	pubkey, err := crypto.Ecrecover(SealHash(header).Bytes(), signature)
 	if err != nil {
 		return common.Address{}, err
 	}
@@ -646,7 +616,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results c
 		log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle))
 	}
 	// Sign all the things!
-	sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes())
+	sighash, err := signFn(accounts.Account{Address: signer}, accounts.MimetypeClique, CliqueRLP(header))
 	if err != nil {
 		return err
 	}
@@ -663,7 +633,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results c
 		select {
 		case results <- block.WithSeal(header):
 		default:
-			log.Warn("Sealing result is not read by miner", "sealhash", c.SealHash(header))
+			log.Warn("Sealing result is not read by miner", "sealhash", SealHash(header))
 		}
 	}()
 
@@ -693,7 +663,7 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int {
 
 // SealHash returns the hash of a block prior to it being sealed.
 func (c *Clique) SealHash(header *types.Header) common.Hash {
-	return sigHash(header)
+	return SealHash(header)
 }
 
 // Close implements consensus.Engine. It's a noop for clique as there are no background threads.
@@ -711,3 +681,47 @@ func (c *Clique) APIs(chain consensus.ChainReader) []rpc.API {
 		Public:    false,
 	}}
 }
+
+// SealHash returns the hash of a block prior to it being sealed.
+func SealHash(header *types.Header) (hash common.Hash) {
+	hasher := sha3.NewLegacyKeccak256()
+	encodeSigHeader(hasher, header)
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+// CliqueRLP returns the rlp bytes which needs to be signed for the proof-of-authority
+// sealing. The RLP to sign consists of the entire header apart from the 65 byte signature
+// contained at the end of the extra data.
+//
+// Note, the method requires the extra data to be at least 65 bytes, otherwise it
+// panics. This is done to avoid accidentally using both forms (signature present
+// or not), which could be abused to produce different hashes for the same header.
+func CliqueRLP(header *types.Header) []byte {
+	b := new(bytes.Buffer)
+	encodeSigHeader(b, header)
+	return b.Bytes()
+}
+
+func encodeSigHeader(w io.Writer, header *types.Header) {
+	err := rlp.Encode(w, []interface{}{
+		header.ParentHash,
+		header.UncleHash,
+		header.Coinbase,
+		header.Root,
+		header.TxHash,
+		header.ReceiptHash,
+		header.Bloom,
+		header.Difficulty,
+		header.Number,
+		header.GasLimit,
+		header.GasUsed,
+		header.Time,
+		header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
+		header.MixDigest,
+		header.Nonce,
+	})
+	if err != nil {
+		panic("can't encode: " + err.Error())
+	}
+}

consensus/clique/snapshot_test.go

@@ -80,7 +80,7 @@ func (ap *testerAccountPool) sign(header *types.Header, signer string) {
 		ap.accounts[signer], _ = crypto.GenerateKey()
 	}
 	// Sign the header and embed the signature in extra data
-	sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
+	sig, _ := crypto.Sign(SealHash(header).Bytes(), ap.accounts[signer])
 	copy(header.Extra[len(header.Extra)-extraSeal:], sig)
 }
 

console/console.go

@@ -120,10 +120,10 @@ func (c *Console) init(preload []string) error {
 	consoleObj.Object().Set("error", c.consoleOutput)
 
 	// Load all the internal utility JavaScript libraries
-	if err := c.jsre.Compile("bignumber.js", jsre.BigNumber_JS); err != nil {
+	if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil {
 		return fmt.Errorf("bignumber.js: %v", err)
 	}
-	if err := c.jsre.Compile("web3.js", jsre.Web3_JS); err != nil {
+	if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil {
 		return fmt.Errorf("web3.js: %v", err)
 	}
 	if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil {
@@ -236,7 +236,7 @@ func (c *Console) clearHistory() {
 // consoleOutput is an override for the console.log and console.error methods to
 // stream the output into the configured output stream instead of stdout.
 func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value {
-	output := []string{}
+	var output []string
 	for _, argument := range call.ArgumentList {
 		output = append(output, fmt.Sprintf("%v", argument))
 	}

console/console_test.go

@@ -149,8 +149,8 @@ func (env *tester) Close(t *testing.T) {
 	if err := env.console.Stop(false); err != nil {
 		t.Errorf("failed to stop embedded console: %v", err)
 	}
-	if err := env.stack.Stop(); err != nil {
+	if err := env.stack.Close(); err != nil {
-		t.Errorf("failed to stop embedded node: %v", err)
+		t.Errorf("failed to tear down embedded node: %v", err)
 	}
 	os.RemoveAll(env.workspace)
 }

contracts/chequebook/api.go

@@ -27,40 +27,40 @@ const Version = "1.0"
 
 var errNoChequebook = errors.New("no chequebook")
 
-type Api struct {
+type API struct {
 	chequebookf func() *Chequebook
 }
 
-func NewApi(ch func() *Chequebook) *Api {
+func NewAPI(ch func() *Chequebook) *API {
-	return &Api{ch}
+	return &API{ch}
 }
 
-func (self *Api) Balance() (string, error) {
+func (a *API) Balance() (string, error) {
-	ch := self.chequebookf()
+	ch := a.chequebookf()
 	if ch == nil {
 		return "", errNoChequebook
 	}
 	return ch.Balance().String(), nil
 }
 
-func (self *Api) Issue(beneficiary common.Address, amount *big.Int) (cheque *Cheque, err error) {
+func (a *API) Issue(beneficiary common.Address, amount *big.Int) (cheque *Cheque, err error) {
-	ch := self.chequebookf()
+	ch := a.chequebookf()
 	if ch == nil {
 		return nil, errNoChequebook
 	}
 	return ch.Issue(beneficiary, amount)
 }
 
-func (self *Api) Cash(cheque *Cheque) (txhash string, err error) {
+func (a *API) Cash(cheque *Cheque) (txhash string, err error) {
-	ch := self.chequebookf()
+	ch := a.chequebookf()
 	if ch == nil {
 		return "", errNoChequebook
 	}
 	return ch.Cash(cheque)
 }
 
-func (self *Api) Deposit(amount *big.Int) (txhash string, err error) {
+func (a *API) Deposit(amount *big.Int) (txhash string, err error) {
-	ch := self.chequebookf()
+	ch := a.chequebookf()
 	if ch == nil {
 		return "", errNoChequebook
 	}

contracts/chequebook/cheque.go

@@ -75,8 +75,8 @@ type Cheque struct {
 	Sig         []byte   // signature Sign(Keccak256(contract, beneficiary, amount), prvKey)
 }
 
-func (self *Cheque) String() string {
+func (ch *Cheque) String() string {
-	return fmt.Sprintf("contract: %s, beneficiary: %s, amount: %v, signature: %x", self.Contract.Hex(), self.Beneficiary.Hex(), self.Amount, self.Sig)
+	return fmt.Sprintf("contract: %s, beneficiary: %s, amount: %v, signature: %x", ch.Contract.Hex(), ch.Beneficiary.Hex(), ch.Amount, ch.Sig)
 }
 
 type Params struct {
@@ -109,12 +109,12 @@ type Chequebook struct {
 	log log.Logger // contextual logger with the contract address embedded
 }
 
-func (self *Chequebook) String() string {
+func (cb *Chequebook) String() string {
-	return fmt.Sprintf("contract: %s, owner: %s, balance: %v, signer: %x", self.contractAddr.Hex(), self.owner.Hex(), self.balance, self.prvKey.PublicKey)
+	return fmt.Sprintf("contract: %s, owner: %s, balance: %v, signer: %x", cb.contractAddr.Hex(), cb.owner.Hex(), cb.balance, cb.prvKey.PublicKey)
 }
 
 // NewChequebook creates a new Chequebook.
-func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.PrivateKey, backend Backend) (self *Chequebook, err error) {
+func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.PrivateKey, backend Backend) (*Chequebook, error) {
 	balance := new(big.Int)
 	sent := make(map[common.Address]*big.Int)
 
@@ -128,7 +128,7 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva
 		TransactOpts: *transactOpts,
 	}
 
-	self = &Chequebook{
+	cb := &Chequebook{
 		prvKey:       prvKey,
 		balance:      balance,
 		contractAddr: contractAddr,
@@ -140,42 +140,39 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva
 		session:      session,
 		log:          log.New("contract", contractAddr),
 	}
-
 	if (contractAddr != common.Address{}) {
-		self.setBalanceFromBlockChain()
+		cb.setBalanceFromBlockChain()
-		self.log.Trace("New chequebook initialised", "owner", self.owner, "balance", self.balance)
+		cb.log.Trace("New chequebook initialised", "owner", cb.owner, "balance", cb.balance)
 	}
-	return
+	return cb, nil
 }
 
-func (self *Chequebook) setBalanceFromBlockChain() {
+func (cb *Chequebook) setBalanceFromBlockChain() {
-	balance, err := self.backend.BalanceAt(context.TODO(), self.contractAddr, nil)
+	balance, err := cb.backend.BalanceAt(context.TODO(), cb.contractAddr, nil)
 	if err != nil {
 		log.Error("Failed to retrieve chequebook balance", "err", err)
 	} else {
-		self.balance.Set(balance)
+		cb.balance.Set(balance)
 	}
 }
 
 // LoadChequebook loads a chequebook from disk (file path).
-func LoadChequebook(path string, prvKey *ecdsa.PrivateKey, backend Backend, checkBalance bool) (self *Chequebook, err error) {
+func LoadChequebook(path string, prvKey *ecdsa.PrivateKey, backend Backend, checkBalance bool) (*Chequebook, error) {
-	var data []byte
+	data, err := ioutil.ReadFile(path)
-	data, err = ioutil.ReadFile(path)
-	if err != nil {
-		return
-	}
-	self, _ = NewChequebook(path, common.Address{}, prvKey, backend)
-
-	err = json.Unmarshal(data, self)
 	if err != nil {
 		return nil, err
 	}
+	cb, _ := NewChequebook(path, common.Address{}, prvKey, backend)
+
+	if err = json.Unmarshal(data, cb); err != nil {
+		return nil, err
+	}
 	if checkBalance {
-		self.setBalanceFromBlockChain()
+		cb.setBalanceFromBlockChain()
 	}
 	log.Trace("Loaded chequebook from disk", "path", path)
 
-	return
+	return cb, nil
 }
 
 // chequebookFile is the JSON representation of a chequebook.
@@ -187,19 +184,19 @@ type chequebookFile struct {
 }
 
 // UnmarshalJSON deserialises a chequebook.
-func (self *Chequebook) UnmarshalJSON(data []byte) error {
+func (cb *Chequebook) UnmarshalJSON(data []byte) error {
 	var file chequebookFile
 	err := json.Unmarshal(data, &file)
 	if err != nil {
 		return err
 	}
-	_, ok := self.balance.SetString(file.Balance, 10)
+	_, ok := cb.balance.SetString(file.Balance, 10)
 	if !ok {
 		return fmt.Errorf("cumulative amount sent: unable to convert string to big integer: %v", file.Balance)
 	}
-	self.contractAddr = common.HexToAddress(file.Contract)
+	cb.contractAddr = common.HexToAddress(file.Contract)
 	for addr, sent := range file.Sent {
-		self.sent[common.HexToAddress(addr)], ok = new(big.Int).SetString(sent, 10)
+		cb.sent[common.HexToAddress(addr)], ok = new(big.Int).SetString(sent, 10)
 		if !ok {
 			return fmt.Errorf("beneficiary %v cumulative amount sent: unable to convert string to big integer: %v", addr, sent)
 		}
@@ -208,14 +205,14 @@ func (self *Chequebook) UnmarshalJSON(data []byte) error {
 }
 
 // MarshalJSON serialises a chequebook.
-func (self *Chequebook) MarshalJSON() ([]byte, error) {
+func (cb *Chequebook) MarshalJSON() ([]byte, error) {
 	var file = &chequebookFile{
-		Balance:  self.balance.String(),
+		Balance:  cb.balance.String(),
-		Contract: self.contractAddr.Hex(),
+		Contract: cb.contractAddr.Hex(),
-		Owner:    self.owner.Hex(),
+		Owner:    cb.owner.Hex(),
 		Sent:     make(map[string]string),
 	}
-	for addr, sent := range self.sent {
+	for addr, sent := range cb.sent {
 		file.Sent[addr.Hex()] = sent.String()
 	}
 	return json.Marshal(file)
@@ -223,76 +220,78 @@ func (self *Chequebook) MarshalJSON() ([]byte, error) {
 
 // Save persists the chequebook on disk, remembering balance, contract address and
 // cumulative amount of funds sent for each beneficiary.
-func (self *Chequebook) Save() (err error) {
+func (cb *Chequebook) Save() error {
-	data, err := json.MarshalIndent(self, "", " ")
+	data, err := json.MarshalIndent(cb, "", " ")
 	if err != nil {
 		return err
 	}
-	self.log.Trace("Saving chequebook to disk", self.path)
+	cb.log.Trace("Saving chequebook to disk", cb.path)
 
-	return ioutil.WriteFile(self.path, data, os.ModePerm)
+	return ioutil.WriteFile(cb.path, data, os.ModePerm)
 }
 
 // Stop quits the autodeposit go routine to terminate
-func (self *Chequebook) Stop() {
+func (cb *Chequebook) Stop() {
-	defer self.lock.Unlock()
+	defer cb.lock.Unlock()
-	self.lock.Lock()
+	cb.lock.Lock()
-	if self.quit != nil {
+	if cb.quit != nil {
-		close(self.quit)
+		close(cb.quit)
-		self.quit = nil
+		cb.quit = nil
 	}
 }
 
 // Issue creates a cheque signed by the chequebook owner's private key. The
 // signer commits to a contract (one that they own), a beneficiary and amount.
-func (self *Chequebook) Issue(beneficiary common.Address, amount *big.Int) (ch *Cheque, err error) {
+func (cb *Chequebook) Issue(beneficiary common.Address, amount *big.Int) (*Cheque, error) {
-	defer self.lock.Unlock()
+	defer cb.lock.Unlock()
-	self.lock.Lock()
+	cb.lock.Lock()
 
 	if amount.Sign() <= 0 {
 		return nil, fmt.Errorf("amount must be greater than zero (%v)", amount)
 	}
-	if self.balance.Cmp(amount) < 0 {
+	var (
-		err = fmt.Errorf("insufficient funds to issue cheque for amount: %v. balance: %v", amount, self.balance)
+		ch  *Cheque
+		err error
+	)
+	if cb.balance.Cmp(amount) < 0 {
+		err = fmt.Errorf("insufficient funds to issue cheque for amount: %v. balance: %v", amount, cb.balance)
 	} else {
 		var sig []byte
-		sent, found := self.sent[beneficiary]
+		sent, found := cb.sent[beneficiary]
 		if !found {
 			sent = new(big.Int)
-			self.sent[beneficiary] = sent
+			cb.sent[beneficiary] = sent
 		}
 		sum := new(big.Int).Set(sent)
 		sum.Add(sum, amount)
 
-		sig, err = crypto.Sign(sigHash(self.contractAddr, beneficiary, sum), self.prvKey)
+		sig, err = crypto.Sign(sigHash(cb.contractAddr, beneficiary, sum), cb.prvKey)
 		if err == nil {
 			ch = &Cheque{
-				Contract:    self.contractAddr,
+				Contract:    cb.contractAddr,
 				Beneficiary: beneficiary,
 				Amount:      sum,
 				Sig:         sig,
 			}
 			sent.Set(sum)
-			self.balance.Sub(self.balance, amount) // subtract amount from balance
+			cb.balance.Sub(cb.balance, amount) // subtract amount from balance
 		}
 	}
-
 	// auto deposit if threshold is set and balance is less then threshold
 	// note this is called even if issuing cheque fails
 	// so we reattempt depositing
-	if self.threshold != nil {
+	if cb.threshold != nil {
-		if self.balance.Cmp(self.threshold) < 0 {
+		if cb.balance.Cmp(cb.threshold) < 0 {
-			send := new(big.Int).Sub(self.buffer, self.balance)
+			send := new(big.Int).Sub(cb.buffer, cb.balance)
-			self.deposit(send)
+			cb.deposit(send)
 		}
 	}
-
+	return ch, err
-	return
 }
 
 // Cash is a convenience method to cash any cheque.
-func (self *Chequebook) Cash(ch *Cheque) (txhash string, err error) {
+func (cb *Chequebook) Cash(ch *Cheque) (string, error) {
-	return ch.Cash(self.session)
+	return ch.Cash(cb.session)
 }
 
 // data to sign: contract address, beneficiary, cumulative amount of funds ever sent
@@ -309,73 +308,73 @@ func sigHash(contract, beneficiary common.Address, sum *big.Int) []byte {
 }
 
 // Balance returns the current balance of the chequebook.
-func (self *Chequebook) Balance() *big.Int {
+func (cb *Chequebook) Balance() *big.Int {
-	defer self.lock.Unlock()
+	defer cb.lock.Unlock()
-	self.lock.Lock()
+	cb.lock.Lock()
-	return new(big.Int).Set(self.balance)
+	return new(big.Int).Set(cb.balance)
 }
 
 // Owner returns the owner account of the chequebook.
-func (self *Chequebook) Owner() common.Address {
+func (cb *Chequebook) Owner() common.Address {
-	return self.owner
+	return cb.owner
 }
 
 // Address returns the on-chain contract address of the chequebook.
-func (self *Chequebook) Address() common.Address {
+func (cb *Chequebook) Address() common.Address {
-	return self.contractAddr
+	return cb.contractAddr
 }
 
 // Deposit deposits money to the chequebook account.
-func (self *Chequebook) Deposit(amount *big.Int) (string, error) {
+func (cb *Chequebook) Deposit(amount *big.Int) (string, error) {
-	defer self.lock.Unlock()
+	defer cb.lock.Unlock()
-	self.lock.Lock()
+	cb.lock.Lock()
-	return self.deposit(amount)
+	return cb.deposit(amount)
 }
 
 // deposit deposits amount to the chequebook account.
-// The caller must hold self.lock.
+// The caller must hold lock.
-func (self *Chequebook) deposit(amount *big.Int) (string, error) {
+func (cb *Chequebook) deposit(amount *big.Int) (string, error) {
 	// since the amount is variable here, we do not use sessions
-	depositTransactor := bind.NewKeyedTransactor(self.prvKey)
+	depositTransactor := bind.NewKeyedTransactor(cb.prvKey)
 	depositTransactor.Value = amount
-	chbookRaw := &contract.ChequebookRaw{Contract: self.contract}
+	chbookRaw := &contract.ChequebookRaw{Contract: cb.contract}
 	tx, err := chbookRaw.Transfer(depositTransactor)
 	if err != nil {
-		self.log.Warn("Failed to fund chequebook", "amount", amount, "balance", self.balance, "target", self.buffer, "err", err)
+		cb.log.Warn("Failed to fund chequebook", "amount", amount, "balance", cb.balance, "target", cb.buffer, "err", err)
 		return "", err
 	}
 	// assume that transaction is actually successful, we add the amount to balance right away
-	self.balance.Add(self.balance, amount)
+	cb.balance.Add(cb.balance, amount)
-	self.log.Trace("Deposited funds to chequebook", "amount", amount, "balance", self.balance, "target", self.buffer)
+	cb.log.Trace("Deposited funds to chequebook", "amount", amount, "balance", cb.balance, "target", cb.buffer)
 	return tx.Hash().Hex(), nil
 }
 
 // AutoDeposit (re)sets interval time and amount which triggers sending funds to the
 // chequebook. Contract backend needs to be set if threshold is not less than buffer, then
 // deposit will be triggered on every new cheque issued.
-func (self *Chequebook) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
+func (cb *Chequebook) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
-	defer self.lock.Unlock()
+	defer cb.lock.Unlock()
-	self.lock.Lock()
+	cb.lock.Lock()
-	self.threshold = threshold
+	cb.threshold = threshold
-	self.buffer = buffer
+	cb.buffer = buffer
-	self.autoDeposit(interval)
+	cb.autoDeposit(interval)
 }
 
 // autoDeposit starts a goroutine that periodically sends funds to the chequebook
 // contract caller holds the lock the go routine terminates if Chequebook.quit is closed.
-func (self *Chequebook) autoDeposit(interval time.Duration) {
+func (cb *Chequebook) autoDeposit(interval time.Duration) {
-	if self.quit != nil {
+	if cb.quit != nil {
-		close(self.quit)
+		close(cb.quit)
-		self.quit = nil
+		cb.quit = nil
 	}
 	// if threshold >= balance autodeposit after every cheque issued
-	if interval == time.Duration(0) || self.threshold != nil && self.buffer != nil && self.threshold.Cmp(self.buffer) >= 0 {
+	if interval == time.Duration(0) || cb.threshold != nil && cb.buffer != nil && cb.threshold.Cmp(cb.buffer) >= 0 {
 		return
 	}
 
 	ticker := time.NewTicker(interval)
-	self.quit = make(chan bool)
+	cb.quit = make(chan bool)
-	quit := self.quit
+	quit := cb.quit
 
 	go func() {
 		for {
@@ -383,15 +382,15 @@ func (self *Chequebook) autoDeposit(interval time.Duration) {
 			case <-quit:
 				return
 			case <-ticker.C:
-				self.lock.Lock()
+				cb.lock.Lock()
-				if self.balance.Cmp(self.buffer) < 0 {
+				if cb.balance.Cmp(cb.buffer) < 0 {
-					amount := new(big.Int).Sub(self.buffer, self.balance)
+					amount := new(big.Int).Sub(cb.buffer, cb.balance)
-					txhash, err := self.deposit(amount)
+					txhash, err := cb.deposit(amount)
 					if err == nil {
-						self.txhash = txhash
+						cb.txhash = txhash
 					}
 				}
-				self.lock.Unlock()
+				cb.lock.Unlock()
 			}
 		}
 	}()
@@ -404,26 +403,26 @@ type Outbox struct {
 }
 
 // NewOutbox creates an outbox.
-func NewOutbox(chbook *Chequebook, beneficiary common.Address) *Outbox {
+func NewOutbox(cb *Chequebook, beneficiary common.Address) *Outbox {
-	return &Outbox{chbook, beneficiary}
+	return &Outbox{cb, beneficiary}
 }
 
 // Issue creates cheque.
-func (self *Outbox) Issue(amount *big.Int) (swap.Promise, error) {
+func (o *Outbox) Issue(amount *big.Int) (swap.Promise, error) {
-	return self.chequeBook.Issue(self.beneficiary, amount)
+	return o.chequeBook.Issue(o.beneficiary, amount)
 }
 
 // AutoDeposit enables auto-deposits on the underlying chequebook.
-func (self *Outbox) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
+func (o *Outbox) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
-	self.chequeBook.AutoDeposit(interval, threshold, buffer)
+	o.chequeBook.AutoDeposit(interval, threshold, buffer)
 }
 
 // Stop helps satisfy the swap.OutPayment interface.
-func (self *Outbox) Stop() {}
+func (o *Outbox) Stop() {}
 
 // String implements fmt.Stringer.
-func (self *Outbox) String() string {
+func (o *Outbox) String() string {
-	return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", self.chequeBook.Address().Hex(), self.beneficiary.Hex(), self.chequeBook.Balance())
+	return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", o.chequeBook.Address().Hex(), o.beneficiary.Hex(), o.chequeBook.Balance())
 }
 
 // Inbox can deposit, verify and cash cheques from a single contract to a single
@@ -445,7 +444,7 @@ type Inbox struct {
 
 // NewInbox creates an Inbox. An Inboxes is not persisted, the cumulative sum is updated
 // from blockchain when first cheque is received.
-func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend) (self *Inbox, err error) {
+func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend) (*Inbox, error) {
 	if signer == nil {
 		return nil, fmt.Errorf("signer is null")
 	}
@@ -461,7 +460,7 @@ func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address
 	}
 	sender := transactOpts.From
 
-	self = &Inbox{
+	inbox := &Inbox{
 		contract:    contractAddr,
 		beneficiary: beneficiary,
 		sender:      sender,
@@ -470,59 +469,61 @@ func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address
 		cashed:      new(big.Int).Set(common.Big0),
 		log:         log.New("contract", contractAddr),
 	}
-	self.log.Trace("New chequebook inbox initialized", "beneficiary", self.beneficiary, "signer", hexutil.Bytes(crypto.FromECDSAPub(signer)))
+	inbox.log.Trace("New chequebook inbox initialized", "beneficiary", inbox.beneficiary, "signer", hexutil.Bytes(crypto.FromECDSAPub(signer)))
-	return
+	return inbox, nil
 }
 
-func (self *Inbox) String() string {
+func (i *Inbox) String() string {
-	return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", self.contract.Hex(), self.beneficiary.Hex(), self.cheque.Amount)
+	return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", i.contract.Hex(), i.beneficiary.Hex(), i.cheque.Amount)
 }
 
 // Stop quits the autocash goroutine.
-func (self *Inbox) Stop() {
+func (i *Inbox) Stop() {
-	defer self.lock.Unlock()
+	defer i.lock.Unlock()
-	self.lock.Lock()
+	i.lock.Lock()
-	if self.quit != nil {
+	if i.quit != nil {
-		close(self.quit)
+		close(i.quit)
-		self.quit = nil
+		i.quit = nil
 	}
 }
 
 // Cash attempts to cash the current cheque.
-func (self *Inbox) Cash() (txhash string, err error) {
+func (i *Inbox) Cash() (string, error) {
-	if self.cheque != nil {
+	if i.cheque == nil {
-		txhash, err = self.cheque.Cash(self.session)
+		return "", nil
-		self.log.Trace("Cashing in chequebook cheque", "amount", self.cheque.Amount, "beneficiary", self.beneficiary)
-		self.cashed = self.cheque.Amount
 	}
-	return
+	txhash, err := i.cheque.Cash(i.session)
+	i.log.Trace("Cashing in chequebook cheque", "amount", i.cheque.Amount, "beneficiary", i.beneficiary)
+	i.cashed = i.cheque.Amount
+
+	return txhash, err
 }
 
 // AutoCash (re)sets maximum time and amount which triggers cashing of the last uncashed
 // cheque if maxUncashed is set to 0, then autocash on receipt.
-func (self *Inbox) AutoCash(cashInterval time.Duration, maxUncashed *big.Int) {
+func (i *Inbox) AutoCash(cashInterval time.Duration, maxUncashed *big.Int) {
-	defer self.lock.Unlock()
+	defer i.lock.Unlock()
-	self.lock.Lock()
+	i.lock.Lock()
-	self.maxUncashed = maxUncashed
+	i.maxUncashed = maxUncashed
-	self.autoCash(cashInterval)
+	i.autoCash(cashInterval)
 }
 
 // autoCash starts a loop that periodically clears the last cheque
 // if the peer is trusted. Clearing period could be 24h or a week.
-// The caller must hold self.lock.
+// The caller must hold lock.
-func (self *Inbox) autoCash(cashInterval time.Duration) {
+func (i *Inbox) autoCash(cashInterval time.Duration) {
-	if self.quit != nil {
+	if i.quit != nil {
-		close(self.quit)
+		close(i.quit)
-		self.quit = nil
+		i.quit = nil
 	}
 	// if maxUncashed is set to 0, then autocash on receipt
-	if cashInterval == time.Duration(0) || self.maxUncashed != nil && self.maxUncashed.Sign() == 0 {
+	if cashInterval == time.Duration(0) || i.maxUncashed != nil && i.maxUncashed.Sign() == 0 {
 		return
 	}
 
 	ticker := time.NewTicker(cashInterval)
-	self.quit = make(chan bool)
+	i.quit = make(chan bool)
-	quit := self.quit
+	quit := i.quit
 
 	go func() {
 		for {
@@ -530,14 +531,14 @@ func (self *Inbox) autoCash(cashInterval time.Duration) {
 			case <-quit:
 				return
 			case <-ticker.C:
-				self.lock.Lock()
+				i.lock.Lock()
-				if self.cheque != nil && self.cheque.Amount.Cmp(self.cashed) != 0 {
+				if i.cheque != nil && i.cheque.Amount.Cmp(i.cashed) != 0 {
-					txhash, err := self.Cash()
+					txhash, err := i.Cash()
 					if err == nil {
-						self.txhash = txhash
+						i.txhash = txhash
 					}
 				}
-				self.lock.Unlock()
+				i.lock.Unlock()
 			}
 		}
 	}()
@@ -545,56 +546,56 @@ func (self *Inbox) autoCash(cashInterval time.Duration) {
 
 // Receive is called to deposit the latest cheque to the incoming Inbox.
 // The given promise must be a *Cheque.
-func (self *Inbox) Receive(promise swap.Promise) (*big.Int, error) {
+func (i *Inbox) Receive(promise swap.Promise) (*big.Int, error) {
 	ch := promise.(*Cheque)
 
-	defer self.lock.Unlock()
+	defer i.lock.Unlock()
-	self.lock.Lock()
+	i.lock.Lock()
 
 	var sum *big.Int
-	if self.cheque == nil {
+	if i.cheque == nil {
 		// the sum is checked against the blockchain once a cheque is received
-		tally, err := self.session.Sent(self.beneficiary)
+		tally, err := i.session.Sent(i.beneficiary)
 		if err != nil {
 			return nil, fmt.Errorf("inbox: error calling backend to set amount: %v", err)
 		}
 		sum = tally
 	} else {
-		sum = self.cheque.Amount
+		sum = i.cheque.Amount
 	}
 
-	amount, err := ch.Verify(self.signer, self.contract, self.beneficiary, sum)
+	amount, err := ch.Verify(i.signer, i.contract, i.beneficiary, sum)
 	var uncashed *big.Int
 	if err == nil {
-		self.cheque = ch
+		i.cheque = ch
 
-		if self.maxUncashed != nil {
+		if i.maxUncashed != nil {
-			uncashed = new(big.Int).Sub(ch.Amount, self.cashed)
+			uncashed = new(big.Int).Sub(ch.Amount, i.cashed)
-			if self.maxUncashed.Cmp(uncashed) < 0 {
+			if i.maxUncashed.Cmp(uncashed) < 0 {
-				self.Cash()
+				i.Cash()
 			}
 		}
-		self.log.Trace("Received cheque in chequebook inbox", "amount", amount, "uncashed", uncashed)
+		i.log.Trace("Received cheque in chequebook inbox", "amount", amount, "uncashed", uncashed)
 	}
 
 	return amount, err
 }
 
 // Verify verifies cheque for signer, contract, beneficiary, amount, valid signature.
-func (self *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary common.Address, sum *big.Int) (*big.Int, error) {
+func (ch *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary common.Address, sum *big.Int) (*big.Int, error) {
-	log.Trace("Verifying chequebook cheque", "cheque", self, "sum", sum)
+	log.Trace("Verifying chequebook cheque", "cheque", ch, "sum", sum)
 	if sum == nil {
 		return nil, fmt.Errorf("invalid amount")
 	}
 
-	if self.Beneficiary != beneficiary {
+	if ch.Beneficiary != beneficiary {
-		return nil, fmt.Errorf("beneficiary mismatch: %v != %v", self.Beneficiary.Hex(), beneficiary.Hex())
+		return nil, fmt.Errorf("beneficiary mismatch: %v != %v", ch.Beneficiary.Hex(), beneficiary.Hex())
 	}
-	if self.Contract != contract {
+	if ch.Contract != contract {
-		return nil, fmt.Errorf("contract mismatch: %v != %v", self.Contract.Hex(), contract.Hex())
+		return nil, fmt.Errorf("contract mismatch: %v != %v", ch.Contract.Hex(), contract.Hex())
 	}
 
-	amount := new(big.Int).Set(self.Amount)
+	amount := new(big.Int).Set(ch.Amount)
 	if sum != nil {
 		amount.Sub(amount, sum)
 		if amount.Sign() <= 0 {
@@ -602,7 +603,7 @@ func (self *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary com
 		}
 	}
 
-	pubKey, err := crypto.SigToPub(sigHash(self.Contract, beneficiary, self.Amount), self.Sig)
+	pubKey, err := crypto.SigToPub(sigHash(ch.Contract, beneficiary, ch.Amount), ch.Sig)
 	if err != nil {
 		return nil, fmt.Errorf("invalid signature: %v", err)
 	}
@@ -621,9 +622,9 @@ func sig2vrs(sig []byte) (v byte, r, s [32]byte) {
 }
 
 // Cash cashes the cheque by sending an Ethereum transaction.
-func (self *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
+func (ch *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
-	v, r, s := sig2vrs(self.Sig)
+	v, r, s := sig2vrs(ch.Sig)
-	tx, err := session.Cash(self.Beneficiary, self.Amount, v, r, s)
+	tx, err := session.Cash(ch.Beneficiary, ch.Amount, v, r, s)
 	if err != nil {
 		return "", err
 	}
@@ -632,7 +633,7 @@ func (self *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
 
 // ValidateCode checks that the on-chain code at address matches the expected chequebook
 // contract code. This is used to detect suicided chequebooks.
-func ValidateCode(ctx context.Context, b Backend, address common.Address) (ok bool, err error) {
+func ValidateCode(ctx context.Context, b Backend, address common.Address) (bool, error) {
 	code, err := b.CodeAt(ctx, address, nil)
 	if err != nil {
 		return false, err

contracts/chequebook/contract/chequebook.go

@@ -205,22 +205,22 @@ func (_Chequebook *ChequebookCallerSession) Sent(arg0 common.Address) (*big.Int,
 // Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
 //
 // Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
-func (_Chequebook *ChequebookTransactor) Cash(opts *bind.TransactOpts, beneficiary common.Address, amount *big.Int, sig_v uint8, sig_r [32]byte, sig_s [32]byte) (*types.Transaction, error) {
+func (_Chequebook *ChequebookTransactor) Cash(opts *bind.TransactOpts, beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
-	return _Chequebook.contract.Transact(opts, "cash", beneficiary, amount, sig_v, sig_r, sig_s)
+	return _Chequebook.contract.Transact(opts, "cash", beneficiary, amount, sigV, sigR, sigS)
 }
 
 // Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
 //
 // Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
-func (_Chequebook *ChequebookSession) Cash(beneficiary common.Address, amount *big.Int, sig_v uint8, sig_r [32]byte, sig_s [32]byte) (*types.Transaction, error) {
+func (_Chequebook *ChequebookSession) Cash(beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
-	return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sig_v, sig_r, sig_s)
+	return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sigV, sigR, sigS)
 }
 
 // Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
 //
 // Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
-func (_Chequebook *ChequebookTransactorSession) Cash(beneficiary common.Address, amount *big.Int, sig_v uint8, sig_r [32]byte, sig_s [32]byte) (*types.Transaction, error) {
+func (_Chequebook *ChequebookTransactorSession) Cash(beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
-	return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sig_v, sig_r, sig_s)
+	return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sigV, sigR, sigS)
 }
 
 // Kill is a paid mutator transaction binding the contract method 0x41c0e1b5.

contracts/ens/contract/ens.go

@@ -227,10 +227,10 @@ func (_ENS *ENSCallerSession) Resolver(node [32]byte) (common.Address, error) {
 	return _ENS.Contract.Resolver(&_ENS.CallOpts, node)
 }
 
-// Ttl is a free data retrieval call binding the contract method 0x16a25cbd.
+// TTL is a free data retrieval call binding the contract method 0x16a25cbd.
 //
 // Solidity: function ttl(node bytes32) constant returns(uint64)
-func (_ENS *ENSCaller) Ttl(opts *bind.CallOpts, node [32]byte) (uint64, error) {
+func (_ENS *ENSCaller) TTL(opts *bind.CallOpts, node [32]byte) (uint64, error) {
 	var (
 		ret0 = new(uint64)
 	)
@@ -239,18 +239,18 @@ func (_ENS *ENSCaller) Ttl(opts *bind.CallOpts, node [32]byte) (uint64, error) {
 	return *ret0, err
 }
 
-// Ttl is a free data retrieval call binding the contract method 0x16a25cbd.
+// TTL is a free data retrieval call binding the contract method 0x16a25cbd.
 //
 // Solidity: function ttl(node bytes32) constant returns(uint64)
-func (_ENS *ENSSession) Ttl(node [32]byte) (uint64, error) {
+func (_ENS *ENSSession) TTL(node [32]byte) (uint64, error) {
-	return _ENS.Contract.Ttl(&_ENS.CallOpts, node)
+	return _ENS.Contract.TTL(&_ENS.CallOpts, node)
 }
 
-// Ttl is a free data retrieval call binding the contract method 0x16a25cbd.
+// TTL is a free data retrieval call binding the contract method 0x16a25cbd.
 //
 // Solidity: function ttl(node bytes32) constant returns(uint64)
-func (_ENS *ENSCallerSession) Ttl(node [32]byte) (uint64, error) {
+func (_ENS *ENSCallerSession) TTL(node [32]byte) (uint64, error) {
-	return _ENS.Contract.Ttl(&_ENS.CallOpts, node)
+	return _ENS.Contract.TTL(&_ENS.CallOpts, node)
 }
 
 // SetOwner is a paid mutator transaction binding the contract method 0x5b0fc9c3.
@@ -682,7 +682,7 @@ func (it *ENSNewTTLIterator) Close() error {
 // ENSNewTTL represents a NewTTL event raised by the ENS contract.
 type ENSNewTTL struct {
 	Node [32]byte
-	Ttl  uint64
+	TTL  uint64
 	Raw  types.Log // Blockchain specific contextual infos
 }
 

contracts/ens/ens.go

@@ -35,7 +35,7 @@ var (
 	TestNetAddress = common.HexToAddress("0x112234455c3a32fd11230c42e7bccd4a84e02010")
 )
 
-// swarm domain name registry and resolver
+// ENS is the swarm domain name registry and resolver
 type ENS struct {
 	*contract.ENSSession
 	contractBackend bind.ContractBackend
@@ -48,7 +48,6 @@ func NewENS(transactOpts *bind.TransactOpts, contractAddr common.Address, contra
 	if err != nil {
 		return nil, err
 	}
-
 	return &ENS{
 		&contract.ENSSession{
 			Contract:     ens,
@@ -60,27 +59,24 @@ func NewENS(transactOpts *bind.TransactOpts, contractAddr common.Address, contra
 
 // DeployENS deploys an instance of the ENS nameservice, with a 'first-in, first-served' root registrar.
 func DeployENS(transactOpts *bind.TransactOpts, contractBackend bind.ContractBackend) (common.Address, *ENS, error) {
-	// Deploy the ENS registry.
+	// Deploy the ENS registry
 	ensAddr, _, _, err := contract.DeployENS(transactOpts, contractBackend)
 	if err != nil {
 		return ensAddr, nil, err
 	}
-
 	ens, err := NewENS(transactOpts, ensAddr, contractBackend)
 	if err != nil {
 		return ensAddr, nil, err
 	}
-
+	// Deploy the registrar
-	// Deploy the registrar.
 	regAddr, _, _, err := contract.DeployFIFSRegistrar(transactOpts, contractBackend, ensAddr, [32]byte{})
 	if err != nil {
 		return ensAddr, nil, err
 	}
-	// Set the registrar as owner of the ENS root.
+	// Set the registrar as owner of the ENS root
 	if _, err = ens.SetOwner([32]byte{}, regAddr); err != nil {
 		return ensAddr, nil, err
 	}
-
 	return ensAddr, ens, nil
 }
 
@@ -89,10 +85,9 @@ func ensParentNode(name string) (common.Hash, common.Hash) {
 	label := crypto.Keccak256Hash([]byte(parts[0]))
 	if len(parts) == 1 {
 		return [32]byte{}, label
-	} else {
-		parentNode, parentLabel := ensParentNode(parts[1])
-		return crypto.Keccak256Hash(parentNode[:], parentLabel[:]), label
 	}
+	parentNode, parentLabel := ensParentNode(parts[1])
+	return crypto.Keccak256Hash(parentNode[:], parentLabel[:]), label
 }
 
 func EnsNode(name string) common.Hash {
@@ -100,111 +95,101 @@ func EnsNode(name string) common.Hash {
 	return crypto.Keccak256Hash(parentNode[:], parentLabel[:])
 }
 
-func (self *ENS) getResolver(node [32]byte) (*contract.PublicResolverSession, error) {
+func (ens *ENS) getResolver(node [32]byte) (*contract.PublicResolverSession, error) {
-	resolverAddr, err := self.Resolver(node)
+	resolverAddr, err := ens.Resolver(node)
 	if err != nil {
 		return nil, err
 	}
-
+	resolver, err := contract.NewPublicResolver(resolverAddr, ens.contractBackend)
-	resolver, err := contract.NewPublicResolver(resolverAddr, self.contractBackend)
 	if err != nil {
 		return nil, err
 	}
-
 	return &contract.PublicResolverSession{
 		Contract:     resolver,
-		TransactOpts: self.TransactOpts,
+		TransactOpts: ens.TransactOpts,
 	}, nil
 }
 
-func (self *ENS) getRegistrar(node [32]byte) (*contract.FIFSRegistrarSession, error) {
+func (ens *ENS) getRegistrar(node [32]byte) (*contract.FIFSRegistrarSession, error) {
-	registrarAddr, err := self.Owner(node)
+	registrarAddr, err := ens.Owner(node)
 	if err != nil {
 		return nil, err
 	}
-
+	registrar, err := contract.NewFIFSRegistrar(registrarAddr, ens.contractBackend)
-	registrar, err := contract.NewFIFSRegistrar(registrarAddr, self.contractBackend)
 	if err != nil {
 		return nil, err
 	}
-
 	return &contract.FIFSRegistrarSession{
 		Contract:     registrar,
-		TransactOpts: self.TransactOpts,
+		TransactOpts: ens.TransactOpts,
 	}, nil
 }
 
 // Resolve is a non-transactional call that returns the content hash associated with a name.
-func (self *ENS) Resolve(name string) (common.Hash, error) {
+func (ens *ENS) Resolve(name string) (common.Hash, error) {
 	node := EnsNode(name)
 
-	resolver, err := self.getResolver(node)
+	resolver, err := ens.getResolver(node)
 	if err != nil {
 		return common.Hash{}, err
 	}
-
 	ret, err := resolver.Content(node)
 	if err != nil {
 		return common.Hash{}, err
 	}
-
 	return common.BytesToHash(ret[:]), nil
 }
 
 // Addr is a non-transactional call that returns the address associated with a name.
-func (self *ENS) Addr(name string) (common.Address, error) {
+func (ens *ENS) Addr(name string) (common.Address, error) {
 	node := EnsNode(name)
 
-	resolver, err := self.getResolver(node)
+	resolver, err := ens.getResolver(node)
 	if err != nil {
 		return common.Address{}, err
 	}
-
 	ret, err := resolver.Addr(node)
 	if err != nil {
 		return common.Address{}, err
 	}
-
 	return common.BytesToAddress(ret[:]), nil
 }
 
 // SetAddress sets the address associated with a name. Only works if the caller
 // owns the name, and the associated resolver implements a `setAddress` function.
-func (self *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) {
+func (ens *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) {
 	node := EnsNode(name)
 
-	resolver, err := self.getResolver(node)
+	resolver, err := ens.getResolver(node)
 	if err != nil {
 		return nil, err
 	}
-
+	opts := ens.TransactOpts
-	opts := self.TransactOpts
 	opts.GasLimit = 200000
 	return resolver.Contract.SetAddr(&opts, node, addr)
 }
 
 // Register registers a new domain name for the caller, making them the owner of the new name.
 // Only works if the registrar for the parent domain implements the FIFS registrar protocol.
-func (self *ENS) Register(name string) (*types.Transaction, error) {
+func (ens *ENS) Register(name string) (*types.Transaction, error) {
 	parentNode, label := ensParentNode(name)
-	registrar, err := self.getRegistrar(parentNode)
+	registrar, err := ens.getRegistrar(parentNode)
 	if err != nil {
 		return nil, err
 	}
-	return registrar.Contract.Register(&self.TransactOpts, label, self.TransactOpts.From)
+	return registrar.Contract.Register(&ens.TransactOpts, label, ens.TransactOpts.From)
 }
 
 // SetContentHash sets the content hash associated with a name. Only works if the caller
 // owns the name, and the associated resolver implements a `setContent` function.
-func (self *ENS) SetContentHash(name string, hash common.Hash) (*types.Transaction, error) {
+func (ens *ENS) SetContentHash(name string, hash common.Hash) (*types.Transaction, error) {
 	node := EnsNode(name)
 
-	resolver, err := self.getResolver(node)
+	resolver, err := ens.getResolver(node)
 	if err != nil {
 		return nil, err
 	}
-
+	opts := ens.TransactOpts
-	opts := self.TransactOpts
 	opts.GasLimit = 200000
 	return resolver.Contract.SetContent(&opts, node, hash)
 }

core/blockchain.go

@@ -979,20 +979,26 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 				triedb.Cap(limit - ethdb.IdealBatchSize)
 			}
 			// Find the next state trie we need to commit
-			header := bc.GetHeaderByNumber(current - triesInMemory)
+			chosen := current - triesInMemory
-			chosen := header.Number.Uint64()
 
 			// If we exceeded out time allowance, flush an entire trie to disk
 			if bc.gcproc > bc.cacheConfig.TrieTimeLimit {
-				// If we're exceeding limits but haven't reached a large enough memory gap,
+				// If the header is missing (canonical chain behind), we're reorging a low
-				// warn the user that the system is becoming unstable.
+				// diff sidechain. Suspend committing until this operation is completed.
-				if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
+				header := bc.GetHeaderByNumber(chosen)
-					log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+				if header == nil {
+					log.Warn("Reorg in progress, trie commit postponed", "number", chosen)
+				} else {
+					// If we're exceeding limits but haven't reached a large enough memory gap,
+					// warn the user that the system is becoming unstable.
+					if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
+						log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+					}
+					// Flush an entire trie and restart the counters
+					triedb.Commit(header.Root, true)
+					lastWrite = chosen
+					bc.gcproc = 0
 				}
-				// Flush an entire trie and restart the counters
-				triedb.Commit(header.Root, true)
-				lastWrite = chosen
-				bc.gcproc = 0
 			}
 			// Garbage collect anything below our required write retention
 			for !bc.triegc.Empty() {
@@ -1143,7 +1149,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 	switch {
 	// First block is pruned, insert as sidechain and reorg only if TD grows enough
 	case err == consensus.ErrPrunedAncestor:
-		return bc.insertSidechain(it)
+		return bc.insertSidechain(block, it)
 
 	// First block is future, shove it (and all children) to the future queue (unknown ancestor)
 	case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
@@ -1253,8 +1259,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 		stats.processed++
 		stats.usedGas += usedGas
 
-		cache, _ := bc.stateCache.TrieDB().Size()
+		dirty, _ := bc.stateCache.TrieDB().Size()
-		stats.report(chain, it.index, cache)
+		stats.report(chain, it.index, dirty)
 	}
 	// Any blocks remaining here? The only ones we care about are the future ones
 	if block != nil && err == consensus.ErrFutureBlock {
@@ -1285,7 +1291,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 //
 // The method writes all (header-and-body-valid) blocks to disk, then tries to
 // switch over to the new chain if the TD exceeded the current chain.
-func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
+func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (int, []interface{}, []*types.Log, error) {
 	var (
 		externTd *big.Int
 		current  = bc.CurrentBlock()
@@ -1294,7 +1300,7 @@ func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, [
 	// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
 	// ones. Any other errors means that the block is invalid, and should not be written
 	// to disk.
-	block, err := it.current(), consensus.ErrPrunedAncestor
+	err := consensus.ErrPrunedAncestor
 	for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
 		// Check the canonical state root for that number
 		if number := block.NumberU64(); current.NumberU64() >= number {
@@ -1324,7 +1330,7 @@ func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, [
 			if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
 				return it.index, nil, nil, err
 			}
-			log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
+			log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(),
 				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
 				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
 				"root", block.Root())

core/blockchain_insert.go

@@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second
 
 // report prints statistics if some number of blocks have been processed
 // or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
+func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize) {
 	// Fetch the timings for the batch
 	var (
 		now     = mclock.Now()
@@ -63,7 +63,7 @@ func (st *insertStats) report(chain []*types.Block, index int, cache common.Stor
 		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
 		}
-		context = append(context, []interface{}{"cache", cache}...)
+		context = append(context, []interface{}{"dirty", dirty}...)
 
 		if st.queued > 0 {
 			context = append(context, []interface{}{"queued", st.queued}...)
@@ -111,14 +111,6 @@ func (it *insertIterator) next() (*types.Block, error) {
 	return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
 }
 
-// current returns the current block that's being processed.
-func (it *insertIterator) current() *types.Block {
-	if it.index < 0 || it.index+1 >= len(it.chain) {
-		return nil
-	}
-	return it.chain[it.index]
-}
-
 // previous returns the previous block was being processed, or nil
 func (it *insertIterator) previous() *types.Block {
 	if it.index < 1 {

core/blockchain_test.go

@@ -17,7 +17,6 @@
 package core
 
 import (
-	"fmt"
 	"math/big"
 	"math/rand"
 	"sync"
@@ -126,13 +125,6 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara
 	comparator(tdPre, tdPost)
 }
 
-func printChain(bc *BlockChain) {
-	for i := bc.CurrentBlock().Number().Uint64(); i > 0; i-- {
-		b := bc.GetBlockByNumber(uint64(i))
-		fmt.Printf("\t%x %v\n", b.Hash(), b.Difficulty())
-	}
-}
-
 // testBlockChainImport tries to process a chain of blocks, writing them into
 // the database if successful.
 func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
@@ -188,15 +180,6 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
 	return nil
 }
 
-func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *testing.T) {
-	_, err := blockchain.InsertChain(chain)
-	if err != nil {
-		fmt.Println(err)
-		t.FailNow()
-	}
-	done <- true
-}
-
 func TestLastBlock(t *testing.T) {
 	_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
 	if err != nil {
@@ -1483,3 +1466,58 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
 
 	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
 }
+
+// Tests that importing a very large side fork, which is larger than the canon chain,
+// but where the difficulty per block is kept low: this means that it will not
+// overtake the 'canon' chain until after it's passed canon by about 200 blocks.
+//
+// Details at:
+//  - https://github.com/ethereum/go-ethereum/issues/18977
+//  - https://github.com/ethereum/go-ethereum/pull/18988
+func TestLowDiffLongChain(t *testing.T) {
+	// Generate a canonical chain to act as the main dataset
+	engine := ethash.NewFaker()
+	db := ethdb.NewMemDatabase()
+	genesis := new(Genesis).MustCommit(db)
+
+	// We must use a pretty long chain to ensure that the fork doesn't overtake us
+	// until after at least 128 blocks post tip
+	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*triesInMemory, func(i int, b *BlockGen) {
+		b.SetCoinbase(common.Address{1})
+		b.OffsetTime(-9)
+	})
+
+	// Import the canonical chain
+	diskdb := ethdb.NewMemDatabase()
+	new(Genesis).MustCommit(diskdb)
+
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	if n, err := chain.InsertChain(blocks); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+	}
+	// Generate fork chain, starting from an early block
+	parent := blocks[10]
+	fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*triesInMemory, func(i int, b *BlockGen) {
+		b.SetCoinbase(common.Address{2})
+	})
+
+	// And now import the fork
+	if i, err := chain.InsertChain(fork); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", i, err)
+	}
+	head := chain.CurrentBlock()
+	if got := fork[len(fork)-1].Hash(); got != head.Hash() {
+		t.Fatalf("head wrong, expected %x got %x", head.Hash(), got)
+	}
+	// Sanity check that all the canonical numbers are present
+	header := chain.CurrentHeader()
+	for number := head.NumberU64(); number > 0; number-- {
+		if hash := chain.GetHeaderByNumber(number).Hash(); hash != header.Hash() {
+			t.Fatalf("header %d: canonical hash mismatch: have %x, want %x", number, hash, header.Hash())
+		}
+		header = chain.GetHeader(header.ParentHash, number-1)
+	}
+}

core/chain_makers.go

@@ -149,7 +149,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block {
 // associated difficulty. It's useful to test scenarios where forking is not
 // tied to chain length directly.
 func (b *BlockGen) OffsetTime(seconds int64) {
-	b.header.Time.Add(b.header.Time, new(big.Int).SetInt64(seconds))
+	b.header.Time.Add(b.header.Time, big.NewInt(seconds))
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}

core/genesis.go

@@ -157,7 +157,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, constant
 	if genesis != nil && genesis.Config == nil {
 		return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
 	}
-
 	// Just commit the new block if there is no stored genesis block.
 	stored := rawdb.ReadCanonicalHash(db, 0)
 	if (stored == common.Hash{}) {
@@ -340,6 +339,18 @@ func DefaultRinkebyGenesisBlock() *Genesis {
 	}
 }
 
+// DefaultGoerliGenesisBlock returns the Görli network genesis block.
+func DefaultGoerliGenesisBlock() *Genesis {
+	return &Genesis{
+		Config:     params.GoerliChainConfig,
+		Timestamp:  1548854791,
+		ExtraData:  hexutil.MustDecode("0x22466c6578692069732061207468696e6722202d204166726900000000000000e0a2bd4258d2768837baa26a28fe71dc079f84c70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
+		GasLimit:   10485760,
+		Difficulty: big.NewInt(1),
+		Alloc:      decodePrealloc(goerliAllocData),
+	}
+}
+
 // DeveloperGenesisBlock returns the 'geth --dev' genesis block. Note, this must
 // be seeded with the
 func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis {

core/state/statedb.go

@@ -502,7 +502,7 @@ func (self *StateDB) Copy() *StateDB {
 		refund:            self.refund,
 		logs:              make(map[common.Hash][]*types.Log, len(self.logs)),
 		logSize:           self.logSize,
-		preimages:         make(map[common.Hash][]byte),
+		preimages:         make(map[common.Hash][]byte, len(self.preimages)),
 		journal:           newJournal(),
 	}
 	// Copy the dirty states, logs, and preimages

core/state/statedb_test.go

@@ -276,6 +276,15 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction {
 			},
 			args: make([]int64, 1),
 		},
+		{
+			name: "AddPreimage",
+			fn: func(a testAction, s *StateDB) {
+				preimage := []byte{1}
+				hash := common.BytesToHash(preimage)
+				s.AddPreimage(hash, preimage)
+			},
+			args: make([]int64, 1),
+		},
 	}
 	action := actions[r.Intn(len(actions))]
 	var nameargs []string

core/tx_pool_test.go

@@ -1274,14 +1274,14 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
 	}
 	// Create transaction (both pending and queued) with a linearly growing gasprice
 	for i := uint64(0); i < 500; i++ {
-		// Add pending
+		// Add pending transaction.
-		p_tx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
+		pendingTx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
-		if err := pool.AddLocal(p_tx); err != nil {
+		if err := pool.AddLocal(pendingTx); err != nil {
 			t.Fatal(err)
 		}
-		// Add queued
+		// Add queued transaction.
-		q_tx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
+		queuedTx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
-		if err := pool.AddLocal(q_tx); err != nil {
+		if err := pool.AddLocal(queuedTx); err != nil {
 			t.Fatal(err)
 		}
 	}

core/types/block.go

@@ -21,10 +21,10 @@ import (
 	"encoding/binary"
 	"io"
 	"math/big"
+	"reflect"
 	"sort"
 	"sync/atomic"
 	"time"
-	"unsafe"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -102,10 +102,12 @@ func (h *Header) Hash() common.Hash {
 	return rlpHash(h)
 }
 
+var headerSize = common.StorageSize(reflect.TypeOf(Header{}).Size())
+
 // Size returns the approximate memory used by all internal contents. It is used
 // to approximate and limit the memory consumption of various caches.
 func (h *Header) Size() common.StorageSize {
-	return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8)
+	return headerSize + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8)
 }
 
 func rlpHash(x interface{}) (h common.Hash) {

core/vm/evm.go

@@ -267,9 +267,8 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 		snapshot = evm.StateDB.Snapshot()
 		to       = AccountRef(caller.Address())
 	)
-	// initialise a new contract and set the code that is to be used by the
+	// Initialise a new contract and set the code that is to be used by the EVM.
-	// EVM. The contract is a scoped environment for this execution context
+	// The contract is a scoped environment for this execution context only.
-	// only.
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
@@ -333,9 +332,8 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 		to       = AccountRef(addr)
 		snapshot = evm.StateDB.Snapshot()
 	)
-	// Initialise a new contract and set the code that is to be used by the
+	// Initialise a new contract and set the code that is to be used by the EVM.
-	// EVM. The contract is a scoped environment for this execution context
+	// The contract is a scoped environment for this execution context only.
-	// only.
 	contract := NewContract(caller, to, new(big.Int), gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
@@ -396,9 +394,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
 	}
 	evm.Transfer(evm.StateDB, caller.Address(), address, value)
 
-	// initialise a new contract and set the code that is to be used by the
+	// Initialise a new contract and set the code that is to be used by the EVM.
-	// EVM. The contract is a scoped environment for this execution context
+	// The contract is a scoped environment for this execution context only.
-	// only.
 	contract := NewContract(caller, AccountRef(address), value, gas)
 	contract.SetCodeOptionalHash(&address, codeAndHash)
 

core/vm/interpreter.go

@@ -182,6 +182,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
 		pcCopy  uint64 // needed for the deferred Tracer
 		gasCopy uint64 // for Tracer to log gas remaining before execution
 		logged  bool   // deferred Tracer should ignore already logged steps
+		res     []byte // result of the opcode execution function
 	)
 	contract.Input = input
 
@@ -216,11 +217,11 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
 		if !operation.valid {
 			return nil, fmt.Errorf("invalid opcode 0x%x", int(op))
 		}
-		if err := operation.validateStack(stack); err != nil {
+		if err = operation.validateStack(stack); err != nil {
 			return nil, err
 		}
 		// If the operation is valid, enforce and write restrictions
-		if err := in.enforceRestrictions(op, operation, stack); err != nil {
+		if err = in.enforceRestrictions(op, operation, stack); err != nil {
 			return nil, err
 		}
 
@@ -254,7 +255,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
 		}
 
 		// execute the operation
-		res, err := operation.execute(&pc, in, contract, mem, stack)
+		res, err = operation.execute(&pc, in, contract, mem, stack)
 		// verifyPool is a build flag. Pool verification makes sure the integrity
 		// of the integer pool by comparing values to a default value.
 		if verifyPool {

crypto/ecies/ecies_test.go

@@ -93,19 +93,6 @@ func cmpPublic(pub1, pub2 PublicKey) bool {
 	return bytes.Equal(pub1Out, pub2Out)
 }
 
-// cmpPrivate returns true if the two private keys are the same.
-func cmpPrivate(prv1, prv2 *PrivateKey) bool {
-	if prv1 == nil || prv1.D == nil {
-		return false
-	} else if prv2 == nil || prv2.D == nil {
-		return false
-	} else if prv1.D.Cmp(prv2.D) != 0 {
-		return false
-	} else {
-		return cmpPublic(prv1.PublicKey, prv2.PublicKey)
-	}
-}
-
 // Validate the ECDH component.
 func TestSharedKey(t *testing.T) {
 	prv1, err := GenerateKey(rand.Reader, DefaultCurve, nil)

eth/api_tracer.go

@@ -214,7 +214,8 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 						log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err)
 						break
 					}
-					task.statedb.Finalise(true)
+					// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
+					task.statedb.Finalise(api.eth.blockchain.Config().IsEIP158(task.block.Number()))
 					task.results[i] = &txTraceResult{Result: res}
 				}
 				// Stream the result back to the user or abort on teardown
@@ -506,7 +507,8 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block,
 			break
 		}
 		// Finalize the state so any modifications are written to the trie
-		statedb.Finalise(true)
+		// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
+		statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number()))
 	}
 	close(jobs)
 	pend.Wait()
@@ -612,7 +614,8 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block
 			return dumps, err
 		}
 		// Finalize the state so any modifications are written to the trie
-		statedb.Finalise(true)
+		// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
+		statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number()))
 
 		// If we've traced the transaction we were looking for, abort
 		if tx.Hash() == txHash {
@@ -803,7 +806,8 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree
 			return nil, vm.Context{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err)
 		}
 		// Ensure any modifications are committed to the state
-		statedb.Finalise(true)
+		// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
+		statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number()))
 	}
 	return nil, vm.Context{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, blockHash)
 }

eth/backend.go

@@ -113,6 +113,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		log.Warn("Sanitizing invalid miner gas price", "provided", config.MinerGasPrice, "updated", DefaultConfig.MinerGasPrice)
 		config.MinerGasPrice = new(big.Int).Set(DefaultConfig.MinerGasPrice)
 	}
+	if config.NoPruning && config.TrieDirtyCache > 0 {
+		config.TrieCleanCache += config.TrieDirtyCache
+		config.TrieDirtyCache = 0
+	}
+	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
+
 	// Assemble the Ethereum object
 	chainDb, err := CreateDB(ctx, config, "chaindata")
 	if err != nil {
@@ -434,7 +440,7 @@ func (s *Ethereum) StartMining(threads int) error {
 				log.Error("Etherbase account unavailable locally", "err", err)
 				return fmt.Errorf("signer missing: %v", err)
 			}
-			clique.Authorize(eb, wallet.SignHash)
+			clique.Authorize(eb, wallet.SignData)
 		}
 		// If mining is started, we can disable the transaction rejection mechanism
 		// introduced to speed sync times.

eth/downloader/downloader.go

@@ -719,7 +719,7 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header)
 			}
 			// Make sure the peer's reply conforms to the request
 			for i, header := range headers {
-				expectNumber := from + int64(i)*int64((skip+1))
+				expectNumber := from + int64(i)*int64(skip+1)
 				if number := header.Number.Int64(); number != expectNumber {
 					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
 					return 0, errInvalidChain

eth/fetcher/fetcher.go

@@ -110,7 +110,6 @@ type Fetcher struct {
 	notify chan *announce
 	inject chan *inject
 
-	blockFilter  chan chan []*types.Block
 	headerFilter chan chan *headerFilterTask
 	bodyFilter   chan chan *bodyFilterTask
 
@@ -150,7 +149,6 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc
 	return &Fetcher{
 		notify:         make(chan *announce),
 		inject:         make(chan *inject),
-		blockFilter:    make(chan chan []*types.Block),
 		headerFilter:   make(chan chan *headerFilterTask),
 		bodyFilter:     make(chan chan *bodyFilterTask),
 		done:           make(chan common.Hash),

eth/tracers/tracers_test.go

@@ -121,7 +121,7 @@ type callTracerTest struct {
 }
 
 func TestPrestateTracerCreate2(t *testing.T) {
-	unsigned_tx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"),
+	unsignedTx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"),
 		new(big.Int), 5000000, big.NewInt(1), []byte{})
 
 	privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader)
@@ -129,7 +129,7 @@ func TestPrestateTracerCreate2(t *testing.T) {
 		t.Fatalf("err %v", err)
 	}
 	signer := types.NewEIP155Signer(big.NewInt(1))
-	tx, err := types.SignTx(unsigned_tx, signer, privateKeyECDSA)
+	tx, err := types.SignTx(unsignedTx, signer, privateKeyECDSA)
 	if err != nil {
 		t.Fatalf("err %v", err)
 	}

ethdb/database.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -70,7 +71,7 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
 	if handles < 16 {
 		handles = 16
 	}
-	logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles)
+	logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles)
 
 	// Open the db and recover any potential corruptions
 	db, err := leveldb.OpenFile(file, &opt.Options{

interfaces.go

@@ -144,9 +144,9 @@ type FilterQuery struct {
 	// Examples:
 	// {} or nil          matches any topic list
 	// {{A}}              matches topic A in first position
-	// {{}, {B}}          matches any topic in first position, B in second position
+	// {{}, {B}}          matches any topic in first position AND B in second position
-	// {{A}, {B}}         matches topic A in first position, B in second position
+	// {{A}, {B}}         matches topic A in first position AND B in second position
-	// {{A, B}, {C, D}}   matches topic (A OR B) in first position, (C OR D) in second position
+	// {{A, B}, {C, D}}   matches topic (A OR B) in first position AND (C OR D) in second position
 	Topics [][]common.Hash
 }
 

internal/build/util.go

@@ -177,3 +177,34 @@ func ExpandPackagesNoVendor(patterns []string) []string {
 	}
 	return patterns
 }
+
+// UploadSFTP uploads files to a remote host using the sftp command line tool.
+// The destination host may be specified either as [user@]host[: or as a URI in
+// the form sftp://[user@]host[:port].
+func UploadSFTP(identityFile, host, dir string, files []string) error {
+	sftp := exec.Command("sftp")
+	sftp.Stdout = nil
+	sftp.Stderr = os.Stderr
+	if identityFile != "" {
+		sftp.Args = append(sftp.Args, "-i", identityFile)
+	}
+	sftp.Args = append(sftp.Args, host)
+	fmt.Println(">>>", strings.Join(sftp.Args, " "))
+	if *DryRunFlag {
+		return nil
+	}
+
+	stdin, err := sftp.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("can't create stdin pipe for sftp: %v", err)
+	}
+	if err := sftp.Start(); err != nil {
+		return err
+	}
+	in := io.MultiWriter(stdin, os.Stdout)
+	for _, f := range files {
+		fmt.Fprintln(in, "put", f, path.Join(dir, filepath.Base(f)))
+	}
+	stdin.Close()
+	return sftp.Wait()
+}

internal/ethapi/api.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -217,7 +218,7 @@ func NewPrivateAccountAPI(b Backend, nonceLock *AddrLocker) *PrivateAccountAPI {
 	}
 }
 
-// ListAccounts will return a list of addresses for accounts this node manages.
+// listAccounts will return a list of addresses for accounts this node manages.
 func (s *PrivateAccountAPI) ListAccounts() []common.Address {
 	addresses := make([]common.Address, 0) // return [] instead of nil if empty
 	for _, wallet := range s.am.Wallets() {
@@ -356,11 +357,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArg
 	// Assemble the transaction and sign with the wallet
 	tx := args.toTransaction()
 
-	var chainID *big.Int
+	return wallet.SignTxWithPassphrase(account, passwd, tx, s.b.ChainConfig().ChainID)
-	if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
-		chainID = config.ChainID
-	}
-	return wallet.SignTxWithPassphrase(account, passwd, tx, chainID)
 }
 
 // SendTransaction will create a transaction from the given arguments and
@@ -409,18 +406,6 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs
 	return &SignTransactionResult{data, signed}, nil
 }
 
-// signHash is a helper function that calculates a hash for the given message that can be
-// safely used to calculate a signature from.
-//
-// The hash is calulcated as
-//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
-//
-// This gives context to the signed message and prevents signing of transactions.
-func signHash(data []byte) []byte {
-	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
-	return crypto.Keccak256([]byte(msg))
-}
-
 // Sign calculates an Ethereum ECDSA signature for:
 // keccack256("\x19Ethereum Signed Message:\n" + len(message) + message))
 //
@@ -439,7 +424,7 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr c
 		return nil, err
 	}
 	// Assemble sign the data with the wallet
-	signature, err := wallet.SignHashWithPassphrase(account, passwd, signHash(data))
+	signature, err := wallet.SignTextWithPassphrase(account, passwd, data)
 	if err != nil {
 		log.Warn("Failed data sign attempt", "address", addr, "err", err)
 		return nil, err
@@ -467,7 +452,7 @@ func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Byt
 	}
 	sig[64] -= 27 // Transform yellow paper V from 27/28 to 0/1
 
-	rpk, err := crypto.SigToPub(signHash(data), sig)
+	rpk, err := crypto.SigToPub(accounts.TextHash(data), sig)
 	if err != nil {
 		return common.Address{}, err
 	}
@@ -1198,11 +1183,7 @@ func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transacti
 		return nil, err
 	}
 	// Request the wallet to sign the transaction
-	var chainID *big.Int
+	return wallet.SignTx(account, tx, s.b.ChainConfig().ChainID)
-	if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
-		chainID = config.ChainID
-	}
-	return wallet.SignTx(account, tx, chainID)
 }
 
 // SendTxArgs represents the arguments to sumbit a new transaction into the transaction pool.
@@ -1318,11 +1299,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen
 	// Assemble the transaction and sign with the wallet
 	tx := args.toTransaction()
 
-	var chainID *big.Int
+	signed, err := wallet.SignTx(account, tx, s.b.ChainConfig().ChainID)
-	if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
-		chainID = config.ChainID
-	}
-	signed, err := wallet.SignTx(account, tx, chainID)
 	if err != nil {
 		return common.Hash{}, err
 	}
@@ -1357,7 +1334,7 @@ func (s *PublicTransactionPoolAPI) Sign(addr common.Address, data hexutil.Bytes)
 		return nil, err
 	}
 	// Sign the requested hash with the wallet
-	signature, err := wallet.SignHash(account, signHash(data))
+	signature, err := wallet.SignText(account, data)
 	if err == nil {
 		signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
 	}
@@ -1493,6 +1470,45 @@ func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (stri
 	return fmt.Sprintf("%x", encoded), nil
 }
 
+// TestSignCliqueBlock fetches the given block number, and attempts to sign it as a clique header with the
+// given address, returning the address of the recovered signature
+//
+// This is a temporary method to debug the externalsigner integration,
+// TODO: Remove this method when the integration is mature
+func (api *PublicDebugAPI) TestSignCliqueBlock(ctx context.Context, address common.Address, number uint64) (common.Address, error) {
+	block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))
+	if block == nil {
+		return common.Address{}, fmt.Errorf("block #%d not found", number)
+	}
+	header := block.Header()
+	header.Extra = make([]byte, 32+65)
+	encoded := clique.CliqueRLP(header)
+
+	// Look up the wallet containing the requested signer
+	account := accounts.Account{Address: address}
+	wallet, err := api.b.AccountManager().Find(account)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	signature, err := wallet.SignData(account, accounts.MimetypeClique, encoded)
+	if err != nil {
+		return common.Address{}, err
+	}
+	sealHash := clique.SealHash(header).Bytes()
+	log.Info("test signing of clique block",
+		"Sealhash", fmt.Sprintf("%x", sealHash),
+		"signature", fmt.Sprintf("%x", signature))
+	pubkey, err := crypto.Ecrecover(sealHash, signature)
+	if err != nil {
+		return common.Address{}, err
+	}
+	var signer common.Address
+	copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
+
+	return signer, nil
+}
+
 // PrintBlock retrieves a block and returns its pretty printed form.
 func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) {
 	block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))

internal/jsre/jsre.go

@@ -32,8 +32,8 @@ import (
 )
 
 var (
-	BigNumber_JS = deps.MustAsset("bignumber.js")
+	BignumberJs = deps.MustAsset("bignumber.js")
-	Web3_JS      = deps.MustAsset("web3.js")
+	Web3Js      = deps.MustAsset("web3.js")
 )
 
 /*

internal/web3ext/web3ext.go

@@ -18,23 +18,23 @@
 package web3ext
 
 var Modules = map[string]string{
-	"accounting": Accounting_JS,
+	"accounting": AccountingJs,
-	"admin":      Admin_JS,
+	"admin":      AdminJs,
-	"chequebook": Chequebook_JS,
+	"chequebook": ChequebookJs,
-	"clique":     Clique_JS,
+	"clique":     CliqueJs,
-	"ethash":     Ethash_JS,
+	"ethash":     EthashJs,
-	"debug":      Debug_JS,
+	"debug":      DebugJs,
-	"eth":        Eth_JS,
+	"eth":        EthJs,
-	"miner":      Miner_JS,
+	"miner":      MinerJs,
-	"net":        Net_JS,
+	"net":        NetJs,
-	"personal":   Personal_JS,
+	"personal":   PersonalJs,
-	"rpc":        RPC_JS,
+	"rpc":        RpcJs,
-	"shh":        Shh_JS,
+	"shh":        ShhJs,
-	"swarmfs":    SWARMFS_JS,
+	"swarmfs":    SwarmfsJs,
-	"txpool":     TxPool_JS,
+	"txpool":     TxpoolJs,
 }
 
-const Chequebook_JS = `
+const ChequebookJs = `
 web3._extend({
 	property: 'chequebook',
 	methods: [
@@ -65,7 +65,7 @@ web3._extend({
 });
 `
 
-const Clique_JS = `
+const CliqueJs = `
 web3._extend({
 	property: 'clique',
 	methods: [
@@ -111,7 +111,7 @@ web3._extend({
 });
 `
 
-const Ethash_JS = `
+const EthashJs = `
 web3._extend({
 	property: 'ethash',
 	methods: [
@@ -139,7 +139,7 @@ web3._extend({
 });
 `
 
-const Admin_JS = `
+const AdminJs = `
 web3._extend({
 	property: 'admin',
 	methods: [
@@ -217,7 +217,7 @@ web3._extend({
 });
 `
 
-const Debug_JS = `
+const DebugJs = `
 web3._extend({
 	property: 'debug',
 	methods: [
@@ -231,6 +231,12 @@ web3._extend({
 			call: 'debug_getBlockRlp',
 			params: 1
 		}),
+		new web3._extend.Method({
+			name: 'testSignCliqueBlock',
+			call: 'debug_testSignCliqueBlock',
+			params: 2, 
+			inputFormatters: [web3._extend.formatters.inputAddressFormatter, null],
+		}),
 		new web3._extend.Method({
 			name: 'setHead',
 			call: 'debug_setHead',
@@ -448,7 +454,7 @@ web3._extend({
 });
 `
 
-const Eth_JS = `
+const EthJs = `
 web3._extend({
 	property: 'eth',
 	methods: [
@@ -518,7 +524,7 @@ web3._extend({
 });
 `
 
-const Miner_JS = `
+const MinerJs = `
 web3._extend({
 	property: 'miner',
 	methods: [
@@ -563,7 +569,7 @@ web3._extend({
 });
 `
 
-const Net_JS = `
+const NetJs = `
 web3._extend({
 	property: 'net',
 	methods: [],
@@ -576,7 +582,7 @@ web3._extend({
 });
 `
 
-const Personal_JS = `
+const PersonalJs = `
 web3._extend({
 	property: 'personal',
 	methods: [
@@ -622,7 +628,7 @@ web3._extend({
 })
 `
 
-const RPC_JS = `
+const RpcJs = `
 web3._extend({
 	property: 'rpc',
 	methods: [],
@@ -635,7 +641,7 @@ web3._extend({
 });
 `
 
-const Shh_JS = `
+const ShhJs = `
 web3._extend({
 	property: 'shh',
 	methods: [
@@ -655,7 +661,7 @@ web3._extend({
 });
 `
 
-const SWARMFS_JS = `
+const SwarmfsJs = `
 web3._extend({
 	property: 'swarmfs',
 	methods:
@@ -679,7 +685,7 @@ web3._extend({
 });
 `
 
-const TxPool_JS = `
+const TxpoolJs = `
 web3._extend({
 	property: 'txpool',
 	methods: [],
@@ -706,7 +712,7 @@ web3._extend({
 });
 `
 
-const Accounting_JS = `
+const AccountingJs = `
 web3._extend({
 	property: 'accounting',
 	methods: [

light/lightchain.go

@@ -117,45 +117,45 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.
 }
 
 // addTrustedCheckpoint adds a trusted checkpoint to the blockchain
-func (self *LightChain) addTrustedCheckpoint(cp *params.TrustedCheckpoint) {
+func (lc *LightChain) addTrustedCheckpoint(cp *params.TrustedCheckpoint) {
-	if self.odr.ChtIndexer() != nil {
+	if lc.odr.ChtIndexer() != nil {
-		StoreChtRoot(self.chainDb, cp.SectionIndex, cp.SectionHead, cp.CHTRoot)
+		StoreChtRoot(lc.chainDb, cp.SectionIndex, cp.SectionHead, cp.CHTRoot)
-		self.odr.ChtIndexer().AddCheckpoint(cp.SectionIndex, cp.SectionHead)
+		lc.odr.ChtIndexer().AddCheckpoint(cp.SectionIndex, cp.SectionHead)
 	}
-	if self.odr.BloomTrieIndexer() != nil {
+	if lc.odr.BloomTrieIndexer() != nil {
-		StoreBloomTrieRoot(self.chainDb, cp.SectionIndex, cp.SectionHead, cp.BloomRoot)
+		StoreBloomTrieRoot(lc.chainDb, cp.SectionIndex, cp.SectionHead, cp.BloomRoot)
-		self.odr.BloomTrieIndexer().AddCheckpoint(cp.SectionIndex, cp.SectionHead)
+		lc.odr.BloomTrieIndexer().AddCheckpoint(cp.SectionIndex, cp.SectionHead)
 	}
-	if self.odr.BloomIndexer() != nil {
+	if lc.odr.BloomIndexer() != nil {
-		self.odr.BloomIndexer().AddCheckpoint(cp.SectionIndex, cp.SectionHead)
+		lc.odr.BloomIndexer().AddCheckpoint(cp.SectionIndex, cp.SectionHead)
 	}
-	log.Info("Added trusted checkpoint", "chain", cp.Name, "block", (cp.SectionIndex+1)*self.indexerConfig.ChtSize-1, "hash", cp.SectionHead)
+	log.Info("Added trusted checkpoint", "chain", cp.Name, "block", (cp.SectionIndex+1)*lc.indexerConfig.ChtSize-1, "hash", cp.SectionHead)
 }
 
-func (self *LightChain) getProcInterrupt() bool {
+func (lc *LightChain) getProcInterrupt() bool {
-	return atomic.LoadInt32(&self.procInterrupt) == 1
+	return atomic.LoadInt32(&lc.procInterrupt) == 1
 }
 
 // Odr returns the ODR backend of the chain
-func (self *LightChain) Odr() OdrBackend {
+func (lc *LightChain) Odr() OdrBackend {
-	return self.odr
+	return lc.odr
 }
 
 // loadLastState loads the last known chain state from the database. This method
 // assumes that the chain manager mutex is held.
-func (self *LightChain) loadLastState() error {
+func (lc *LightChain) loadLastState() error {
-	if head := rawdb.ReadHeadHeaderHash(self.chainDb); head == (common.Hash{}) {
+	if head := rawdb.ReadHeadHeaderHash(lc.chainDb); head == (common.Hash{}) {
 		// Corrupt or empty database, init from scratch
-		self.Reset()
+		lc.Reset()
 	} else {
-		if header := self.GetHeaderByHash(head); header != nil {
+		if header := lc.GetHeaderByHash(head); header != nil {
-			self.hc.SetCurrentHeader(header)
+			lc.hc.SetCurrentHeader(header)
 		}
 	}
 
 	// Issue a status log and return
-	header := self.hc.CurrentHeader()
+	header := lc.hc.CurrentHeader()
-	headerTd := self.GetTd(header.Hash(), header.Number.Uint64())
+	headerTd := lc.GetTd(header.Hash(), header.Number.Uint64())
 	log.Info("Loaded most recent local header", "number", header.Number, "hash", header.Hash(), "td", headerTd, "age", common.PrettyAge(time.Unix(header.Time.Int64(), 0)))
 
 	return nil
@@ -163,181 +163,181 @@ func (self *LightChain) loadLastState() error {
 
 // SetHead rewinds the local chain to a new head. Everything above the new
 // head will be deleted and the new one set.
-func (bc *LightChain) SetHead(head uint64) {
+func (lc *LightChain) SetHead(head uint64) {
-	bc.chainmu.Lock()
+	lc.chainmu.Lock()
-	defer bc.chainmu.Unlock()
+	defer lc.chainmu.Unlock()
 
-	bc.hc.SetHead(head, nil)
+	lc.hc.SetHead(head, nil)
-	bc.loadLastState()
+	lc.loadLastState()
 }
 
 // GasLimit returns the gas limit of the current HEAD block.
-func (self *LightChain) GasLimit() uint64 {
+func (lc *LightChain) GasLimit() uint64 {
-	return self.hc.CurrentHeader().GasLimit
+	return lc.hc.CurrentHeader().GasLimit
 }
 
 // Reset purges the entire blockchain, restoring it to its genesis state.
-func (bc *LightChain) Reset() {
+func (lc *LightChain) Reset() {
-	bc.ResetWithGenesisBlock(bc.genesisBlock)
+	lc.ResetWithGenesisBlock(lc.genesisBlock)
 }
 
 // ResetWithGenesisBlock purges the entire blockchain, restoring it to the
 // specified genesis state.
-func (bc *LightChain) ResetWithGenesisBlock(genesis *types.Block) {
+func (lc *LightChain) ResetWithGenesisBlock(genesis *types.Block) {
 	// Dump the entire block chain and purge the caches
-	bc.SetHead(0)
+	lc.SetHead(0)
 
-	bc.chainmu.Lock()
+	lc.chainmu.Lock()
-	defer bc.chainmu.Unlock()
+	defer lc.chainmu.Unlock()
 
 	// Prepare the genesis block and reinitialise the chain
-	rawdb.WriteTd(bc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty())
+	rawdb.WriteTd(lc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty())
-	rawdb.WriteBlock(bc.chainDb, genesis)
+	rawdb.WriteBlock(lc.chainDb, genesis)
 
-	bc.genesisBlock = genesis
+	lc.genesisBlock = genesis
-	bc.hc.SetGenesis(bc.genesisBlock.Header())
+	lc.hc.SetGenesis(lc.genesisBlock.Header())
-	bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
+	lc.hc.SetCurrentHeader(lc.genesisBlock.Header())
 }
 
 // Accessors
 
 // Engine retrieves the light chain's consensus engine.
-func (bc *LightChain) Engine() consensus.Engine { return bc.engine }
+func (lc *LightChain) Engine() consensus.Engine { return lc.engine }
 
 // Genesis returns the genesis block
-func (bc *LightChain) Genesis() *types.Block {
+func (lc *LightChain) Genesis() *types.Block {
-	return bc.genesisBlock
+	return lc.genesisBlock
 }
 
 // State returns a new mutable state based on the current HEAD block.
-func (bc *LightChain) State() (*state.StateDB, error) {
+func (lc *LightChain) State() (*state.StateDB, error) {
 	return nil, errors.New("not implemented, needs client/server interface split")
 }
 
 // GetBody retrieves a block body (transactions and uncles) from the database
 // or ODR service by hash, caching it if found.
-func (self *LightChain) GetBody(ctx context.Context, hash common.Hash) (*types.Body, error) {
+func (lc *LightChain) GetBody(ctx context.Context, hash common.Hash) (*types.Body, error) {
 	// Short circuit if the body's already in the cache, retrieve otherwise
-	if cached, ok := self.bodyCache.Get(hash); ok {
+	if cached, ok := lc.bodyCache.Get(hash); ok {
 		body := cached.(*types.Body)
 		return body, nil
 	}
-	number := self.hc.GetBlockNumber(hash)
+	number := lc.hc.GetBlockNumber(hash)
 	if number == nil {
 		return nil, errors.New("unknown block")
 	}
-	body, err := GetBody(ctx, self.odr, hash, *number)
+	body, err := GetBody(ctx, lc.odr, hash, *number)
 	if err != nil {
 		return nil, err
 	}
 	// Cache the found body for next time and return
-	self.bodyCache.Add(hash, body)
+	lc.bodyCache.Add(hash, body)
 	return body, nil
 }
 
 // GetBodyRLP retrieves a block body in RLP encoding from the database or
 // ODR service by hash, caching it if found.
-func (self *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.RawValue, error) {
+func (lc *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.RawValue, error) {
 	// Short circuit if the body's already in the cache, retrieve otherwise
-	if cached, ok := self.bodyRLPCache.Get(hash); ok {
+	if cached, ok := lc.bodyRLPCache.Get(hash); ok {
 		return cached.(rlp.RawValue), nil
 	}
-	number := self.hc.GetBlockNumber(hash)
+	number := lc.hc.GetBlockNumber(hash)
 	if number == nil {
 		return nil, errors.New("unknown block")
 	}
-	body, err := GetBodyRLP(ctx, self.odr, hash, *number)
+	body, err := GetBodyRLP(ctx, lc.odr, hash, *number)
 	if err != nil {
 		return nil, err
 	}
 	// Cache the found body for next time and return
-	self.bodyRLPCache.Add(hash, body)
+	lc.bodyRLPCache.Add(hash, body)
 	return body, nil
 }
 
 // HasBlock checks if a block is fully present in the database or not, caching
 // it if present.
-func (bc *LightChain) HasBlock(hash common.Hash, number uint64) bool {
+func (lc *LightChain) HasBlock(hash common.Hash, number uint64) bool {
-	blk, _ := bc.GetBlock(NoOdr, hash, number)
+	blk, _ := lc.GetBlock(NoOdr, hash, number)
 	return blk != nil
 }
 
 // GetBlock retrieves a block from the database or ODR service by hash and number,
 // caching it if found.
-func (self *LightChain) GetBlock(ctx context.Context, hash common.Hash, number uint64) (*types.Block, error) {
+func (lc *LightChain) GetBlock(ctx context.Context, hash common.Hash, number uint64) (*types.Block, error) {
 	// Short circuit if the block's already in the cache, retrieve otherwise
-	if block, ok := self.blockCache.Get(hash); ok {
+	if block, ok := lc.blockCache.Get(hash); ok {
 		return block.(*types.Block), nil
 	}
-	block, err := GetBlock(ctx, self.odr, hash, number)
+	block, err := GetBlock(ctx, lc.odr, hash, number)
 	if err != nil {
 		return nil, err
 	}
 	// Cache the found block for next time and return
-	self.blockCache.Add(block.Hash(), block)
+	lc.blockCache.Add(block.Hash(), block)
 	return block, nil
 }
 
 // GetBlockByHash retrieves a block from the database or ODR service by hash,
 // caching it if found.
-func (self *LightChain) GetBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) {
+func (lc *LightChain) GetBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) {
-	number := self.hc.GetBlockNumber(hash)
+	number := lc.hc.GetBlockNumber(hash)
 	if number == nil {
 		return nil, errors.New("unknown block")
 	}
-	return self.GetBlock(ctx, hash, *number)
+	return lc.GetBlock(ctx, hash, *number)
 }
 
 // GetBlockByNumber retrieves a block from the database or ODR service by
 // number, caching it (associated with its hash) if found.
-func (self *LightChain) GetBlockByNumber(ctx context.Context, number uint64) (*types.Block, error) {
+func (lc *LightChain) GetBlockByNumber(ctx context.Context, number uint64) (*types.Block, error) {
-	hash, err := GetCanonicalHash(ctx, self.odr, number)
+	hash, err := GetCanonicalHash(ctx, lc.odr, number)
 	if hash == (common.Hash{}) || err != nil {
 		return nil, err
 	}
-	return self.GetBlock(ctx, hash, number)
+	return lc.GetBlock(ctx, hash, number)
 }
 
 // Stop stops the blockchain service. If any imports are currently in progress
 // it will abort them using the procInterrupt.
-func (bc *LightChain) Stop() {
+func (lc *LightChain) Stop() {
-	if !atomic.CompareAndSwapInt32(&bc.running, 0, 1) {
+	if !atomic.CompareAndSwapInt32(&lc.running, 0, 1) {
 		return
 	}
-	close(bc.quit)
+	close(lc.quit)
-	atomic.StoreInt32(&bc.procInterrupt, 1)
+	atomic.StoreInt32(&lc.procInterrupt, 1)
 
-	bc.wg.Wait()
+	lc.wg.Wait()
 	log.Info("Blockchain manager stopped")
 }
 
 // Rollback is designed to remove a chain of links from the database that aren't
 // certain enough to be valid.
-func (self *LightChain) Rollback(chain []common.Hash) {
+func (lc *LightChain) Rollback(chain []common.Hash) {
-	self.chainmu.Lock()
+	lc.chainmu.Lock()
-	defer self.chainmu.Unlock()
+	defer lc.chainmu.Unlock()
 
 	for i := len(chain) - 1; i >= 0; i-- {
 		hash := chain[i]
 
-		if head := self.hc.CurrentHeader(); head.Hash() == hash {
+		if head := lc.hc.CurrentHeader(); head.Hash() == hash {
-			self.hc.SetCurrentHeader(self.GetHeader(head.ParentHash, head.Number.Uint64()-1))
+			lc.hc.SetCurrentHeader(lc.GetHeader(head.ParentHash, head.Number.Uint64()-1))
 		}
 	}
 }
 
 // postChainEvents iterates over the events generated by a chain insertion and
 // posts them into the event feed.
-func (self *LightChain) postChainEvents(events []interface{}) {
+func (lc *LightChain) postChainEvents(events []interface{}) {
 	for _, event := range events {
 		switch ev := event.(type) {
 		case core.ChainEvent:
-			if self.CurrentHeader().Hash() == ev.Hash {
+			if lc.CurrentHeader().Hash() == ev.Hash {
-				self.chainHeadFeed.Send(core.ChainHeadEvent{Block: ev.Block})
+				lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: ev.Block})
 			}
-			self.chainFeed.Send(ev)
+			lc.chainFeed.Send(ev)
 		case core.ChainSideEvent:
-			self.chainSideFeed.Send(ev)
+			lc.chainSideFeed.Send(ev)
 		}
 	}
 }
@@ -353,25 +353,25 @@ func (self *LightChain) postChainEvents(events []interface{}) {
 //
 // In the case of a light chain, InsertHeaderChain also creates and posts light
 // chain events when necessary.
-func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) {
+func (lc *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) {
-	if atomic.LoadInt32(&self.disableCheckFreq) == 1 {
+	if atomic.LoadInt32(&lc.disableCheckFreq) == 1 {
 		checkFreq = 0
 	}
 	start := time.Now()
-	if i, err := self.hc.ValidateHeaderChain(chain, checkFreq); err != nil {
+	if i, err := lc.hc.ValidateHeaderChain(chain, checkFreq); err != nil {
 		return i, err
 	}
 
 	// Make sure only one thread manipulates the chain at once
-	self.chainmu.Lock()
+	lc.chainmu.Lock()
-	defer self.chainmu.Unlock()
+	defer lc.chainmu.Unlock()
 
-	self.wg.Add(1)
+	lc.wg.Add(1)
-	defer self.wg.Done()
+	defer lc.wg.Done()
 
 	var events []interface{}
 	whFunc := func(header *types.Header) error {
-		status, err := self.hc.WriteHeader(header)
+		status, err := lc.hc.WriteHeader(header)
 
 		switch status {
 		case core.CanonStatTy:
@@ -384,51 +384,51 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int)
 		}
 		return err
 	}
-	i, err := self.hc.InsertHeaderChain(chain, whFunc, start)
+	i, err := lc.hc.InsertHeaderChain(chain, whFunc, start)
-	self.postChainEvents(events)
+	lc.postChainEvents(events)
 	return i, err
 }
 
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
-func (self *LightChain) CurrentHeader() *types.Header {
+func (lc *LightChain) CurrentHeader() *types.Header {
-	return self.hc.CurrentHeader()
+	return lc.hc.CurrentHeader()
 }
 
 // GetTd retrieves a block's total difficulty in the canonical chain from the
 // database by hash and number, caching it if found.
-func (self *LightChain) GetTd(hash common.Hash, number uint64) *big.Int {
+func (lc *LightChain) GetTd(hash common.Hash, number uint64) *big.Int {
-	return self.hc.GetTd(hash, number)
+	return lc.hc.GetTd(hash, number)
 }
 
 // GetTdByHash retrieves a block's total difficulty in the canonical chain from the
 // database by hash, caching it if found.
-func (self *LightChain) GetTdByHash(hash common.Hash) *big.Int {
+func (lc *LightChain) GetTdByHash(hash common.Hash) *big.Int {
-	return self.hc.GetTdByHash(hash)
+	return lc.hc.GetTdByHash(hash)
 }
 
 // GetHeader retrieves a block header from the database by hash and number,
 // caching it if found.
-func (self *LightChain) GetHeader(hash common.Hash, number uint64) *types.Header {
+func (lc *LightChain) GetHeader(hash common.Hash, number uint64) *types.Header {
-	return self.hc.GetHeader(hash, number)
+	return lc.hc.GetHeader(hash, number)
 }
 
 // GetHeaderByHash retrieves a block header from the database by hash, caching it if
 // found.
-func (self *LightChain) GetHeaderByHash(hash common.Hash) *types.Header {
+func (lc *LightChain) GetHeaderByHash(hash common.Hash) *types.Header {
-	return self.hc.GetHeaderByHash(hash)
+	return lc.hc.GetHeaderByHash(hash)
 }
 
 // HasHeader checks if a block header is present in the database or not, caching
 // it if present.
-func (bc *LightChain) HasHeader(hash common.Hash, number uint64) bool {
+func (lc *LightChain) HasHeader(hash common.Hash, number uint64) bool {
-	return bc.hc.HasHeader(hash, number)
+	return lc.hc.HasHeader(hash, number)
 }
 
 // GetBlockHashesFromHash retrieves a number of block hashes starting at a given
 // hash, fetching towards the genesis block.
-func (self *LightChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash {
+func (lc *LightChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash {
-	return self.hc.GetBlockHashesFromHash(hash, max)
+	return lc.hc.GetBlockHashesFromHash(hash, max)
 }
 
 // GetAncestor retrieves the Nth ancestor of a given block. It assumes that either the given block or
@@ -436,56 +436,56 @@ func (self *LightChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []c
 // number of blocks to be individually checked before we reach the canonical chain.
 //
 // Note: ancestor == 0 returns the same block, 1 returns its parent and so on.
-func (bc *LightChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) {
+func (lc *LightChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) {
-	bc.chainmu.RLock()
+	lc.chainmu.RLock()
-	defer bc.chainmu.RUnlock()
+	defer lc.chainmu.RUnlock()
 
-	return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical)
+	return lc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical)
 }
 
 // GetHeaderByNumber retrieves a block header from the database by number,
 // caching it (associated with its hash) if found.
-func (self *LightChain) GetHeaderByNumber(number uint64) *types.Header {
+func (lc *LightChain) GetHeaderByNumber(number uint64) *types.Header {
-	return self.hc.GetHeaderByNumber(number)
+	return lc.hc.GetHeaderByNumber(number)
 }
 
 // GetHeaderByNumberOdr retrieves a block header from the database or network
 // by number, caching it (associated with its hash) if found.
-func (self *LightChain) GetHeaderByNumberOdr(ctx context.Context, number uint64) (*types.Header, error) {
+func (lc *LightChain) GetHeaderByNumberOdr(ctx context.Context, number uint64) (*types.Header, error) {
-	if header := self.hc.GetHeaderByNumber(number); header != nil {
+	if header := lc.hc.GetHeaderByNumber(number); header != nil {
 		return header, nil
 	}
-	return GetHeaderByNumber(ctx, self.odr, number)
+	return GetHeaderByNumber(ctx, lc.odr, number)
 }
 
 // Config retrieves the header chain's chain configuration.
-func (self *LightChain) Config() *params.ChainConfig { return self.hc.Config() }
+func (lc *LightChain) Config() *params.ChainConfig { return lc.hc.Config() }
 
-func (self *LightChain) SyncCht(ctx context.Context) bool {
+func (lc *LightChain) SyncCht(ctx context.Context) bool {
 	// If we don't have a CHT indexer, abort
-	if self.odr.ChtIndexer() == nil {
+	if lc.odr.ChtIndexer() == nil {
 		return false
 	}
 	// Ensure the remote CHT head is ahead of us
-	head := self.CurrentHeader().Number.Uint64()
+	head := lc.CurrentHeader().Number.Uint64()
-	sections, _, _ := self.odr.ChtIndexer().Sections()
+	sections, _, _ := lc.odr.ChtIndexer().Sections()
 
-	latest := sections*self.indexerConfig.ChtSize - 1
+	latest := sections*lc.indexerConfig.ChtSize - 1
-	if clique := self.hc.Config().Clique; clique != nil {
+	if clique := lc.hc.Config().Clique; clique != nil {
 		latest -= latest % clique.Epoch // epoch snapshot for clique
 	}
 	if head >= latest {
 		return false
 	}
 	// Retrieve the latest useful header and update to it
-	if header, err := GetHeaderByNumber(ctx, self.odr, latest); header != nil && err == nil {
+	if header, err := GetHeaderByNumber(ctx, lc.odr, latest); header != nil && err == nil {
-		self.chainmu.Lock()
+		lc.chainmu.Lock()
-		defer self.chainmu.Unlock()
+		defer lc.chainmu.Unlock()
 
 		// Ensure the chain didn't move past the latest block while retrieving it
-		if self.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() {
+		if lc.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() {
 			log.Info("Updated latest header based on CHT", "number", header.Number, "hash", header.Hash(), "age", common.PrettyAge(time.Unix(header.Time.Int64(), 0)))
-			self.hc.SetCurrentHeader(header)
+			lc.hc.SetCurrentHeader(header)
 		}
 		return true
 	}
@@ -494,48 +494,48 @@ func (self *LightChain) SyncCht(ctx context.Context) bool {
 
 // LockChain locks the chain mutex for reading so that multiple canonical hashes can be
 // retrieved while it is guaranteed that they belong to the same version of the chain
-func (self *LightChain) LockChain() {
+func (lc *LightChain) LockChain() {
-	self.chainmu.RLock()
+	lc.chainmu.RLock()
 }
 
 // UnlockChain unlocks the chain mutex
-func (self *LightChain) UnlockChain() {
+func (lc *LightChain) UnlockChain() {
-	self.chainmu.RUnlock()
+	lc.chainmu.RUnlock()
 }
 
 // SubscribeChainEvent registers a subscription of ChainEvent.
-func (self *LightChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+func (lc *LightChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
-	return self.scope.Track(self.chainFeed.Subscribe(ch))
+	return lc.scope.Track(lc.chainFeed.Subscribe(ch))
 }
 
 // SubscribeChainHeadEvent registers a subscription of ChainHeadEvent.
-func (self *LightChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription {
+func (lc *LightChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription {
-	return self.scope.Track(self.chainHeadFeed.Subscribe(ch))
+	return lc.scope.Track(lc.chainHeadFeed.Subscribe(ch))
 }
 
 // SubscribeChainSideEvent registers a subscription of ChainSideEvent.
-func (self *LightChain) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription {
+func (lc *LightChain) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription {
-	return self.scope.Track(self.chainSideFeed.Subscribe(ch))
+	return lc.scope.Track(lc.chainSideFeed.Subscribe(ch))
 }
 
 // SubscribeLogsEvent implements the interface of filters.Backend
 // LightChain does not send logs events, so return an empty subscription.
-func (self *LightChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
+func (lc *LightChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
-	return self.scope.Track(new(event.Feed).Subscribe(ch))
+	return lc.scope.Track(new(event.Feed).Subscribe(ch))
 }
 
 // SubscribeRemovedLogsEvent implements the interface of filters.Backend
 // LightChain does not send core.RemovedLogsEvent, so return an empty subscription.
-func (self *LightChain) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
+func (lc *LightChain) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
-	return self.scope.Track(new(event.Feed).Subscribe(ch))
+	return lc.scope.Track(new(event.Feed).Subscribe(ch))
 }
 
 // DisableCheckFreq disables header validation. This is used for ultralight mode.
-func (self *LightChain) DisableCheckFreq() {
+func (lc *LightChain) DisableCheckFreq() {
-	atomic.StoreInt32(&self.disableCheckFreq, 1)
+	atomic.StoreInt32(&lc.disableCheckFreq, 1)
 }
 
 // EnableCheckFreq enables header validation.
-func (self *LightChain) EnableCheckFreq() {
+func (lc *LightChain) EnableCheckFreq() {
-	atomic.StoreInt32(&self.disableCheckFreq, 0)
+	atomic.StoreInt32(&lc.disableCheckFreq, 0)
 }

light/odr_util.go

@@ -28,7 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-var sha3_nil = crypto.Keccak256Hash(nil)
+var sha3Nil = crypto.Keccak256Hash(nil)
 
 func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) {
 	db := odr.Database()