Merge branch 'master' into abigen-vyper

.travis.yml

@@ -68,8 +68,11 @@ matrix:
             - debhelper
             - dput
             - fakeroot
+            - python-bzrlib
+            - python-paramiko
       script:
-        - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - if: type = push

accounts/abi/bind/backends/simulated.go

@@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
 	return receipt, nil
 }
 
+// TransactionByHash checks the pool of pending transactions in addition to the
+// blockchain. The isPending return value indicates whether the transaction has been
+// mined yet. Note that the transaction may not be part of the canonical chain even if
+// it's not pending.
+func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) {
+
+	tx = b.pendingBlock.Transaction(txHash)
+	if tx != nil {
+		return tx, true, nil
+	}
+
+	tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
+	if tx != nil {
+		return tx, false, nil
+	}
+
+	return nil, false, ethereum.NotFound
+}
+
 // PendingCodeAt returns the code associated with an account in the pending state.
 func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
 	b.mu.Lock()

accounts/abi/bind/backends/simulated_test.go

@@ -0,0 +1,66 @@
+package backends_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+func TestSimulatedBackend(t *testing.T) {
+	var gasLimit uint64 = 8000029
+	key, _ := crypto.GenerateKey() // nolint: gosec
+	auth := bind.NewKeyedTransactor(key)
+	genAlloc := make(core.GenesisAlloc)
+	genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
+
+	sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
+
+	// should return an error if the tx is not found
+	txHash := common.HexToHash("2")
+	_, isPending, err := sim.TransactionByHash(context.Background(), txHash)
+
+	if isPending {
+		t.Fatal("transaction should not be pending")
+	}
+	if err != ethereum.NotFound {
+		t.Fatalf("err should be `ethereum.NotFound` but received %v", err)
+	}
+
+	// generate a transaction and confirm you can retrieve it
+	code := `6060604052600a8060106000396000f360606040526008565b00`
+	var gas uint64 = 3000000
+	tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code))
+	tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key)
+
+	err = sim.SendTransaction(context.Background(), tx)
+	if err != nil {
+		t.Fatal("error sending transaction")
+	}
+
+	txHash = tx.Hash()
+	_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if !isPending {
+		t.Fatal("transaction should have pending status")
+	}
+
+	sim.Commit()
+	tx, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if isPending {
+		t.Fatal("transaction should not have pending status")
+	}
+
+}

accounts/external/backend.go

@@ -26,7 +26,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
@@ -154,13 +153,31 @@ func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]by
 
 // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
 func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
-	// TODO! Replace this with a call to clef SignData with correct mime-type for Clique, once we
+	var res hexutil.Bytes
-	// have that in place
+	var signAddress = common.NewMixedcaseAddress(account.Address)
-	return api.signHash(account, crypto.Keccak256(data))
+	if err := api.client.Call(&res, "account_signData",
+		mimeType,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(data)); err != nil {
+		return nil, err
+	}
+	// If V is on 27/28-form, convert to to 0/1 for Clique
+	if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) {
+		res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
+	}
+	return res, nil
 }
 
 func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
-	return api.signHash(account, accounts.TextHash(text))
+	var res hexutil.Bytes
+	var signAddress = common.NewMixedcaseAddress(account.Address)
+	if err := api.client.Call(&res, "account_signData",
+		accounts.MimetypeTextPlain,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(text)); err != nil {
+		return nil, err
+	}
+	return res, nil
 }
 
 func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
@@ -202,18 +219,6 @@ func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
 	return res, nil
 }
 
-func (api *ExternalSigner) signCliqueBlock(a common.Address, rlpBlock hexutil.Bytes) (hexutil.Bytes, error) {
-	var sig hexutil.Bytes
-	if err := api.client.Call(&sig, "account_signData", core.ApplicationClique.Mime, a, rlpBlock); err != nil {
-		return nil, err
-	}
-	if sig[64] != 27 && sig[64] != 28 {
-		return nil, fmt.Errorf("invalid Ethereum signature (V is not 27 or 28)")
-	}
-	sig[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
-	return sig, nil
-}
-
 func (api *ExternalSigner) pingVersion() (string, error) {
 	var v string
 	if err := api.client.Call(&v, "account_version"); err != nil {

build/ci-notes.md

@@ -7,11 +7,18 @@ Canonical.
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
 
+## Launchpad
+
 The packages are built and served by launchpad.net. We generate a Debian source package
 for each distribution and upload it. Their builder picks up the source package, builds it
 and installs the new version into the PPA repository. Launchpad requires a valid signature
-by a team member for source package uploads. The signing key is stored in an environment
+by a team member for source package uploads.
-variable which Travis CI makes available to certain builds.
+
+The signing key is stored in an environment variable which Travis CI makes available to
+certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
+packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
+and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
+variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
@@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 

build/ci.go

@@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
 func archiveUpload(archive string, blobstore string, signer string) error {
 	// If signing was requested, generate the signature files
 	if signer != "" {
-		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
+		key := getenvBase64(signer)
-		if err != nil {
+		if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
-			return fmt.Errorf("invalid base64 %s", signer)
-		}
-		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
 			return err
 		}
 	}
@@ -488,7 +485,8 @@ func maybeSkipArchive(env build.Environment) {
 func doDebianSource(cmdline []string) {
 	var (
 		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+		upload  = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+		sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
 		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
 		now     = time.Now()
 	)
@@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
 	maybeSkipArchive(env)
 
 	// Import the signing key.
-	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
+	if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatal("invalid base64 PPA_SIGNING_KEY")
-		}
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
@@ -513,22 +507,58 @@ func doDebianSource(cmdline []string) {
 		for _, distro := range debDistros {
 			meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
-			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d")
+			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
 			debuild.Dir = pkgdir
 			build.MustRun(debuild)
 
-			changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
+			var (
-			changes = filepath.Join(*workdir, changes)
+				basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString())
+				source   = filepath.Join(*workdir, basename+".tar.xz")
+				dsc      = filepath.Join(*workdir, basename+".dsc")
+				changes  = filepath.Join(*workdir, basename+"_source.changes")
+			)
 			if *signer != "" {
 				build.MustRunCommand("debsign", changes)
 			}
 			if *upload != "" {
-				build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes)
+				ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes})
 			}
 		}
 	}
 }
 
+func ppaUpload(workdir, ppa, sshUser string, files []string) {
+	p := strings.Split(ppa, "/")
+	if len(p) != 2 {
+		log.Fatal("-upload PPA name must contain single /")
+	}
+	if sshUser == "" {
+		sshUser = p[0]
+	}
+	incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1])
+	// Create the SSH identity file if it doesn't exist.
+	var idfile string
+	if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
+		idfile = filepath.Join(workdir, "sshkey")
+		if _, err := os.Stat(idfile); os.IsNotExist(err) {
+			ioutil.WriteFile(idfile, sshkey, 0600)
+		}
+	}
+	// Upload
+	dest := sshUser + "@ppa.launchpad.net"
+	if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getenvBase64(variable string) []byte {
+	dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
+	if err != nil {
+		log.Fatal("invalid base64 " + variable)
+	}
+	return []byte(dec)
+}
+
 func makeWorkdir(wdflag string) string {
 	var err error
 	if wdflag != "" {
@@ -800,15 +830,10 @@ func doAndroidArchive(cmdline []string) {
 	os.Rename(archive, meta.Package+".aar")
 	if *signer != "" && *deploy != "" {
 		// Import the signing key into the local GPG instance
-		b64key := os.Getenv(*signer)
+		key := getenvBase64(*signer)
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatalf("invalid base64 %s", *signer)
-		}
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
-
 		keyID, err := build.PGPKeyID(string(key))
 		if err != nil {
 			log.Fatal(err)

cmd/abigen/main.go

@@ -136,7 +136,7 @@ func main() {
         }
         abis = append(abis, string(abi))
 
-        bin := []byte{}
+		var bin []byte
 		if *binFlag != "" {
 			if bin, err = ioutil.ReadFile(*binFlag); err != nil {
 				fmt.Printf("Failed to read input bytecode: %v\n", err)

cmd/clef/README.md

@@ -16,7 +16,8 @@ Check out
 
 * the [tutorial](tutorial.md) for some concrete examples on how the signer works.
 * the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. 
-
+* the [data types](datatypes.md) for detailed information on the json types used in the communication between
+  clef and an external UI 
 
 ## Command line flags
 Clef accepts the following command line options:
@@ -24,25 +25,31 @@ Clef accepts the following command line options:
 COMMANDS:
    init    Initialize the signer, generate secret storage
    attest  Attest that a js-file is to be used
-   addpw   Store a credential for a keystore file
+   setpw   Store a credential for a keystore file
+   gendoc  Generate documentation about json-rpc format
    help    Shows a list of commands or help for one command
    
 GLOBAL OPTIONS:
    --loglevel value        log level to emit to the screen (default: 4)
    --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
-   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
+   --configdir value       Directory for Clef configuration (default: "$HOME/.clef")
-   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --chainid value         Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli) (default: 1)
    --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
    --nousb                 Disables monitoring for and managing USB hardware wallets
    --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcvhosts value       Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
+   --ipcdisable            Disable the IPC-RPC server
+   --ipcpath               Filename for IPC socket/pipe within the datadir (explicit paths escape it)
+   --rpc                   Enable the HTTP-RPC server
    --rpcport value         HTTP-RPC server listening port (default: 8550)
-   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --signersecret value    A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
    --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
    --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
    --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
    --rules value           Enable rule-engine (default: "rules.json")
-   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
-   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --stdio-ui-test         Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
+   --advanced              If enabled, issues warnings instead of rejections for suspicious requests. Default off
    --help, -h              show help
    --version, -v           print the version
    

cmd/clef/datatypes.md

@@ -0,0 +1,219 @@
+## UI Client interface
+
+These data types are defined in the channel between clef and the UI
+### SignDataRequest
+
+SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to presentthe user with the contents of the `message`
+
+Example:
+```json
+{
+  "content_type": "text/plain",
+  "address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+  "raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
+  "message": [
+    {
+      "name": "message",
+      "value": "\u0019Ethereum Signed Message:\n11hello world",
+      "type": "text/plain"
+    }
+  ],
+  "hash": "0xd9eba16ed0ecae432b71fe008c98cc872bb4cc214d3220a36f365326cf807d68",
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### SignDataResponse - approve
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "approved": true,
+  "Password": "apassword"
+}
+```
+### SignDataResponse - deny
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "approved": false,
+  "Password": ""
+}
+```
+### SignTxRequest
+
+SignTxRequest contains information about a pending request to sign a transaction. Aside from the transaction itself, there is also a `call_info`-struct. That struct contains messages of various types, that the user should be informed of.
+
+As in any request, it's important to consider that the `meta` info also contains untrusted data.
+
+The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, they must be identical, otherwise an error is generated. However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+    "to": null,
+    "gas": "0x3e8",
+    "gasPrice": "0x5",
+    "value": "0x6",
+    "nonce": "0x1",
+    "data": "0x01020304"
+  },
+  "call_info": [
+    {
+      "type": "Warning",
+      "message": "Something looks odd, show this message as a warning"
+    },
+    {
+      "type": "Info",
+      "message": "User should see this aswell"
+    }
+  ],
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### SignDataResponse - approve
+
+Response to SignDataRequest. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
+
+Example:
+```json
+{
+  "transaction": {
+    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
+    "to": null,
+    "gas": "0x3e8",
+    "gasPrice": "0x5",
+    "value": "0x6",
+    "nonce": "0x4",
+    "data": "0x04030201"
+  },
+  "approved": true,
+  "password": "apassword"
+}
+```
+### SignDataResponse - deny
+
+Response to SignDataRequest. When denying a request, there's no need to provide the transaction in return
+
+Example:
+```json
+{
+  "approved": false,
+  "Password": ""
+}
+```
+### OnApproved - SignTransactionResult
+
+SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`
+
+This occurs _after_ successful completion of the entire signing procedure, but right before the signed transaction is passed to the external caller. This method (and data) can be used by the UI to signal to the user that the transaction was signed, but it is primarily useful for ruleset implementations.
+
+A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
+
+**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. 
+
+The `OnApproved` method cannot be responded to, it's purely informative
+
+Example:
+```json
+{
+  "raw": "0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
+  "tx": {
+    "nonce": "0x64",
+    "gasPrice": "0x1",
+    "gas": "0x1",
+    "to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
+    "value": "0x1",
+    "input": "0x",
+    "v": "0x26",
+    "r": "0x716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293",
+    "s": "0x4e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
+    "hash": "0x662f6d772692dd692f1b5e8baa77a9ff95bbd909362df3fc3d301aafebde5441"
+  }
+}
+```
+### UserInputRequest
+
+Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)
+
+Example:
+```json
+{
+  "prompt": "The question to ask the user",
+  "title": "The title here",
+  "isPassword": true
+}
+```
+### UserInputResponse
+
+Response to SignDataRequest
+
+Example:
+```json
+{
+  "text": "The textual response from user"
+}
+```
+### ListRequest
+
+Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es. 
+
+Example:
+```json
+{
+  "accounts": [
+    {
+      "address": "0xdeadbeef000000000000000000000000deadbeef",
+      "url": "keystore:///path/to/keyfile/a"
+    },
+    {
+      "address": "0x1111111122222222222233333333334444444444",
+      "url": "keystore:///path/to/keyfile/b"
+    }
+  ],
+  "meta": {
+    "remote": "localhost:9999",
+    "local": "localhost:8545",
+    "scheme": "http",
+    "User-Agent": "Firefox 3.2",
+    "Origin": "www.malicious.ru"
+  }
+}
+```
+### UserInputResponse
+
+Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
+
+Example:
+```json
+{
+  "accounts": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "url": ".. ignored .."
+    },
+    {
+      "address": "0xffffffffffffffffffffffffffffffffffffffff",
+      "url": ""
+    }
+  ]
+}
+```

cmd/clef/extapi_changelog.md

@@ -1,5 +1,10 @@
 ### Changelog for external API
 
+### 6.0.0
+
+* `New` was changed to deliver only an address, not the full `Account` data
+* `Export` was moved from External API to the UI Server API
+
 #### 5.0.0
 
 * The external `account_EcRecover`-method was reimplemented.

cmd/clef/intapi_changelog.md

@@ -1,8 +1,39 @@
 ### Changelog for internal API (ui-api)
 
+### 4.0.0
+
+* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
+  - `clef_listWallets` 
+  - `clef_listAccounts`
+  - `clef_listWallets`
+  - `clef_deriveAccount`
+  - `clef_importRawKey`
+  - `clef_openWallet`
+  - `clef_chainId`
+  - `clef_setChainId`
+  - `clef_export`
+  - `clef_import`
+ 
+* The type `Account` was modified (the json-field `type` was removed), to consist of 
+
+```golang
+type Account struct {
+	Address common.Address `json:"address"` // Ethereum account address derived from the key
+	URL     URL            `json:"url"`     // Optional resource locator within a backend
+}
+```
+
+
+### 3.2.0
+
+* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
+
+> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
+> 
+>  Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
 ### 3.1.0
 
-* Add `ContentType string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
+* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
 
 ### 3.0.0
 

cmd/clef/main.go

@@ -36,14 +36,18 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/signer/core"
@@ -83,6 +87,11 @@ var (
 		Value: DefaultConfigDir(),
 		Usage: "Directory for Clef configuration",
 	}
+	chainIdFlag = cli.Int64Flag{
+		Name:  "chainid",
+		Value: params.MainnetChainConfig.ChainID.Int64(),
+		Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
+	}
 	rpcPortFlag = cli.IntFlag{
 		Name:  "rpcport",
 		Usage: "HTTP-RPC server listening port",
@@ -167,8 +176,14 @@ Clef that the file is 'safe' to execute.`,
 		Description: `
 The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
-`,
+`}
-	}
+	gendocCommand = cli.Command{
+		Action: GenDoc,
+		Name:   "gendoc",
+		Usage:  "Generate documentation about json-rpc format",
+		Description: `
+The gendoc generates example structures of the json-rpc communication types.
+`}
 )
 
 func init() {
@@ -178,7 +193,7 @@ func init() {
 		logLevelFlag,
 		keystoreFlag,
 		configdirFlag,
-		utils.NetworkIdFlag,
+		chainIdFlag,
 		utils.LightKDFFlag,
 		utils.NoUSBFlag,
 		utils.RPCListenAddrFlag,
@@ -197,7 +212,7 @@ func init() {
 		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, gendocCommand}
 
 }
 func main() {
@@ -338,7 +353,7 @@ func signer(c *cli.Context) error {
 		return err
 	}
 	var (
-		ui core.SignerUI
+		ui core.UIClientAPI
 	)
 	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
@@ -402,14 +417,21 @@ func signer(c *cli.Context) error {
 			}
 		}
 	}
+	var (
+		chainId  = c.GlobalInt64(chainIdFlag.Name)
+		ksLoc    = c.GlobalString(keystoreFlag.Name)
+		lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
+		advanced = c.GlobalBool(advancedMode.Name)
+		nousb    = c.GlobalBool(utils.NoUSBFlag.Name)
+	)
+	log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
+		"light-kdf", lightKdf, "advanced", advanced)
+	am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
+	apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced)
 
-	apiImpl := core.NewSignerAPI(
+	// Establish the bidirectional communication, by creating a new UI backend and registering
-		c.GlobalInt64(utils.NetworkIdFlag.Name),
+	// it with the UI.
-		c.GlobalString(keystoreFlag.Name),
+	ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
-		c.GlobalBool(utils.NoUSBFlag.Name),
-		ui, db,
-		c.GlobalBool(utils.LightKDFFlag.Name),
-		c.GlobalBool(advancedMode.Name))
 	api = apiImpl
 	// Audit logging
 	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
@@ -529,7 +551,7 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	var (
 		file      string
 		configDir = ctx.GlobalString(configdirFlag.Name)
@@ -664,10 +686,6 @@ func testExternalUI(api *core.SignerAPI) {
 	checkErr("List", err)
 	_, err = api.New(ctx)
 	checkErr("New", err)
-	_, err = api.Export(ctx, common.Address{})
-	checkErr("Export", err)
-	_, err = api.Import(ctx, json.RawMessage{})
-	checkErr("Import", err)
 
 	api.UI.ShowInfo("Tests completed")
 
@@ -734,6 +752,154 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
 	return seed, err
 }
 
+// GenDoc outputs examples of all structures used in json-rpc communication
+func GenDoc(ctx *cli.Context) {
+
+	var (
+		a    = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
+		b    = common.HexToAddress("0x1111111122222222222233333333334444444444")
+		meta = core.Metadata{
+			Scheme:    "http",
+			Local:     "localhost:8545",
+			Origin:    "www.malicious.ru",
+			Remote:    "localhost:9999",
+			UserAgent: "Firefox 3.2",
+		}
+		output []string
+		add    = func(name, desc string, v interface{}) {
+			if data, err := json.MarshalIndent(v, "", "  "); err == nil {
+				output = append(output, fmt.Sprintf("### %s\n\n%s\n\nExample:\n```json\n%s\n```", name, desc, data))
+			} else {
+				log.Error("Error generating output", err)
+			}
+		}
+	)
+
+	{ // Sign plain text request
+		desc := "SignDataRequest contains information about a pending request to sign some data. " +
+			"The data to be signed can be of various types, defined by content-type. Clef has done most " +
+			"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
+			"the user with the contents of the `message`"
+		sighash, msg := accounts.TextAndHash([]byte("hello world"))
+		message := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
+
+		add("SignDataRequest", desc, &core.SignDataRequest{
+			Address:     common.NewMixedcaseAddress(a),
+			Meta:        meta,
+			ContentType: accounts.MimetypeTextPlain,
+			Rawdata:     []byte(msg),
+			Message:     message,
+			Hash:        sighash})
+	}
+	{ // Sign plain text response
+		add("SignDataResponse - approve", "Response to SignDataRequest",
+			&core.SignDataResponse{Password: "apassword", Approved: true})
+		add("SignDataResponse - deny", "Response to SignDataRequest",
+			&core.SignDataResponse{})
+	}
+	{ // Sign transaction request
+		desc := "SignTxRequest contains information about a pending request to sign a transaction. " +
+			"Aside from the transaction itself, there is also a `call_info`-struct. That struct contains " +
+			"messages of various types, that the user should be informed of." +
+			"\n\n" +
+			"As in any request, it's important to consider that the `meta` info also contains untrusted data." +
+			"\n\n" +
+			"The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, " +
+			"they must be identical, otherwise an error is generated. " +
+			"However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)"
+
+		data := hexutil.Bytes([]byte{0x01, 0x02, 0x03, 0x04})
+		add("SignTxRequest", desc, &core.SignTxRequest{
+			Meta: meta,
+			Callinfo: []core.ValidationInfo{
+				{"Warning", "Something looks odd, show this message as a warning"},
+				{"Info", "User should see this aswell"},
+			},
+			Transaction: core.SendTxArgs{
+				Data:     &data,
+				Nonce:    0x1,
+				Value:    hexutil.Big(*big.NewInt(6)),
+				From:     common.NewMixedcaseAddress(a),
+				To:       nil,
+				GasPrice: hexutil.Big(*big.NewInt(5)),
+				Gas:      1000,
+				Input:    nil,
+			}})
+	}
+	{ // Sign tx response
+		data := hexutil.Bytes([]byte{0x04, 0x03, 0x02, 0x01})
+		add("SignDataResponse - approve", "Response to SignDataRequest. This response needs to contain the `transaction`"+
+			", because the UI is free to make modifications to the transaction.",
+			&core.SignTxResponse{Password: "apassword", Approved: true,
+				Transaction: core.SendTxArgs{
+					Data:     &data,
+					Nonce:    0x4,
+					Value:    hexutil.Big(*big.NewInt(6)),
+					From:     common.NewMixedcaseAddress(a),
+					To:       nil,
+					GasPrice: hexutil.Big(*big.NewInt(5)),
+					Gas:      1000,
+					Input:    nil,
+				}})
+		add("SignDataResponse - deny", "Response to SignDataRequest. When denying a request, there's no need to "+
+			"provide the transaction in return",
+			&core.SignDataResponse{})
+	}
+	{ // WHen a signed tx is ready to go out
+		desc := "SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`" +
+			"\n\n" +
+			"This occurs _after_ successful completion of the entire signing procedure, but right before the signed " +
+			"transaction is passed to the external caller. This method (and data) can be used by the UI to signal " +
+			"to the user that the transaction was signed, but it is primarily useful for ruleset implementations." +
+			"\n\n" +
+			"A ruleset that implements a rate limitation needs to know what transactions are sent out to the external " +
+			"interface. By hooking into this methods, the ruleset can maintain track of that count." +
+			"\n\n" +
+			"**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time" +
+			" (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. " +
+			"\n\n" +
+			"The `OnApproved` method cannot be responded to, it's purely informative"
+
+		rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed")
+		var tx types.Transaction
+		rlp.DecodeBytes(rlpdata, &tx)
+		add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
+
+	}
+	{ // User input
+		add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
+			&core.UserInputRequest{IsPassword: true, Title: "The title here", Prompt: "The question to ask the user"})
+		add("UserInputResponse", "Response to SignDataRequest",
+			&core.UserInputResponse{Text: "The textual response from user"})
+	}
+	{ // List request
+		add("ListRequest", "Sent when a request has been made to list addresses. The UI is provided with the "+
+			"full `account`s, including local directory names. Note: this information is not passed back to the external caller, "+
+			"who only sees the `address`es. ",
+			&core.ListRequest{
+				Meta: meta,
+				Accounts: []accounts.Account{
+					{a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
+					{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
+			})
+
+		add("UserInputResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
+			"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
+			&core.ListResponse{
+				Accounts: []accounts.Account{
+					{common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}},
+					{common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}},
+				}})
+	}
+
+	fmt.Println(`## UI Client interface
+
+These data types are defined in the channel between clef and the UI`)
+	for _, elem := range output {
+		fmt.Println(elem)
+	}
+}
+
 /**
 //Create Account
 

cmd/clef/tests/testsigner.js

@@ -0,0 +1,73 @@
+// This file is a test-utility for testing clef-functionality
+//
+// Start clef with
+//
+// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
+//
+// Start geth with
+//
+// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
+//
+// and in the console simply invoke
+//
+// > test()
+//
+// You can reload the file via `reload()`
+
+function reload(){
+	loadScript("./cmd/clef/tests/testsigner.js");
+}
+
+function init(){
+    if (typeof accts == 'undefined' || accts.length == 0){
+        accts = eth.accounts
+        console.log("Got accounts ", accts);
+    }
+}
+init()
+function testTx(){
+    if( accts && accts.length > 0) {
+        var a = accts[0]
+        var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
+        var v = parseInt(txdata.tx.v)
+        console.log("V value: ", v)
+        if (v == 37 || v == 38){
+            console.log("Mainnet 155-protected chainid was used")
+        }
+        if (v == 27 || v == 28){
+            throw new Error("Mainnet chainid was used, but without replay protection!")
+        }
+    }
+}
+function testSignText(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
+        console.log("signing response",  r)
+    }
+}
+function testClique(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
+        console.log("signing response",  r)
+        if( a != r){
+            throw new Error("Requested signing by "+a+ " but got sealer "+r)
+        }
+    }
+}
+
+function test(){
+    var tests = [
+        testTx,
+        testSignText,
+        testClique,
+    ]
+    for( i in tests){
+        try{
+            tests[i]()
+        }catch(err){
+            console.log(err)
+        }
+    }
+ }

cmd/geth/monitorcmd.go

@@ -169,7 +169,7 @@ func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) {
 // resolveMetrics takes a list of input metric patterns, and resolves each to one
 // or more canonical metric names.
 func resolveMetrics(metrics map[string]interface{}, patterns []string) []string {
-	res := []string{}
+	var res []string
 	for _, pattern := range patterns {
 		res = append(res, resolveMetric(metrics, pattern, "")...)
 	}
@@ -179,7 +179,7 @@ func resolveMetrics(metrics map[string]interface{}, patterns []string) []string
 // resolveMetrics takes a single of input metric pattern, and resolves it to one
 // or more canonical metric names.
 func resolveMetric(metrics map[string]interface{}, pattern string, path string) []string {
-	results := []string{}
+	var results []string
 
 	// If a nested metric was requested, recurse optionally branching (via comma)
 	parts := strings.SplitN(pattern, "/", 2)
@@ -215,7 +215,7 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
 // expandMetrics expands the entire tree of metrics into a flat list of paths.
 func expandMetrics(metrics map[string]interface{}, path string) []string {
 	// Iterate over all fields and expand individually
-	list := []string{}
+	var list []string
 	for name, metric := range metrics {
 		switch metric := metric.(type) {
 		case float64:

cmd/geth/usage.go

@@ -308,7 +308,7 @@ func init() {
 					categorized[flag.String()] = struct{}{}
 				}
 			}
-			uncategorized := []cli.Flag{}
+			var uncategorized []cli.Flag
 			for _, flag := range data.(*cli.App).Flags {
 				if _, ok := categorized[flag.String()]; !ok {
 					if strings.HasPrefix(flag.GetName(), "dashboard") {

cmd/swarm/access.go

@@ -178,8 +178,8 @@ func accessNewACT(ctx *cli.Context) {
 		accessKey            []byte
 		err                  error
 		ref                  = args[0]
-		pkGrantees           = []string{}
+		pkGrantees           []string
-		passGrantees         = []string{}
+		passGrantees         []string
 		pkGranteesFilename   = ctx.String(SwarmAccessGrantKeysFlag.Name)
 		passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
 		privateKey           = getPrivKey(ctx)

cmd/swarm/access_test.go

@@ -397,7 +397,7 @@ func testACT(t *testing.T, bogusEntries int) {
 	}
 
 	ref := matches[0]
-	grantees := []string{}
+	var grantees []string
 	for i, v := range cluster.Nodes {
 		if i == nodeToSkip {
 			continue
@@ -408,7 +408,7 @@ func testACT(t *testing.T, bogusEntries int) {
 	}
 
 	if bogusEntries > 0 {
-		bogusGrantees := []string{}
+		var bogusGrantees []string
 
 		for i := 0; i < bogusEntries; i++ {
 			prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)

cmd/swarm/fs.go

@@ -123,7 +123,7 @@ func listMounts(cliContext *cli.Context) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	mf := []fuse.MountInfo{}
+	var mf []fuse.MountInfo
 	err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
 	if err != nil {
 		utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)

cmd/swarm/swarm-smoke/main.go

@@ -46,6 +46,7 @@ var (
 	verbosity    int
 	timeout      int
 	single       bool
+	trackTimeout int
 )
 
 func main() {
@@ -102,6 +103,12 @@ func main() {
 			Usage:       "whether to fetch content from a single node or from all nodes",
 			Destination: &single,
 		},
+		cli.IntFlag{
+			Name:        "track-timeout",
+			Value:       5,
+			Usage:       "timeout in seconds to wait for GetAllReferences to return",
+			Destination: &trackTimeout,
+		},
 	}
 
 	app.Flags = append(app.Flags, []cli.Flag{

cmd/swarm/swarm-smoke/sliding_window.go

@@ -56,7 +56,7 @@ func slidingWindowCmd(ctx *cli.Context, tuid string) error {
 }
 
 func slidingWindow(ctx *cli.Context, tuid string) error {
-	hashes := []uploadResult{} //swarm hashes of the uploads
+	var hashes []uploadResult //swarm hashes of the uploads
 	nodes := len(hosts)
 	const iterationTimeout = 30 * time.Second
 	log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -18,13 +18,19 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 	"github.com/pborman/uuid"
 
@@ -49,10 +55,73 @@ func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
 	case <-time.After(time.Duration(timeout) * time.Second):
 		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
 
+		e := fmt.Errorf("timeout after %v sec", timeout)
 		// trigger debug functionality on randomBytes
-
+		err := trackChunks(randomBytes[:])
-		return fmt.Errorf("timeout after %v sec", timeout)
+		if err != nil {
+			e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err)
 		}
+
+		return e
+	}
+}
+
+func trackChunks(testData []byte) error {
+	log.Warn("Test timed out; running chunk debug sequence")
+
+	addrs, err := getAllRefs(testData)
+	if err != nil {
+		return err
+	}
+	log.Trace("All references retrieved")
+
+	// has-chunks
+	for _, host := range hosts {
+		httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
+		log.Trace("Calling `Has` on host", "httpHost", httpHost)
+		rpcClient, err := rpc.Dial(httpHost)
+		if err != nil {
+			log.Trace("Error dialing host", "err", err)
+			return err
+		}
+		log.Trace("rpc dial ok")
+		var hasInfo []api.HasInfo
+		err = rpcClient.Call(&hasInfo, "bzz_has", addrs)
+		if err != nil {
+			log.Trace("Error calling host", "err", err)
+			return err
+		}
+		log.Trace("rpc call ok")
+		count := 0
+		for _, info := range hasInfo {
+			if !info.Has {
+				count++
+				log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr)
+			}
+		}
+		if count == 0 {
+			log.Info("Host reported to have all chunks", "host", httpHost)
+		}
+	}
+	return nil
+}
+
+func getAllRefs(testData []byte) (storage.AddressCollection, error) {
+	log.Trace("Getting all references for given root hash")
+	datadir, err := ioutil.TempDir("", "chunk-debug")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(datadir)
+	fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second)
+	defer cancel()
+
+	reader := bytes.NewReader(testData)
+	return fileStore.GetAllReferences(ctx, reader, false)
 }
 
 func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error {

cmd/utils/flags.go

@@ -950,10 +950,11 @@ func makeDatabaseHandles() int {
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if err := fdlimit.Raise(uint64(limit)); err != nil {
+	raised, err := fdlimit.Raise(uint64(limit))
+	if err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
-	return limit / 2 // Leave half for networking and other stuff
+	return int(raised / 2) // Leave half for networking and other stuff
 }
 
 // MakeAddress converts an account specified directly as a hex encoded string or
@@ -1612,7 +1613,7 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
 		return nil
 	}
 	// Otherwise resolve absolute paths and return them
-	preloads := []string{}
+	var preloads []string
 
 	assets := ctx.GlobalString(JSpathFlag.Name)
 	for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") {

common/fdlimit/fdlimit_freebsd.go

@@ -26,11 +26,11 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
@@ -38,9 +38,12 @@ func Raise(max uint64) error {
 		limit.Cur = int64(max)
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
-	return nil
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_test.go

@@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
 	if limit, err := Current(); err != nil || limit <= 0 {
 		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
 	}
-	if err := Raise(uint64(target)); err != nil {
+	if _, err := Raise(uint64(target)); err != nil {
 		t.Fatalf("failed to raise file allowance")
 	}
 	if limit, err := Current(); err != nil || limit < target {

common/fdlimit/fdlimit_unix.go

@@ -22,11 +22,12 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
@@ -34,9 +35,13 @@ func Raise(max uint64) error {
 		limit.Cur = max
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
-	return nil
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_windows.go

@@ -16,28 +16,30 @@
 
 package fdlimit
 
-import "errors"
+import "fmt"
+
+const hardlimit = 16384
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// This method is NOP by design:
 	//  * Linux/Darwin counterparts need to manually increase per process limits
 	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
 	//    changeable from within a running process
 	// This way we can always "request" raising the limits, which will either have
 	// or not have effect based on the platform we're running on.
-	if max > 16384 {
+	if max > hardlimit {
-		return errors.New("file descriptor limit (16384) reached")
+		return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
 	}
-	return nil
+	return max, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this
 // process.
 func Current() (int, error) {
 	// Please see Raise for the reason why we use hard coded 16K as the limit
-	return 16384, nil
+	return hardlimit, nil
 }
 
 // Maximum retrieves the maximum number of file descriptors this process is

console/console.go

@@ -120,10 +120,10 @@ func (c *Console) init(preload []string) error {
 	consoleObj.Object().Set("error", c.consoleOutput)
 
 	// Load all the internal utility JavaScript libraries
-	if err := c.jsre.Compile("bignumber.js", jsre.BigNumber_JS); err != nil {
+	if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil {
 		return fmt.Errorf("bignumber.js: %v", err)
 	}
-	if err := c.jsre.Compile("web3.js", jsre.Web3_JS); err != nil {
+	if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil {
 		return fmt.Errorf("web3.js: %v", err)
 	}
 	if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil {
@@ -236,7 +236,7 @@ func (c *Console) clearHistory() {
 // consoleOutput is an override for the console.log and console.error methods to
 // stream the output into the configured output stream instead of stdout.
 func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value {
-	output := []string{}
+	var output []string
 	for _, argument := range call.ArgumentList {
 		output = append(output, fmt.Sprintf("%v", argument))
 	}

core/blockchain_test.go

@@ -17,7 +17,6 @@
 package core
 
 import (
-	"fmt"
 	"math/big"
 	"math/rand"
 	"sync"
@@ -126,13 +125,6 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara
 	comparator(tdPre, tdPost)
 }
 
-func printChain(bc *BlockChain) {
-	for i := bc.CurrentBlock().Number().Uint64(); i > 0; i-- {
-		b := bc.GetBlockByNumber(uint64(i))
-		fmt.Printf("\t%x %v\n", b.Hash(), b.Difficulty())
-	}
-}
-
 // testBlockChainImport tries to process a chain of blocks, writing them into
 // the database if successful.
 func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
@@ -188,15 +180,6 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
 	return nil
 }
 
-func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *testing.T) {
-	_, err := blockchain.InsertChain(chain)
-	if err != nil {
-		fmt.Println(err)
-		t.FailNow()
-	}
-	done <- true
-}
-
 func TestLastBlock(t *testing.T) {
 	_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
 	if err != nil {

core/tx_pool_test.go

@@ -1274,14 +1274,14 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
 	}
 	// Create transaction (both pending and queued) with a linearly growing gasprice
 	for i := uint64(0); i < 500; i++ {
-		// Add pending
+		// Add pending transaction.
-		p_tx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
+		pendingTx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
-		if err := pool.AddLocal(p_tx); err != nil {
+		if err := pool.AddLocal(pendingTx); err != nil {
 			t.Fatal(err)
 		}
-		// Add queued
+		// Add queued transaction.
-		q_tx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
+		queuedTx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
-		if err := pool.AddLocal(q_tx); err != nil {
+		if err := pool.AddLocal(queuedTx); err != nil {
 			t.Fatal(err)
 		}
 	}

core/vm/evm.go

@@ -267,9 +267,8 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 		snapshot = evm.StateDB.Snapshot()
 		to       = AccountRef(caller.Address())
 	)
-	// initialise a new contract and set the code that is to be used by the
+	// Initialise a new contract and set the code that is to be used by the EVM.
-	// EVM. The contract is a scoped environment for this execution context
+	// The contract is a scoped environment for this execution context only.
-	// only.
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
@@ -333,9 +332,8 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 		to       = AccountRef(addr)
 		snapshot = evm.StateDB.Snapshot()
 	)
-	// Initialise a new contract and set the code that is to be used by the
+	// Initialise a new contract and set the code that is to be used by the EVM.
-	// EVM. The contract is a scoped environment for this execution context
+	// The contract is a scoped environment for this execution context only.
-	// only.
 	contract := NewContract(caller, to, new(big.Int), gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
@@ -396,9 +394,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
 	}
 	evm.Transfer(evm.StateDB, caller.Address(), address, value)
 
-	// initialise a new contract and set the code that is to be used by the
+	// Initialise a new contract and set the code that is to be used by the EVM.
-	// EVM. The contract is a scoped environment for this execution context
+	// The contract is a scoped environment for this execution context only.
-	// only.
 	contract := NewContract(caller, AccountRef(address), value, gas)
 	contract.SetCodeOptionalHash(&address, codeAndHash)
 

crypto/ecies/ecies_test.go

@@ -93,19 +93,6 @@ func cmpPublic(pub1, pub2 PublicKey) bool {
 	return bytes.Equal(pub1Out, pub2Out)
 }
 
-// cmpPrivate returns true if the two private keys are the same.
-func cmpPrivate(prv1, prv2 *PrivateKey) bool {
-	if prv1 == nil || prv1.D == nil {
-		return false
-	} else if prv2 == nil || prv2.D == nil {
-		return false
-	} else if prv1.D.Cmp(prv2.D) != 0 {
-		return false
-	} else {
-		return cmpPublic(prv1.PublicKey, prv2.PublicKey)
-	}
-}
-
 // Validate the ECDH component.
 func TestSharedKey(t *testing.T) {
 	prv1, err := GenerateKey(rand.Reader, DefaultCurve, nil)

eth/downloader/downloader.go

@@ -719,7 +719,7 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header)
 			}
 			// Make sure the peer's reply conforms to the request
 			for i, header := range headers {
-				expectNumber := from + int64(i)*int64((skip+1))
+				expectNumber := from + int64(i)*int64(skip+1)
 				if number := header.Number.Int64(); number != expectNumber {
 					p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
 					return 0, errInvalidChain

eth/tracers/tracers_test.go

@@ -121,7 +121,7 @@ type callTracerTest struct {
 }
 
 func TestPrestateTracerCreate2(t *testing.T) {
-	unsigned_tx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"),
+	unsignedTx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"),
 		new(big.Int), 5000000, big.NewInt(1), []byte{})
 
 	privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader)
@@ -129,7 +129,7 @@ func TestPrestateTracerCreate2(t *testing.T) {
 		t.Fatalf("err %v", err)
 	}
 	signer := types.NewEIP155Signer(big.NewInt(1))
-	tx, err := types.SignTx(unsigned_tx, signer, privateKeyECDSA)
+	tx, err := types.SignTx(unsignedTx, signer, privateKeyECDSA)
 	if err != nil {
 		t.Fatalf("err %v", err)
 	}

internal/build/util.go

@@ -177,3 +177,34 @@ func ExpandPackagesNoVendor(patterns []string) []string {
 	}
 	return patterns
 }
+
+// UploadSFTP uploads files to a remote host using the sftp command line tool.
+// The destination host may be specified either as [user@]host[: or as a URI in
+// the form sftp://[user@]host[:port].
+func UploadSFTP(identityFile, host, dir string, files []string) error {
+	sftp := exec.Command("sftp")
+	sftp.Stdout = nil
+	sftp.Stderr = os.Stderr
+	if identityFile != "" {
+		sftp.Args = append(sftp.Args, "-i", identityFile)
+	}
+	sftp.Args = append(sftp.Args, host)
+	fmt.Println(">>>", strings.Join(sftp.Args, " "))
+	if *DryRunFlag {
+		return nil
+	}
+
+	stdin, err := sftp.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("can't create stdin pipe for sftp: %v", err)
+	}
+	if err := sftp.Start(); err != nil {
+		return err
+	}
+	in := io.MultiWriter(stdin, os.Stdout)
+	for _, f := range files {
+		fmt.Fprintln(in, "put", f, path.Join(dir, filepath.Base(f)))
+	}
+	stdin.Close()
+	return sftp.Wait()
+}

internal/ethapi/api.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -1469,6 +1470,45 @@ func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (stri
 	return fmt.Sprintf("%x", encoded), nil
 }
 
+// TestSignCliqueBlock fetches the given block number, and attempts to sign it as a clique header with the
+// given address, returning the address of the recovered signature
+//
+// This is a temporary method to debug the externalsigner integration,
+// TODO: Remove this method when the integration is mature
+func (api *PublicDebugAPI) TestSignCliqueBlock(ctx context.Context, address common.Address, number uint64) (common.Address, error) {
+	block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))
+	if block == nil {
+		return common.Address{}, fmt.Errorf("block #%d not found", number)
+	}
+	header := block.Header()
+	header.Extra = make([]byte, 32+65)
+	encoded := clique.CliqueRLP(header)
+
+	// Look up the wallet containing the requested signer
+	account := accounts.Account{Address: address}
+	wallet, err := api.b.AccountManager().Find(account)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	signature, err := wallet.SignData(account, accounts.MimetypeClique, encoded)
+	if err != nil {
+		return common.Address{}, err
+	}
+	sealHash := clique.SealHash(header).Bytes()
+	log.Info("test signing of clique block",
+		"Sealhash", fmt.Sprintf("%x", sealHash),
+		"signature", fmt.Sprintf("%x", signature))
+	pubkey, err := crypto.Ecrecover(sealHash, signature)
+	if err != nil {
+		return common.Address{}, err
+	}
+	var signer common.Address
+	copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
+
+	return signer, nil
+}
+
 // PrintBlock retrieves a block and returns its pretty printed form.
 func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) {
 	block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))

internal/jsre/jsre.go

@@ -32,8 +32,8 @@ import (
 )
 
 var (
-	BigNumber_JS = deps.MustAsset("bignumber.js")
+	BignumberJs = deps.MustAsset("bignumber.js")
-	Web3_JS      = deps.MustAsset("web3.js")
+	Web3Js      = deps.MustAsset("web3.js")
 )
 
 /*

internal/web3ext/web3ext.go

@@ -18,23 +18,23 @@
 package web3ext
 
 var Modules = map[string]string{
-	"accounting": Accounting_JS,
+	"accounting": AccountingJs,
-	"admin":      Admin_JS,
+	"admin":      AdminJs,
-	"chequebook": Chequebook_JS,
+	"chequebook": ChequebookJs,
-	"clique":     Clique_JS,
+	"clique":     CliqueJs,
-	"ethash":     Ethash_JS,
+	"ethash":     EthashJs,
-	"debug":      Debug_JS,
+	"debug":      DebugJs,
-	"eth":        Eth_JS,
+	"eth":        EthJs,
-	"miner":      Miner_JS,
+	"miner":      MinerJs,
-	"net":        Net_JS,
+	"net":        NetJs,
-	"personal":   Personal_JS,
+	"personal":   PersonalJs,
-	"rpc":        RPC_JS,
+	"rpc":        RpcJs,
-	"shh":        Shh_JS,
+	"shh":        ShhJs,
-	"swarmfs":    SWARMFS_JS,
+	"swarmfs":    SwarmfsJs,
-	"txpool":     TxPool_JS,
+	"txpool":     TxpoolJs,
 }
 
-const Chequebook_JS = `
+const ChequebookJs = `
 web3._extend({
 	property: 'chequebook',
 	methods: [
@@ -65,7 +65,7 @@ web3._extend({
 });
 `
 
-const Clique_JS = `
+const CliqueJs = `
 web3._extend({
 	property: 'clique',
 	methods: [
@@ -111,7 +111,7 @@ web3._extend({
 });
 `
 
-const Ethash_JS = `
+const EthashJs = `
 web3._extend({
 	property: 'ethash',
 	methods: [
@@ -139,7 +139,7 @@ web3._extend({
 });
 `
 
-const Admin_JS = `
+const AdminJs = `
 web3._extend({
 	property: 'admin',
 	methods: [
@@ -217,7 +217,7 @@ web3._extend({
 });
 `
 
-const Debug_JS = `
+const DebugJs = `
 web3._extend({
 	property: 'debug',
 	methods: [
@@ -231,6 +231,12 @@ web3._extend({
 			call: 'debug_getBlockRlp',
 			params: 1
 		}),
+		new web3._extend.Method({
+			name: 'testSignCliqueBlock',
+			call: 'debug_testSignCliqueBlock',
+			params: 2, 
+			inputFormatters: [web3._extend.formatters.inputAddressFormatter, null],
+		}),
 		new web3._extend.Method({
 			name: 'setHead',
 			call: 'debug_setHead',
@@ -448,7 +454,7 @@ web3._extend({
 });
 `
 
-const Eth_JS = `
+const EthJs = `
 web3._extend({
 	property: 'eth',
 	methods: [
@@ -518,7 +524,7 @@ web3._extend({
 });
 `
 
-const Miner_JS = `
+const MinerJs = `
 web3._extend({
 	property: 'miner',
 	methods: [
@@ -563,7 +569,7 @@ web3._extend({
 });
 `
 
-const Net_JS = `
+const NetJs = `
 web3._extend({
 	property: 'net',
 	methods: [],
@@ -576,7 +582,7 @@ web3._extend({
 });
 `
 
-const Personal_JS = `
+const PersonalJs = `
 web3._extend({
 	property: 'personal',
 	methods: [
@@ -622,7 +628,7 @@ web3._extend({
 })
 `
 
-const RPC_JS = `
+const RpcJs = `
 web3._extend({
 	property: 'rpc',
 	methods: [],
@@ -635,7 +641,7 @@ web3._extend({
 });
 `
 
-const Shh_JS = `
+const ShhJs = `
 web3._extend({
 	property: 'shh',
 	methods: [
@@ -655,7 +661,7 @@ web3._extend({
 });
 `
 
-const SWARMFS_JS = `
+const SwarmfsJs = `
 web3._extend({
 	property: 'swarmfs',
 	methods:
@@ -679,7 +685,7 @@ web3._extend({
 });
 `
 
-const TxPool_JS = `
+const TxpoolJs = `
 web3._extend({
 	property: 'txpool',
 	methods: [],
@@ -706,7 +712,7 @@ web3._extend({
 });
 `
 
-const Accounting_JS = `
+const AccountingJs = `
 web3._extend({
 	property: 'accounting',
 	methods: [

light/odr_util.go

@@ -28,7 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-var sha3_nil = crypto.Keccak256Hash(nil)
+var sha3Nil = crypto.Keccak256Hash(nil)
 
 func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) {
 	db := odr.Database()

light/trie.go

@@ -67,7 +67,7 @@ func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie {
 }
 
 func (db *odrDatabase) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
-	if codeHash == sha3_nil {
+	if codeHash == sha3Nil {
 		return nil, nil
 	}
 	if code, err := db.backend.Database().Get(codeHash[:]); err == nil {

metrics/metrics.go

@@ -19,7 +19,7 @@ import (
 //
 // This global kill-switch helps quantify the observer effect and makes
 // for less cluttered pprof profiles.
-var Enabled bool = false
+var Enabled = false
 
 // MetricsEnabledFlag is the CLI flag name to use to enable metrics collections.
 const MetricsEnabledFlag = "metrics"

metrics/opentsdb.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var shortHostName string = ""
+var shortHostName = ""
 
 // OpenTSDBConfig provides a container with configuration parameters for
 // the OpenTSDB exporter

node/config.go

@@ -466,7 +466,7 @@ func makeAccountManager(conf *Config) (*accounts.Manager, string, error) {
 		return nil, "", err
 	}
 	// Assemble the account manager and supported backends
-	backends := []accounts.Backend{}
+	var backends []accounts.Backend
 	if len(conf.ExternalSigner) > 0 {
 		log.Info("Using external signer", "url", conf.ExternalSigner)
 		if extapi, err := external.NewExternalBackend(conf.ExternalSigner); err == nil {

node/node.go

@@ -220,7 +220,7 @@ func (n *Node) Start() error {
 		return convertFileLockError(err)
 	}
 	// Start each of the services
-	started := []reflect.Type{}
+	var started []reflect.Type
 	for kind, service := range services {
 		// Start the next service, stopping all previous upon failure
 		if err := service.Start(running); err != nil {

p2p/simulations/connect.go

@@ -32,6 +32,9 @@ var (
 // It is useful when constructing a chain network topology
 // when Network adds and removes nodes dynamically.
 func (net *Network) ConnectToLastNode(id enode.ID) (err error) {
+	net.lock.Lock()
+	defer net.lock.Unlock()
+
 	ids := net.getUpNodeIDs()
 	l := len(ids)
 	if l < 2 {
@@ -41,29 +44,35 @@ func (net *Network) ConnectToLastNode(id enode.ID) (err error) {
 	if last == id {
 		last = ids[l-2]
 	}
-	return net.connect(last, id)
+	return net.connectNotConnected(last, id)
 }
 
 // ConnectToRandomNode connects the node with provided NodeID
 // to a random node that is up.
 func (net *Network) ConnectToRandomNode(id enode.ID) (err error) {
-	selected := net.GetRandomUpNode(id)
+	net.lock.Lock()
+	defer net.lock.Unlock()
+
+	selected := net.getRandomUpNode(id)
 	if selected == nil {
 		return ErrNodeNotFound
 	}
-	return net.connect(selected.ID(), id)
+	return net.connectNotConnected(selected.ID(), id)
 }
 
 // ConnectNodesFull connects all nodes one to another.
 // It provides a complete connectivity in the network
 // which should be rarely needed.
 func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) {
+	net.lock.Lock()
+	defer net.lock.Unlock()
+
 	if ids == nil {
 		ids = net.getUpNodeIDs()
 	}
 	for i, lid := range ids {
 		for _, rid := range ids[i+1:] {
-			if err = net.connect(lid, rid); err != nil {
+			if err = net.connectNotConnected(lid, rid); err != nil {
 				return err
 			}
 		}
@@ -74,12 +83,19 @@ func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) {
 // ConnectNodesChain connects all nodes in a chain topology.
 // If ids argument is nil, all nodes that are up will be connected.
 func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) {
+	net.lock.Lock()
+	defer net.lock.Unlock()
+
+	return net.connectNodesChain(ids)
+}
+
+func (net *Network) connectNodesChain(ids []enode.ID) (err error) {
 	if ids == nil {
 		ids = net.getUpNodeIDs()
 	}
 	l := len(ids)
 	for i := 0; i < l-1; i++ {
-		if err := net.connect(ids[i], ids[i+1]); err != nil {
+		if err := net.connectNotConnected(ids[i], ids[i+1]); err != nil {
 			return err
 		}
 	}
@@ -89,6 +105,9 @@ func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) {
 // ConnectNodesRing connects all nodes in a ring topology.
 // If ids argument is nil, all nodes that are up will be connected.
 func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) {
+	net.lock.Lock()
+	defer net.lock.Unlock()
+
 	if ids == nil {
 		ids = net.getUpNodeIDs()
 	}
@@ -96,15 +115,18 @@ func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) {
 	if l < 2 {
 		return nil
 	}
-	if err := net.ConnectNodesChain(ids); err != nil {
+	if err := net.connectNodesChain(ids); err != nil {
 		return err
 	}
-	return net.connect(ids[l-1], ids[0])
+	return net.connectNotConnected(ids[l-1], ids[0])
 }
 
 // ConnectNodesStar connects all nodes into a star topology
 // If ids argument is nil, all nodes that are up will be connected.
 func (net *Network) ConnectNodesStar(ids []enode.ID, center enode.ID) (err error) {
+	net.lock.Lock()
+	defer net.lock.Unlock()
+
 	if ids == nil {
 		ids = net.getUpNodeIDs()
 	}
@@ -112,16 +134,15 @@ func (net *Network) ConnectNodesStar(ids []enode.ID, center enode.ID) (err error
 		if center == id {
 			continue
 		}
-		if err := net.connect(center, id); err != nil {
+		if err := net.connectNotConnected(center, id); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-// connect connects two nodes but ignores already connected error.
+func (net *Network) connectNotConnected(oneID, otherID enode.ID) error {
-func (net *Network) connect(oneID, otherID enode.ID) error {
+	return ignoreAlreadyConnectedErr(net.connect(oneID, otherID))
-	return ignoreAlreadyConnectedErr(net.Connect(oneID, otherID))
 }
 
 func ignoreAlreadyConnectedErr(err error) error {

p2p/simulations/events.go

@@ -100,7 +100,7 @@ func ControlEvent(v interface{}) *Event {
 func (e *Event) String() string {
 	switch e.Type {
 	case EventTypeNode:
-		return fmt.Sprintf("<node-event> id: %s up: %t", e.Node.ID().TerminalString(), e.Node.Up)
+		return fmt.Sprintf("<node-event> id: %s up: %t", e.Node.ID().TerminalString(), e.Node.Up())
 	case EventTypeConn:
 		return fmt.Sprintf("<conn-event> nodes: %s->%s up: %t", e.Conn.One.TerminalString(), e.Conn.Other.TerminalString(), e.Conn.Up)
 	case EventTypeMsg:

p2p/simulations/http_test.go

@@ -421,14 +421,15 @@ type expectEvents struct {
 }
 
 func (t *expectEvents) nodeEvent(id string, up bool) *Event {
-	return &Event{
+	node := Node{
-		Type: EventTypeNode,
-		Node: &Node{
 		Config: &adapters.NodeConfig{
 			ID: enode.HexID(id),
 		},
-			Up: up,
+		up: up,
-		},
+	}
+	return &Event{
+		Type: EventTypeNode,
+		Node: &node,
 	}
 }
 
@@ -480,6 +481,7 @@ loop:
 }
 
 func (t *expectEvents) expect(events ...*Event) {
+	t.Helper()
 	timeout := time.After(10 * time.Second)
 	i := 0
 	for {
@@ -501,8 +503,8 @@ func (t *expectEvents) expect(events ...*Event) {
 				if event.Node.ID() != expected.Node.ID() {
 					t.Fatalf("expected node event %d to have id %q, got %q", i, expected.Node.ID().TerminalString(), event.Node.ID().TerminalString())
 				}
-				if event.Node.Up != expected.Node.Up {
+				if event.Node.Up() != expected.Node.Up() {
-					t.Fatalf("expected node event %d to have up=%t, got up=%t", i, expected.Node.Up, event.Node.Up)
+					t.Fatalf("expected node event %d to have up=%t, got up=%t", i, expected.Node.Up(), event.Node.Up())
 				}
 
 			case EventTypeConn:

p2p/simulations/mocker_test.go

@@ -90,15 +90,12 @@ func TestMocker(t *testing.T) {
 		for {
 			select {
 			case event := <-events:
-				//if the event is a node Up event only
+				if isNodeUp(event) {
-				if event.Node != nil && event.Node.Up {
 					//add the correspondent node ID to the map
 					nodemap[event.Node.Config.ID] = true
 					//this means all nodes got a nodeUp event, so we can continue the test
 					if len(nodemap) == nodeCount {
 						nodesComplete = true
-						//wait for 3s as the mocker will need time to connect the nodes
-						//time.Sleep( 3 *time.Second)
 					}
 				} else if event.Conn != nil && nodesComplete {
 					connCount += 1
@@ -169,3 +166,7 @@ func TestMocker(t *testing.T) {
 		t.Fatalf("Expected empty list of nodes, got: %d", len(nodesInfo))
 	}
 }
+
+func isNodeUp(event *Event) bool {
+	return event.Node != nil && event.Node.Up()
+}

p2p/simulations/network.go

@@ -137,7 +137,7 @@ func (net *Network) Config() *NetworkConfig {
 // StartAll starts all nodes in the network
 func (net *Network) StartAll() error {
 	for _, node := range net.Nodes {
-		if node.Up {
+		if node.Up() {
 			continue
 		}
 		if err := net.Start(node.ID()); err != nil {
@@ -150,7 +150,7 @@ func (net *Network) StartAll() error {
 // StopAll stops all nodes in the network
 func (net *Network) StopAll() error {
 	for _, node := range net.Nodes {
-		if !node.Up {
+		if !node.Up() {
 			continue
 		}
 		if err := net.Stop(node.ID()); err != nil {
@@ -169,27 +169,23 @@ func (net *Network) Start(id enode.ID) error {
 // snapshots
 func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) error {
 	net.lock.Lock()
+	defer net.lock.Unlock()
 
 	node := net.getNode(id)
 	if node == nil {
-		net.lock.Unlock()
 		return fmt.Errorf("node %v does not exist", id)
 	}
-	if node.Up {
+	if node.Up() {
-		net.lock.Unlock()
 		return fmt.Errorf("node %v already up", id)
 	}
 	log.Trace("Starting node", "id", id, "adapter", net.nodeAdapter.Name())
 	if err := node.Start(snapshots); err != nil {
-		net.lock.Unlock()
 		log.Warn("Node startup failed", "id", id, "err", err)
 		return err
 	}
-	node.Up = true
+	node.SetUp(true)
 	log.Info("Started node", "id", id)
 	ev := NewEvent(node)
-	net.lock.Unlock()
-
 	net.events.Send(ev)
 
 	// subscribe to peer events
@@ -220,7 +216,7 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub
 		if node == nil {
 			return
 		}
-		node.Up = false
+		node.SetUp(false)
 		ev := NewEvent(node)
 		net.events.Send(ev)
 	}()
@@ -258,30 +254,42 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub
 
 // Stop stops the node with the given ID
 func (net *Network) Stop(id enode.ID) error {
+	// IMPORTANT: node.Stop() must NOT be called under net.lock as
+	// node.Reachable() closure has a reference to the network and
+	// calls net.InitConn() what also locks the network. => DEADLOCK
+	// That holds until the following ticket is not resolved:
+
+	var err error
+
+	node, err := func() (*Node, error) {
 		net.lock.Lock()
+		defer net.lock.Unlock()
+
 		node := net.getNode(id)
 		if node == nil {
-		net.lock.Unlock()
+			return nil, fmt.Errorf("node %v does not exist", id)
-		return fmt.Errorf("node %v does not exist", id)
 		}
-	if !node.Up {
+		if !node.Up() {
-		net.lock.Unlock()
+			return nil, fmt.Errorf("node %v already down", id)
-		return fmt.Errorf("node %v already down", id)
 		}
-	node.Up = false
+		node.SetUp(false)
-	net.lock.Unlock()
+		return node, nil
-
+	}()
-	err := node.Stop()
 	if err != nil {
+		return err
+	}
+
+	err = node.Stop() // must be called without net.lock
+
 	net.lock.Lock()
-		node.Up = true
+	defer net.lock.Unlock()
-		net.lock.Unlock()
+
+	if err != nil {
+		node.SetUp(true)
 		return err
 	}
 	log.Info("Stopped node", "id", id, "err", err)
-	net.lock.Lock()
 	ev := ControlEvent(node)
-	net.lock.Unlock()
 	net.events.Send(ev)
 	return nil
 }
@@ -289,8 +297,14 @@ func (net *Network) Stop(id enode.ID) error {
 // Connect connects two nodes together by calling the "admin_addPeer" RPC
 // method on the "one" node so that it connects to the "other" node
 func (net *Network) Connect(oneID, otherID enode.ID) error {
+	net.lock.Lock()
+	defer net.lock.Unlock()
+	return net.connect(oneID, otherID)
+}
+
+func (net *Network) connect(oneID, otherID enode.ID) error {
 	log.Debug("Connecting nodes with addPeer", "id", oneID, "other", otherID)
-	conn, err := net.InitConn(oneID, otherID)
+	conn, err := net.initConn(oneID, otherID)
 	if err != nil {
 		return err
 	}
@@ -388,6 +402,14 @@ func (net *Network) GetNode(id enode.ID) *Node {
 	return net.getNode(id)
 }
 
+func (net *Network) getNode(id enode.ID) *Node {
+	i, found := net.nodeMap[id]
+	if !found {
+		return nil
+	}
+	return net.Nodes[i]
+}
+
 // GetNode gets the node with the given name, returning nil if the node does
 // not exist
 func (net *Network) GetNodeByName(name string) *Node {
@@ -410,28 +432,29 @@ func (net *Network) GetNodes() (nodes []*Node) {
 	net.lock.RLock()
 	defer net.lock.RUnlock()
 
-	nodes = append(nodes, net.Nodes...)
+	return net.getNodes()
-	return nodes
 }
 
-func (net *Network) getNode(id enode.ID) *Node {
+func (net *Network) getNodes() (nodes []*Node) {
-	i, found := net.nodeMap[id]
+	nodes = append(nodes, net.Nodes...)
-	if !found {
+	return nodes
-		return nil
-	}
-	return net.Nodes[i]
 }
 
 // GetRandomUpNode returns a random node on the network, which is running.
 func (net *Network) GetRandomUpNode(excludeIDs ...enode.ID) *Node {
 	net.lock.RLock()
 	defer net.lock.RUnlock()
+	return net.getRandomUpNode(excludeIDs...)
+}
+
+// GetRandomUpNode returns a random node on the network, which is running.
+func (net *Network) getRandomUpNode(excludeIDs ...enode.ID) *Node {
 	return net.getRandomNode(net.getUpNodeIDs(), excludeIDs)
 }
 
 func (net *Network) getUpNodeIDs() (ids []enode.ID) {
 	for _, node := range net.Nodes {
-		if node.Up {
+		if node.Up() {
 			ids = append(ids, node.ID())
 		}
 	}
@@ -446,8 +469,8 @@ func (net *Network) GetRandomDownNode(excludeIDs ...enode.ID) *Node {
 }
 
 func (net *Network) getDownNodeIDs() (ids []enode.ID) {
-	for _, node := range net.GetNodes() {
+	for _, node := range net.getNodes() {
-		if !node.Up {
+		if !node.Up() {
 			ids = append(ids, node.ID())
 		}
 	}
@@ -461,7 +484,7 @@ func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node {
 	if l == 0 {
 		return nil
 	}
-	return net.GetNode(filtered[rand.Intn(l)])
+	return net.getNode(filtered[rand.Intn(l)])
 }
 
 func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID {
@@ -539,6 +562,10 @@ func (net *Network) getConn(oneID, otherID enode.ID) *Conn {
 func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) {
 	net.lock.Lock()
 	defer net.lock.Unlock()
+	return net.initConn(oneID, otherID)
+}
+
+func (net *Network) initConn(oneID, otherID enode.ID) (*Conn, error) {
 	if oneID == otherID {
 		return nil, fmt.Errorf("refusing to connect to self %v", oneID)
 	}
@@ -602,8 +629,21 @@ type Node struct {
 	// Config if the config used to created the node
 	Config *adapters.NodeConfig `json:"config"`
 
-	// Up tracks whether or not the node is running
+	// up tracks whether or not the node is running
-	Up bool `json:"up"`
+	up   bool
+	upMu sync.RWMutex
+}
+
+func (n *Node) Up() bool {
+	n.upMu.RLock()
+	defer n.upMu.RUnlock()
+	return n.up
+}
+
+func (n *Node) SetUp(up bool) {
+	n.upMu.Lock()
+	defer n.upMu.Unlock()
+	n.up = up
 }
 
 // ID returns the ID of the node
@@ -637,10 +677,29 @@ func (n *Node) MarshalJSON() ([]byte, error) {
 	}{
 		Info:   n.NodeInfo(),
 		Config: n.Config,
-		Up:     n.Up,
+		Up:     n.Up(),
 	})
 }
 
+// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose
+// Node.up status. IMPORTANT: The implementation is incomplete; we lose
+// p2p.NodeInfo.
+func (n *Node) UnmarshalJSON(raw []byte) error {
+	// TODO: How should we turn back NodeInfo into n.Node?
+	// Ticket: https://github.com/ethersphere/go-ethereum/issues/1177
+	node := struct {
+		Config *adapters.NodeConfig `json:"config,omitempty"`
+		Up     bool                 `json:"up"`
+	}{}
+	if err := json.Unmarshal(raw, &node); err != nil {
+		return err
+	}
+
+	n.SetUp(node.Up)
+	n.Config = node.Config
+	return nil
+}
+
 // Conn represents a connection between two nodes in the network
 type Conn struct {
 	// One is the node which initiated the connection
@@ -660,10 +719,10 @@ type Conn struct {
 
 // nodesUp returns whether both nodes are currently up
 func (c *Conn) nodesUp() error {
-	if !c.one.Up {
+	if !c.one.Up() {
 		return fmt.Errorf("one %v is not up", c.One)
 	}
-	if !c.other.Up {
+	if !c.other.Up() {
 		return fmt.Errorf("other %v is not up", c.Other)
 	}
 	return nil
@@ -735,7 +794,7 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn
 	}
 	for i, node := range net.Nodes {
 		snap.Nodes[i] = NodeSnapshot{Node: *node}
-		if !node.Up {
+		if !node.Up() {
 			continue
 		}
 		snapshots, err := node.Snapshots()
@@ -790,7 +849,7 @@ func (net *Network) Load(snap *Snapshot) error {
 		if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil {
 			return err
 		}
-		if !n.Node.Up {
+		if !n.Node.Up() {
 			continue
 		}
 		if err := net.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil {
@@ -862,7 +921,7 @@ func (net *Network) Load(snap *Snapshot) error {
 	// Start connecting.
 	for _, conn := range snap.Conns {
 
-		if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up {
+		if !net.GetNode(conn.One).Up() || !net.GetNode(conn.Other).Up() {
 			//in this case, at least one of the nodes of a connection is not up,
 			//so it would result in the snapshot `Load` to fail
 			continue
@@ -916,7 +975,7 @@ func (net *Network) executeControlEvent(event *Event) {
 }
 
 func (net *Network) executeNodeEvent(e *Event) error {
-	if !e.Node.Up {
+	if !e.Node.Up() {
 		return net.Stop(e.Node.ID())
 	}
 

p2p/simulations/network_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"strconv"
 	"strings"
 	"testing"
@@ -485,3 +486,137 @@ func benchmarkMinimalServiceTmp(b *testing.B) {
 		}
 	}
 }
+
+func TestNode_UnmarshalJSON(t *testing.T) {
+	t.Run(
+		"test unmarshal of Node up field",
+		func(t *testing.T) {
+			runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONUpField())
+		},
+	)
+	t.Run(
+		"test unmarshal of Node Config field",
+		func(t *testing.T) {
+			runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONConfigField())
+		},
+	)
+}
+
+func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) {
+	t.Helper()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var got Node
+			if err := got.UnmarshalJSON([]byte(tt.marshaled)); err != nil {
+				expectErrorMessageToContain(t, err, tt.wantErr)
+			}
+			expectNodeEquality(t, got, tt.want)
+		})
+	}
+}
+
+type nodeUnmarshalTestCase struct {
+	name      string
+	marshaled string
+	want      Node
+	wantErr   string
+}
+
+func expectErrorMessageToContain(t *testing.T, got error, want string) {
+	t.Helper()
+	if got == nil && want == "" {
+		return
+	}
+
+	if got == nil && want != "" {
+		t.Errorf("error was expected, got: nil, want: %v", want)
+		return
+	}
+
+	if !strings.Contains(got.Error(), want) {
+		t.Errorf(
+			"unexpected error message, got  %v, want: %v",
+			want,
+			got,
+		)
+	}
+}
+
+func expectNodeEquality(t *testing.T, got Node, want Node) {
+	t.Helper()
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Node.UnmarshalJSON() = %v, want %v", got, want)
+	}
+}
+
+func casesNodeUnmarshalJSONUpField() []nodeUnmarshalTestCase {
+	return []nodeUnmarshalTestCase{
+		{
+			name:      "empty json",
+			marshaled: "{}",
+			want: Node{
+				up: false,
+			},
+		},
+		{
+			name:      "a stopped node",
+			marshaled: "{\"up\": false}",
+			want: Node{
+				up: false,
+			},
+		},
+		{
+			name:      "a running node",
+			marshaled: "{\"up\": true}",
+			want: Node{
+				up: true,
+			},
+		},
+		{
+			name:      "invalid JSON value on valid key",
+			marshaled: "{\"up\": foo}",
+			wantErr:   "invalid character",
+		},
+		{
+			name:      "invalid JSON key and value",
+			marshaled: "{foo: bar}",
+			wantErr:   "invalid character",
+		},
+		{
+			name:      "bool value expected but got something else (string)",
+			marshaled: "{\"up\": \"true\"}",
+			wantErr:   "cannot unmarshal string into Go struct",
+		},
+	}
+}
+
+func casesNodeUnmarshalJSONConfigField() []nodeUnmarshalTestCase {
+	// Don't do a big fuss around testing, as adapters.NodeConfig should
+	// handle it's own serialization. Just do a sanity check.
+	return []nodeUnmarshalTestCase{
+		{
+			name:      "Config field is omitted",
+			marshaled: "{}",
+			want: Node{
+				Config: nil,
+			},
+		},
+		{
+			name:      "Config field is nil",
+			marshaled: "{\"config\": nil}",
+			want: Node{
+				Config: nil,
+			},
+		},
+		{
+			name:      "a non default Config field",
+			marshaled: "{\"config\":{\"name\":\"node_ecdd0\",\"port\":44665}}",
+			want: Node{
+				Config: &adapters.NodeConfig{
+					Name: "node_ecdd0",
+					Port: 44665,
+				},
+			},
+		},
+	}
+}

rpc/stdio.go

@@ -19,6 +19,7 @@ package rpc
 import (
 	"context"
 	"errors"
+	"io"
 	"net"
 	"os"
 	"time"
@@ -26,19 +27,30 @@ import (
 
 // DialStdIO creates a client on stdin/stdout.
 func DialStdIO(ctx context.Context) (*Client, error) {
+	return DialIO(ctx, os.Stdin, os.Stdout)
+}
+
+// DialIO creates a client which uses the given IO channels
+func DialIO(ctx context.Context, in io.Reader, out io.Writer) (*Client, error) {
 	return newClient(ctx, func(_ context.Context) (ServerCodec, error) {
-		return NewJSONCodec(stdioConn{}), nil
+		return NewJSONCodec(stdioConn{
+			in:  in,
+			out: out,
+		}), nil
 	})
 }
 
-type stdioConn struct{}
+type stdioConn struct {
+	in  io.Reader
+	out io.Writer
+}
 
 func (io stdioConn) Read(b []byte) (n int, err error) {
-	return os.Stdin.Read(b)
+	return io.in.Read(b)
 }
 
 func (io stdioConn) Write(b []byte) (n int, err error) {
-	return os.Stdout.Write(b)
+	return io.out.Write(b)
 }
 
 func (io stdioConn) Close() error {

signer/core/api.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"reflect"
 
@@ -39,9 +38,9 @@ const (
 	// numberOfAccountsToDerive For hardware wallets, the number of accounts to derive
 	numberOfAccountsToDerive = 10
 	// ExternalAPIVersion -- see extapi_changelog.md
-	ExternalAPIVersion = "5.0.0"
+	ExternalAPIVersion = "6.0.0"
 	// InternalAPIVersion -- see intapi_changelog.md
-	InternalAPIVersion = "3.1.0"
+	InternalAPIVersion = "4.0.0"
 )
 
 // ExternalAPI defines the external API through which signing requests are made.
@@ -49,7 +48,7 @@ type ExternalAPI interface {
 	// List available accounts
 	List(ctx context.Context) ([]common.Address, error)
 	// New request to create a new account
-	New(ctx context.Context) (accounts.Account, error)
+	New(ctx context.Context) (common.Address, error)
 	// SignTransaction request to sign the specified transaction
 	SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error)
 	// SignData - request to sign the given data (plus prefix)
@@ -58,17 +57,13 @@ type ExternalAPI interface {
 	SignTypedData(ctx context.Context, addr common.MixedcaseAddress, data TypedData) (hexutil.Bytes, error)
 	// EcRecover - recover public key from given message and signature
 	EcRecover(ctx context.Context, data hexutil.Bytes, sig hexutil.Bytes) (common.Address, error)
-	// Export - request to export an account
+	// Version info about the APIs
-	Export(ctx context.Context, addr common.Address) (json.RawMessage, error)
-	// Import - request to import an account
-	// Should be moved to Internal API, in next phase when we have
-	// bi-directional communication
-	//Import(ctx context.Context, keyJSON json.RawMessage) (Account, error)
 	Version(ctx context.Context) (string, error)
 }
 
-// SignerUI specifies what method a UI needs to implement to be able to be used as a UI for the signer
+// UIClientAPI specifies what method a UI needs to implement to be able to be used as a
-type SignerUI interface {
+// UI for the signer
+type UIClientAPI interface {
 	// ApproveTx prompt the user for confirmation to request to sign Transaction
 	ApproveTx(request *SignTxRequest) (SignTxResponse, error)
 	// ApproveSignData prompt the user for confirmation to request to sign data
@@ -95,13 +90,15 @@ type SignerUI interface {
 	// OnInputRequired is invoked when clef requires user input, for example master password or
 	// pin-code for unlocking hardware wallets
 	OnInputRequired(info UserInputRequest) (UserInputResponse, error)
+	// RegisterUIServer tells the UI to use the given UIServerAPI for ui->clef communication
+	RegisterUIServer(api *UIServerAPI)
 }
 
 // SignerAPI defines the actual implementation of ExternalAPI
 type SignerAPI struct {
 	chainID    *big.Int
 	am         *accounts.Manager
-	UI         SignerUI
+	UI         UIClientAPI
 	validator  *Validator
 	rejectMode bool
 }
@@ -115,6 +112,37 @@ type Metadata struct {
 	Origin    string `json:"Origin"`
 }
 
+func StartClefAccountManager(ksLocation string, nousb, lightKDF bool) *accounts.Manager {
+	var (
+		backends []accounts.Backend
+		n, p     = keystore.StandardScryptN, keystore.StandardScryptP
+	)
+	if lightKDF {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	// support password based accounts
+	if len(ksLocation) > 0 {
+		backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
+	}
+	if !nousb {
+		// Start a USB hub for Ledger hardware wallets
+		if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
+			log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
+		} else {
+			backends = append(backends, ledgerhub)
+			log.Debug("Ledger support enabled")
+		}
+		// Start a USB hub for Trezor hardware wallets
+		if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
+			log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
+		} else {
+			backends = append(backends, trezorhub)
+			log.Debug("Trezor support enabled")
+		}
+	}
+	return accounts.NewManager(backends...)
+}
+
 // MetadataFromContext extracts Metadata from a given context.Context
 func MetadataFromContext(ctx context.Context) Metadata {
 	m := Metadata{"NA", "NA", "NA", "", ""} // batman
@@ -199,21 +227,15 @@ type (
 		Password string `json:"password"`
 	}
 	ListRequest struct {
-		Accounts []Account `json:"accounts"`
+		Accounts []accounts.Account `json:"accounts"`
 		Meta     Metadata           `json:"meta"`
 	}
 	ListResponse struct {
-		Accounts []Account `json:"accounts"`
+		Accounts []accounts.Account `json:"accounts"`
 	}
 	Message struct {
 		Text string `json:"text"`
 	}
-	PasswordRequest struct {
-		Prompt string `json:"prompt"`
-	}
-	PasswordResponse struct {
-		Password string `json:"password"`
-	}
 	StartupInfo struct {
 		Info map[string]interface{} `json:"info"`
 	}
@@ -234,38 +256,11 @@ var ErrRequestDenied = errors.New("Request denied")
 // key that is generated when a new Account is created.
 // noUSB disables USB support that is required to support hardware devices such as
 // ledger and trezor.
-func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abidb *AbiDb, lightKDF bool, advancedMode bool) *SignerAPI {
+func NewSignerAPI(am *accounts.Manager, chainID int64, noUSB bool, ui UIClientAPI, abidb *AbiDb, advancedMode bool) *SignerAPI {
-	var (
-		backends []accounts.Backend
-		n, p     = keystore.StandardScryptN, keystore.StandardScryptP
-	)
-	if lightKDF {
-		n, p = keystore.LightScryptN, keystore.LightScryptP
-	}
-	// support password based accounts
-	if len(ksLocation) > 0 {
-		backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
-	}
 	if advancedMode {
 		log.Info("Clef is in advanced mode: will warn instead of reject")
 	}
-	if !noUSB {
+	signer := &SignerAPI{big.NewInt(chainID), am, ui, NewValidator(abidb), !advancedMode}
-		// Start a USB hub for Ledger hardware wallets
-		if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
-			log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
-		} else {
-			backends = append(backends, ledgerhub)
-			log.Debug("Ledger support enabled")
-		}
-		// Start a USB hub for Trezor hardware wallets
-		if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
-			log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
-		} else {
-			backends = append(backends, trezorhub)
-			log.Debug("Trezor support enabled")
-		}
-	}
-	signer := &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb), !advancedMode}
 	if !noUSB {
 		signer.startUSBListener()
 	}
@@ -358,12 +353,9 @@ func (api *SignerAPI) startUSBListener() {
 // List returns the set of wallet this signer manages. Each wallet can contain
 // multiple accounts.
 func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
-	var accs []Account
+	var accs []accounts.Account
 	for _, wallet := range api.am.Wallets() {
-		for _, acc := range wallet.Accounts() {
+		accs = append(accs, wallet.Accounts()...)
-			acc := Account{Typ: "Account", URL: wallet.URL(), Address: acc.Address}
-			accs = append(accs, acc)
-		}
 	}
 	result, err := api.UI.ApproveListing(&ListRequest{Accounts: accs, Meta: MetadataFromContext(ctx)})
 	if err != nil {
@@ -373,7 +365,6 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
 		return nil, ErrRequestDenied
 
 	}
-
 	addresses := make([]common.Address, 0)
 	for _, acc := range result.Accounts {
 		addresses = append(addresses, acc.Address)
@@ -385,10 +376,10 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
 // New creates a new password protected Account. The private key is protected with
 // the given password. Users are responsible to backup the private key that is stored
 // in the keystore location thas was specified when this API was created.
-func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
+func (api *SignerAPI) New(ctx context.Context) (common.Address, error) {
 	be := api.am.Backends(keystore.KeyStoreType)
 	if len(be) == 0 {
-		return accounts.Account{}, errors.New("password based accounts not supported")
+		return common.Address{}, errors.New("password based accounts not supported")
 	}
 	var (
 		resp NewAccountResponse
@@ -398,20 +389,21 @@ func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
 	for i := 0; i < 3; i++ {
 		resp, err = api.UI.ApproveNewAccount(&NewAccountRequest{MetadataFromContext(ctx)})
 		if err != nil {
-			return accounts.Account{}, err
+			return common.Address{}, err
 		}
 		if !resp.Approved {
-			return accounts.Account{}, ErrRequestDenied
+			return common.Address{}, ErrRequestDenied
 		}
 		if pwErr := ValidatePasswordFormat(resp.Password); pwErr != nil {
 			api.UI.ShowError(fmt.Sprintf("Account creation attempt #%d failed due to password requirements: %v", (i + 1), pwErr))
 		} else {
 			// No error
-			return be[0].(*keystore.KeyStore).NewAccount(resp.Password)
+			acc, err := be[0].(*keystore.KeyStore).NewAccount(resp.Password)
+			return acc.Address, err
 		}
 	}
 	// Otherwise fail, with generic error message
-	return accounts.Account{}, errors.New("account creation failed")
+	return common.Address{}, errors.New("account creation failed")
 }
 
 // logDiff logs the difference between the incoming (original) transaction and the one returned from the signer.
@@ -521,57 +513,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth
 
 }
 
-// Export returns encrypted private key associated with the given address in web3 keystore format.
-func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
-	res, err := api.UI.ApproveExport(&ExportRequest{Address: addr, Meta: MetadataFromContext(ctx)})
-
-	if err != nil {
-		return nil, err
-	}
-	if !res.Approved {
-		return nil, ErrRequestDenied
-	}
-	// Look up the wallet containing the requested signer
-	wallet, err := api.am.Find(accounts.Account{Address: addr})
-	if err != nil {
-		return nil, err
-	}
-	if wallet.URL().Scheme != keystore.KeyStoreScheme {
-		return nil, fmt.Errorf("Account is not a keystore-account")
-	}
-	return ioutil.ReadFile(wallet.URL().Path)
-}
-
-// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
-// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
-// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
-// OBS! This method is removed from the public API. It should not be exposed on the external API
-// for a couple of reasons:
-// 1. Even though it is encrypted, it should still be seen as sensitive data
-// 2. It can be used to DoS clef, by using malicious data with e.g. extreme large
-// values for the kdfparams.
-func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) {
-	be := api.am.Backends(keystore.KeyStoreType)
-
-	if len(be) == 0 {
-		return Account{}, errors.New("password based accounts not supported")
-	}
-	res, err := api.UI.ApproveImport(&ImportRequest{Meta: MetadataFromContext(ctx)})
-
-	if err != nil {
-		return Account{}, err
-	}
-	if !res.Approved {
-		return Account{}, ErrRequestDenied
-	}
-	acc, err := be[0].(*keystore.KeyStore).Import(keyJSON, res.OldPassword, res.NewPassword)
-	if err != nil {
-		api.UI.ShowError(err.Error())
-		return Account{}, err
-	}
-	return Account{Typ: "Account", URL: acc.URL, Address: acc.Address}, nil
-}
-
 // Returns the external api version. This method does not require user acceptance. Available methods are
 // available via enumeration anyway, and this info does not contain user-specific data
 func (api *SignerAPI) Version(ctx context.Context) (string, error) {

signer/core/api_test.go

@@ -28,6 +28,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -47,6 +48,8 @@ func (ui *HeadlessUI) OnInputRequired(info UserInputRequest) (UserInputResponse,
 
 func (ui *HeadlessUI) OnSignerStartup(info StartupInfo) {
 }
+func (ui *HeadlessUI) RegisterUIServer(api *UIServerAPI) {
+}
 
 func (ui *HeadlessUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
 	fmt.Printf("OnApproved()\n")
@@ -91,7 +94,7 @@ func (ui *HeadlessUI) ApproveListing(request *ListRequest) (ListResponse, error)
 	case "A":
 		return ListResponse{request.Accounts}, nil
 	case "1":
-		l := make([]Account, 1)
+		l := make([]accounts.Account, 1)
 		l[0] = request.Accounts[1]
 		return ListResponse{l}, nil
 	default:
@@ -138,13 +141,8 @@ func setup(t *testing.T) (*SignerAPI, chan string) {
 	}
 	var (
 		ui  = &HeadlessUI{controller}
-		api = NewSignerAPI(
+		am  = StartClefAccountManager(tmpDirName(t), true, true)
-			1,
+		api = NewSignerAPI(am, 1337, true, ui, db, true)
-			tmpDirName(t),
-			true,
-			ui,
-			db,
-			true, true)
 	)
 	return api, controller
 }
@@ -169,22 +167,22 @@ func failCreateAccountWithPassword(control chan string, api *SignerAPI, password
 	control <- "Y"
 	control <- password
 
-	acc, err := api.New(context.Background())
+	addr, err := api.New(context.Background())
 	if err == nil {
 		t.Fatal("Should have returned an error")
 	}
-	if acc.Address != (common.Address{}) {
+	if addr != (common.Address{}) {
 		t.Fatal("Empty address should be returned")
 	}
 }
 
 func failCreateAccount(control chan string, api *SignerAPI, t *testing.T) {
 	control <- "N"
-	acc, err := api.New(context.Background())
+	addr, err := api.New(context.Background())
 	if err != ErrRequestDenied {
 		t.Fatal(err)
 	}
-	if acc.Address != (common.Address{}) {
+	if addr != (common.Address{}) {
 		t.Fatal("Empty address should be returned")
 	}
 }

signer/core/auditlog.go

@@ -18,9 +18,7 @@ package core
 
 import (
 	"context"
-	"encoding/json"
 
-	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
@@ -40,7 +38,7 @@ func (l *AuditLogger) List(ctx context.Context) ([]common.Address, error) {
 	return res, e
 }
 
-func (l *AuditLogger) New(ctx context.Context) (accounts.Account, error) {
+func (l *AuditLogger) New(ctx context.Context) (common.Address, error) {
 	return l.api.New(ctx)
 }
 
@@ -86,15 +84,6 @@ func (l *AuditLogger) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex
 	return b, e
 }
 
-func (l *AuditLogger) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
-	l.log.Info("Export", "type", "request", "metadata", MetadataFromContext(ctx).String(),
-		"addr", addr.Hex())
-	j, e := l.api.Export(ctx, addr)
-	// In this case, we don't actually log the json-response, which may be extra sensitive
-	l.log.Info("Export", "type", "response", "json response size", len(j), "error", e)
-	return j, e
-}
-
 func (l *AuditLogger) Version(ctx context.Context) (string, error) {
 	l.log.Info("Version", "type", "request", "metadata", MetadataFromContext(ctx).String())
 	data, err := l.api.Version(ctx)

signer/core/cliui.go

@@ -18,12 +18,12 @@ package core
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
 	"sync"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
@@ -39,6 +39,10 @@ func NewCommandlineUI() *CommandlineUI {
 	return &CommandlineUI{in: bufio.NewReader(os.Stdin)}
 }
 
+func (ui *CommandlineUI) RegisterUIServer(api *UIServerAPI) {
+	// noop
+}
+
 // readString reads a single line from stdin, trimming if from spaces, enforcing
 // non-emptyness.
 func (ui *CommandlineUI) readString() string {
@@ -223,7 +227,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err
 	for _, account := range request.Accounts {
 		fmt.Printf("  [x] %v\n", account.Address.Hex())
 		fmt.Printf("    URL: %v\n", account.URL)
-		fmt.Printf("    Type: %v\n", account.Typ)
 	}
 	fmt.Printf("-------------------------------------------\n")
 	showMetadata(request.Meta)
@@ -264,7 +267,11 @@ func (ui *CommandlineUI) ShowInfo(message string) {
 
 func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
 	fmt.Printf("Transaction signed:\n ")
-	spew.Dump(tx.Tx)
+	if jsn, err := json.MarshalIndent(tx.Tx, "  ", "  "); err != nil {
+		fmt.Printf("WARN: marshalling error %v\n", err)
+	} else {
+		fmt.Println(string(jsn))
+	}
 }
 
 func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) {

signer/core/signed_data.go

@@ -121,8 +121,8 @@ var typedDataReferenceTypeRegexp = regexp.MustCompile(`^[A-Z](\w*)(\[\])?$`)
 // sign receives a request and produces a signature
 
 // Note, the produced signature conforms to the secp256k1 curve R, S and V values,
-// where the V value will be 27 or 28 for legacy reasons.
+// where the V value will be 27 or 28 for legacy reasons, if legacyV==true.
-func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (hexutil.Bytes, error) {
+func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest, legacyV bool) (hexutil.Bytes, error) {
 
 	// We make the request prior to looking up if we actually have the account, to prevent
 	// account-enumeration via the API
@@ -140,11 +140,13 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (
 		return nil, err
 	}
 	// Sign the data with the wallet
-	signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Hash)
+	signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Rawdata)
 	if err != nil {
 		return nil, err
 	}
+	if legacyV {
 		signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
+	}
 	return signature, nil
 }
 
@@ -153,17 +155,16 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (
 //
 // Different types of validation occur.
 func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (hexutil.Bytes, error) {
-	var req, err = api.determineSignatureFormat(ctx, contentType, addr, data)
+	var req, transformV, err = api.determineSignatureFormat(ctx, contentType, addr, data)
 	if err != nil {
 		return nil, err
 	}
 
-	signature, err := api.sign(addr, req)
+	signature, err := api.sign(addr, req, transformV)
 	if err != nil {
 		api.UI.ShowError(err.Error())
 		return nil, err
 	}
-
 	return signature, nil
 }
 
@@ -173,12 +174,14 @@ func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr com
 // charset, ok := params["charset"]
 // As it is now, we accept any charset and just treat it as 'raw'.
 // This method returns the mimetype for signing along with the request
-func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, error) {
+func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, bool, error) {
-	var req *SignDataRequest
+	var (
-
+		req          *SignDataRequest
+		useEthereumV = true // Default to use V = 27 or 28, the legacy Ethereum format
+	)
 	mediaType, _, err := mime.ParseMediaType(contentType)
 	if err != nil {
-		return nil, err
+		return nil, useEthereumV, err
 	}
 
 	switch mediaType {
@@ -186,7 +189,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType
 		// Data with an intended validator
 		validatorData, err := UnmarshalValidatorData(data)
 		if err != nil {
-			return nil, err
+			return nil, useEthereumV, err
 		}
 		sighash, msg := SignTextValidator(validatorData)
 		message := []*NameValueType{
@@ -201,55 +204,63 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType
 		// Clique is the Ethereum PoA standard
 		stringData, ok := data.(string)
 		if !ok {
-			return nil, fmt.Errorf("input for %v plain must be an hex-encoded string", ApplicationClique.Mime)
+			return nil, useEthereumV, fmt.Errorf("input for %v must be an hex-encoded string", ApplicationClique.Mime)
 		}
 		cliqueData, err := hexutil.Decode(stringData)
 		if err != nil {
-			return nil, err
+			return nil, useEthereumV, err
 		}
 		header := &types.Header{}
 		if err := rlp.DecodeBytes(cliqueData, header); err != nil {
-			return nil, err
+			return nil, useEthereumV, err
+		}
+		// The incoming clique header is already truncated, sent to us with a extradata already shortened
+		if len(header.Extra) < 65 {
+			// Need to add it back, to get a suitable length for hashing
+			newExtra := make([]byte, len(header.Extra)+65)
+			copy(newExtra, header.Extra)
+			header.Extra = newExtra
 		}
 		// Get back the rlp data, encoded by us
-		cliqueData = clique.CliqueRLP(header)
+		sighash, cliqueRlp, err := cliqueHeaderHashAndRlp(header)
-		sighash, err := SignCliqueHeader(header)
 		if err != nil {
-			return nil, err
+			return nil, useEthereumV, err
 		}
 		message := []*NameValueType{
 			{
-				Name:  "Clique block",
+				Name:  "Clique header",
 				Typ:   "clique",
-				Value: fmt.Sprintf("clique block %d [0x%x]", header.Number, header.Hash()),
+				Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()),
 			},
 		}
-		req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueData, Message: message, Hash: sighash}
+		// Clique uses V on the form 0 or 1
+		useEthereumV = false
+		req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Message: message, Hash: sighash}
 	default: // also case TextPlain.Mime:
 		// Calculates an Ethereum ECDSA signature for:
 		// hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}")
 		// We expect it to be a string
-		stringData, ok := data.(string)
+		if stringData, ok := data.(string); !ok {
-		if !ok {
+			return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string")
-			return nil, fmt.Errorf("input for text/plain must be a string")
+		} else {
-		}
+			if textData, err := hexutil.Decode(stringData); err != nil {
-		//plainData, err := hexutil.Decode(stringdata)
+				return nil, useEthereumV, err
-		//if err != nil {
+			} else {
-		//	return nil, err
+				sighash, msg := accounts.TextAndHash(textData)
-		//}
-		sighash, msg := accounts.TextAndHash([]byte(stringData))
 				message := []*NameValueType{
 					{
 						Name:  "message",
-				Typ:   "text/plain",
+						Typ:   accounts.MimetypeTextPlain,
 						Value: msg,
 					},
 				}
 				req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash}
 			}
+		}
+	}
 	req.Address = addr
 	req.Meta = MetadataFromContext(ctx)
-	return req, nil
+	return req, useEthereumV, nil
 
 }
 
@@ -262,20 +273,21 @@ func SignTextValidator(validatorData ValidatorData) (hexutil.Bytes, string) {
 	return crypto.Keccak256([]byte(msg)), msg
 }
 
-// SignCliqueHeader returns the hash which is used as input for the proof-of-authority
+// cliqueHeaderHashAndRlp returns the hash which is used as input for the proof-of-authority
 // signing. It is the hash of the entire header apart from the 65 byte signature
 // contained at the end of the extra data.
 //
 // The method requires the extra data to be at least 65 bytes -- the original implementation
 // in clique.go panics if this is the case, thus it's been reimplemented here to avoid the panic
 // and simply return an error instead
-func SignCliqueHeader(header *types.Header) (hexutil.Bytes, error) {
+func cliqueHeaderHashAndRlp(header *types.Header) (hash, rlp []byte, err error) {
-	//hash := common.Hash{}
 	if len(header.Extra) < 65 {
-		return nil, fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra))
+		err = fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra))
+		return
 	}
-	hash := clique.SealHash(header)
+	rlp = clique.CliqueRLP(header)
-	return hash.Bytes(), nil
+	hash = clique.SealHash(header).Bytes()
+	return hash, rlp, err
 }
 
 // SignTypedData signs EIP-712 conformant typed data
@@ -293,7 +305,7 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd
 	sighash := crypto.Keccak256(rawData)
 	message := typedData.Format()
 	req := &SignDataRequest{ContentType: DataTyped.Mime, Rawdata: rawData, Message: message, Hash: sighash}
-	signature, err := api.sign(addr, req)
+	signature, err := api.sign(addr, req, true)
 	if err != nil {
 		api.UI.ShowError(err.Error())
 		return nil, err
@@ -722,7 +734,7 @@ func (nvt *NameValueType) Pprint(depth int) string {
 			output.WriteString(sublevel)
 		}
 	} else {
-		output.WriteString(fmt.Sprintf("%s\n", nvt.Value))
+		output.WriteString(fmt.Sprintf("%q\n", nvt.Value))
 	}
 	return output.String()
 }

signer/core/stdioui.go

@@ -32,12 +32,16 @@ type StdIOUI struct {
 }
 
 func NewStdIOUI() *StdIOUI {
-	log.Info("NewStdIOUI")
 	client, err := rpc.DialContext(context.Background(), "stdio://")
 	if err != nil {
 		log.Crit("Could not create stdio client", "err", err)
 	}
-	return &StdIOUI{client: *client}
+	ui := &StdIOUI{client: *client}
+	return ui
+}
+
+func (ui *StdIOUI) RegisterUIServer(api *UIServerAPI) {
+	ui.client.RegisterName("clef", api)
 }
 
 // dispatch sends a request over the stdio
@@ -49,6 +53,16 @@ func (ui *StdIOUI) dispatch(serviceMethod string, args interface{}, reply interf
 	return err
 }
 
+// notify sends a request over the stdio, and does not listen for a response
+func (ui *StdIOUI) notify(serviceMethod string, args interface{}) error {
+	ctx := context.Background()
+	err := ui.client.Notify(ctx, serviceMethod, args)
+	if err != nil {
+		log.Info("Error", "exc", err.Error())
+	}
+	return err
+}
+
 func (ui *StdIOUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) {
 	var result SignTxResponse
 	err := ui.dispatch("ApproveTx", request, &result)
@@ -86,27 +100,27 @@ func (ui *StdIOUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResp
 }
 
 func (ui *StdIOUI) ShowError(message string) {
-	err := ui.dispatch("ShowError", &Message{message}, nil)
+	err := ui.notify("ShowError", &Message{message})
 	if err != nil {
 		log.Info("Error calling 'ShowError'", "exc", err.Error(), "msg", message)
 	}
 }
 
 func (ui *StdIOUI) ShowInfo(message string) {
-	err := ui.dispatch("ShowInfo", Message{message}, nil)
+	err := ui.notify("ShowInfo", Message{message})
 	if err != nil {
 		log.Info("Error calling 'ShowInfo'", "exc", err.Error(), "msg", message)
 	}
 }
 func (ui *StdIOUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
-	err := ui.dispatch("OnApprovedTx", tx, nil)
+	err := ui.notify("OnApprovedTx", tx)
 	if err != nil {
 		log.Info("Error calling 'OnApprovedTx'", "exc", err.Error(), "tx", tx)
 	}
 }
 
 func (ui *StdIOUI) OnSignerStartup(info StartupInfo) {
-	err := ui.dispatch("OnSignerStartup", info, nil)
+	err := ui.notify("OnSignerStartup", info)
 	if err != nil {
 		log.Info("Error calling 'OnSignerStartup'", "exc", err.Error(), "info", info)
 	}

signer/core/types.go

@@ -22,36 +22,11 @@ import (
 	"math/big"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
-type Accounts []Account
-
-func (as Accounts) String() string {
-	var output []string
-	for _, a := range as {
-		output = append(output, a.String())
-	}
-	return strings.Join(output, "\n")
-}
-
-type Account struct {
-	Typ     string         `json:"type"`
-	URL     accounts.URL   `json:"url"`
-	Address common.Address `json:"address"`
-}
-
-func (a Account) String() string {
-	s, err := json.Marshal(a)
-	if err == nil {
-		return string(s)
-	}
-	return err.Error()
-}
-
 type ValidationInfo struct {
 	Typ     string `json:"type"`
 	Message string `json:"message"`
@@ -100,7 +75,7 @@ type SendTxArgs struct {
 	Nonce    hexutil.Uint64           `json:"nonce"`
 	// We accept "data" and "input" for backwards-compatibility reasons.
 	Data  *hexutil.Bytes `json:"data"`
-	Input *hexutil.Bytes `json:"input"`
+	Input *hexutil.Bytes `json:"input,omitempty"`
 }
 
 func (args SendTxArgs) String() string {

signer/core/uiapi.go

@@ -0,0 +1,201 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package core
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// SignerUIAPI implements methods Clef provides for a UI to query, in the bidirectional communication
+// channel.
+// This API is considered secure, since a request can only
+// ever arrive from the UI -- and the UI is capable of approving any action, thus we can consider these
+// requests pre-approved.
+// NB: It's very important that these methods are not ever exposed on the external service
+// registry.
+type UIServerAPI struct {
+	extApi *SignerAPI
+	am     *accounts.Manager
+}
+
+// NewUIServerAPI creates a new UIServerAPI
+func NewUIServerAPI(extapi *SignerAPI) *UIServerAPI {
+	return &UIServerAPI{extapi, extapi.am}
+}
+
+// List available accounts. As opposed to the external API definition, this method delivers
+// the full Account object and not only Address.
+// Example call
+// {"jsonrpc":"2.0","method":"clef_listAccounts","params":[], "id":4}
+func (s *UIServerAPI) ListAccounts(ctx context.Context) ([]accounts.Account, error) {
+	var accs []accounts.Account
+	for _, wallet := range s.am.Wallets() {
+		accs = append(accs, wallet.Accounts()...)
+	}
+	return accs, nil
+}
+
+// rawWallet is a JSON representation of an accounts.Wallet interface, with its
+// data contents extracted into plain fields.
+type rawWallet struct {
+	URL      string             `json:"url"`
+	Status   string             `json:"status"`
+	Failure  string             `json:"failure,omitempty"`
+	Accounts []accounts.Account `json:"accounts,omitempty"`
+}
+
+// ListWallets will return a list of wallets that clef manages
+// Example call
+// {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5}
+func (s *UIServerAPI) ListWallets() []rawWallet {
+	wallets := make([]rawWallet, 0) // return [] instead of nil if empty
+	for _, wallet := range s.am.Wallets() {
+		status, failure := wallet.Status()
+
+		raw := rawWallet{
+			URL:      wallet.URL().String(),
+			Status:   status,
+			Accounts: wallet.Accounts(),
+		}
+		if failure != nil {
+			raw.Failure = failure.Error()
+		}
+		wallets = append(wallets, raw)
+	}
+	return wallets
+}
+
+// DeriveAccount requests a HD wallet to derive a new account, optionally pinning
+// it for later reuse.
+// Example call
+// {"jsonrpc":"2.0","method":"clef_deriveAccount","params":["ledger://","m/44'/60'/0'", false], "id":6}
+func (s *UIServerAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) {
+	wallet, err := s.am.Wallet(url)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	derivPath, err := accounts.ParseDerivationPath(path)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	if pin == nil {
+		pin = new(bool)
+	}
+	return wallet.Derive(derivPath, *pin)
+}
+
+// fetchKeystore retrives the encrypted keystore from the account manager.
+func fetchKeystore(am *accounts.Manager) *keystore.KeyStore {
+	return am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+}
+
+// ImportRawKey stores the given hex encoded ECDSA key into the key directory,
+// encrypting it with the passphrase.
+// Example call (should fail on password too short)
+// {"jsonrpc":"2.0","method":"clef_importRawKey","params":["1111111111111111111111111111111111111111111111111111111111111111","test"], "id":6}
+func (s *UIServerAPI) ImportRawKey(privkey string, password string) (accounts.Account, error) {
+	key, err := crypto.HexToECDSA(privkey)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	if err := ValidatePasswordFormat(password); err != nil {
+		return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err)
+	}
+	// No error
+	return fetchKeystore(s.am).ImportECDSA(key, password)
+}
+
+// OpenWallet initiates a hardware wallet opening procedure, establishing a USB
+// connection and attempting to authenticate via the provided passphrase. Note,
+// the method may return an extra challenge requiring a second open (e.g. the
+// Trezor PIN matrix challenge).
+// Example
+// {"jsonrpc":"2.0","method":"clef_openWallet","params":["ledger://",""], "id":6}
+func (s *UIServerAPI) OpenWallet(url string, passphrase *string) error {
+	wallet, err := s.am.Wallet(url)
+	if err != nil {
+		return err
+	}
+	pass := ""
+	if passphrase != nil {
+		pass = *passphrase
+	}
+	return wallet.Open(pass)
+}
+
+// ChainId returns the chainid in use for Eip-155 replay protection
+// Example call
+// {"jsonrpc":"2.0","method":"clef_chainId","params":[], "id":8}
+func (s *UIServerAPI) ChainId() math.HexOrDecimal64 {
+	return (math.HexOrDecimal64)(s.extApi.chainID.Uint64())
+}
+
+// SetChainId sets the chain id to use when signing transactions.
+// Example call to set Ropsten:
+// {"jsonrpc":"2.0","method":"clef_setChainId","params":["3"], "id":8}
+func (s *UIServerAPI) SetChainId(id math.HexOrDecimal64) math.HexOrDecimal64 {
+	s.extApi.chainID = new(big.Int).SetUint64(uint64(id))
+	return s.ChainId()
+}
+
+// Export returns encrypted private key associated with the given address in web3 keystore format.
+// Example
+// {"jsonrpc":"2.0","method":"clef_export","params":["0x19e7e376e7c213b7e7e7e46cc70a5dd086daff2a"], "id":4}
+func (s *UIServerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
+	// Look up the wallet containing the requested signer
+	wallet, err := s.am.Find(accounts.Account{Address: addr})
+	if err != nil {
+		return nil, err
+	}
+	if wallet.URL().Scheme != keystore.KeyStoreScheme {
+		return nil, fmt.Errorf("Account is not a keystore-account")
+	}
+	return ioutil.ReadFile(wallet.URL().Path)
+}
+
+// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
+// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
+// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
+// Example (the address in question has privkey `11...11`):
+// {"jsonrpc":"2.0","method":"clef_import","params":[{"address":"19e7e376e7c213b7e7e7e46cc70a5dd086daff2a","crypto":{"cipher":"aes-128-ctr","ciphertext":"33e4cd3756091d037862bb7295e9552424a391a6e003272180a455ca2a9fb332","cipherparams":{"iv":"b54b263e8f89c42bb219b6279fba5cce"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"e4ca94644fd30569c1b1afbbc851729953c92637b7fe4bb9840bbb31ffbc64a5"},"mac":"f4092a445c2b21c0ef34f17c9cd0d873702b2869ec5df4439a0c2505823217e7"},"id":"216c7eac-e8c1-49af-a215-fa0036f29141","version":3},"test","yaddayadda"], "id":4}
+func (api *UIServerAPI) Import(ctx context.Context, keyJSON json.RawMessage, oldPassphrase, newPassphrase string) (accounts.Account, error) {
+	be := api.am.Backends(keystore.KeyStoreType)
+
+	if len(be) == 0 {
+		return accounts.Account{}, errors.New("password based accounts not supported")
+	}
+	if err := ValidatePasswordFormat(newPassphrase); err != nil {
+		return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err)
+	}
+	return be[0].(*keystore.KeyStore).Import(keyJSON, oldPassphrase, newPassphrase)
+}
+
+// Other methods to be added, not yet implemented are:
+// - Ruleset interaction: add rules, attest rulefiles
+// - Store metadata about accounts, e.g. naming of accounts

signer/core/validation_test.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
-func hexAddr(a string) common.Address { return common.BytesToAddress(common.FromHex(a)) }
 func mixAddr(a string) (*common.MixedcaseAddress, error) {
 	return common.NewMixedcaseAddressFromString(a)
 }

signer/rules/rules.go

@@ -46,16 +46,16 @@ func consoleOutput(call otto.FunctionCall) otto.Value {
 	return otto.Value{}
 }
 
-// rulesetUI provides an implementation of SignerUI that evaluates a javascript
+// rulesetUI provides an implementation of UIClientAPI that evaluates a javascript
 // file for each defined UI-method
 type rulesetUI struct {
-	next        core.SignerUI // The next handler, for manual processing
+	next        core.UIClientAPI // The next handler, for manual processing
 	storage     storage.Storage
 	credentials storage.Storage
 	jsRules     string // The rules to use
 }
 
-func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) {
+func NewRuleEvaluator(next core.UIClientAPI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) {
 	c := &rulesetUI{
 		next:        next,
 		storage:     jsbackend,
@@ -65,6 +65,9 @@ func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.
 
 	return c, nil
 }
+func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) {
+	// TODO, make it possible to query from js
+}
 
 func (r *rulesetUI) Init(javascriptRules string) error {
 	r.jsRules = javascriptRules

signer/rules/rules_test.go

@@ -77,12 +77,10 @@ type alwaysDenyUI struct{}
 func (alwaysDenyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) {
 	return core.UserInputResponse{}, nil
 }
-
+func (alwaysDenyUI) RegisterUIServer(api *core.UIServerAPI) {
-func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) {
 }
 
-func (alwaysDenyUI) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) {
+func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) {
-	return core.PasswordResponse{}, nil
 }
 
 func (alwaysDenyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {
@@ -133,11 +131,11 @@ func initRuleEngine(js string) (*rulesetUI, error) {
 }
 
 func TestListRequest(t *testing.T) {
-	accs := make([]core.Account, 5)
+	accs := make([]accounts.Account, 5)
 
 	for i := range accs {
 		addr := fmt.Sprintf("000000000000000000000000000000000000000%x", i)
-		acc := core.Account{
+		acc := accounts.Account{
 			Address: common.BytesToAddress(common.Hex2Bytes(addr)),
 			URL:     accounts.URL{Scheme: "test", Path: fmt.Sprintf("acc-%d", i)},
 		}
@@ -208,6 +206,10 @@ type dummyUI struct {
 	calls []string
 }
 
+func (d *dummyUI) RegisterUIServer(api *core.UIServerAPI) {
+	panic("implement me")
+}
+
 func (d *dummyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) {
 	d.calls = append(d.calls, "OnInputRequired")
 	return core.UserInputResponse{}, nil
@@ -255,10 +257,6 @@ func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
 	d.calls = append(d.calls, "OnApprovedTx")
 }
 
-func (d *dummyUI) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) {
-	return core.PasswordResponse{}, nil
-}
-
 func (d *dummyUI) OnSignerStartup(info core.StartupInfo) {
 }
 
@@ -531,12 +529,10 @@ func (d *dontCallMe) OnInputRequired(info core.UserInputRequest) (core.UserInput
 	d.t.Fatalf("Did not expect next-handler to be called")
 	return core.UserInputResponse{}, nil
 }
-
+func (d *dontCallMe) RegisterUIServer(api *core.UIServerAPI) {
-func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) {
 }
 
-func (d *dontCallMe) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) {
+func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) {
-	return core.PasswordResponse{}, nil
 }
 
 func (d *dontCallMe) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {

swarm/docker/Dockerfile

@@ -10,14 +10,23 @@ RUN mkdir -p $GOPATH/src/github.com/ethereum && \
     git checkout ${VERSION} && \
     go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm && \
     go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/swarm-smoke && \
-    go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth && \
+    go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/global-store && \
-    cp $GOPATH/bin/swarm /swarm && cp $GOPATH/bin/geth /geth && cp $GOPATH/bin/swarm-smoke /swarm-smoke
+    go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth
 
 
-# Release image with the required binaries and scripts
+FROM alpine:3.8 as swarm-smoke
-FROM alpine:3.8
 WORKDIR /
-COPY --from=builder /swarm /geth /swarm-smoke /
+COPY --from=builder /go/bin/swarm-smoke /
-ADD run.sh /run.sh
 ADD run-smoke.sh /run-smoke.sh
+ENTRYPOINT ["/run-smoke.sh"]
+
+FROM alpine:3.8 as swarm-global-store
+WORKDIR /
+COPY --from=builder /go/bin/global-store /
+ENTRYPOINT ["/global-store"]
+
+FROM alpine:3.8 as swarm
+WORKDIR /
+COPY --from=builder /go/bin/swarm /go/bin/geth /
+ADD run.sh /run.sh
 ENTRYPOINT ["/run.sh"]

swarm/network/kademlia.go

@@ -353,6 +353,9 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() {
 // Not receiving from the returned channel will block Register function
 // when address count value changes.
 func (k *Kademlia) AddrCountC() <-chan int {
+	k.lock.Lock()
+	defer k.lock.Unlock()
+
 	if k.addrCountC == nil {
 		k.addrCountC = make(chan int)
 	}
@@ -626,6 +629,7 @@ func (k *Kademlia) string() string {
 // TODO move to separate testing tools file
 type PeerPot struct {
 	NNSet       [][]byte
+	PeersPerBin []int
 }
 
 // NewPeerPotMap creates a map of pot record of *BzzAddr with keys
@@ -651,6 +655,7 @@ func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot {
 
 		// all nn-peers
 		var nns [][]byte
+		peersPerBin := make([]int, depth)
 
 		// iterate through the neighbours, going from the deepest to the shallowest
 		np.EachNeighbour(a, Pof, func(val pot.Val, po int) bool {
@@ -664,14 +669,18 @@ func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot {
 			// a neighbor is any peer in or deeper than the depth
 			if po >= depth {
 				nns = append(nns, addr)
-				return true
+			} else {
+				// for peers < depth, we just count the number in each bin
+				// the bin is the index of the slice
+				peersPerBin[po]++
 			}
-			return false
+			return true
 		})
 
-		log.Trace(fmt.Sprintf("%x PeerPotMap NNS: %s", addrs[i][:4], LogAddrs(nns)))
+		log.Trace(fmt.Sprintf("%x PeerPotMap NNS: %s, peersPerBin", addrs[i][:4], LogAddrs(nns)))
 		ppmap[common.Bytes2Hex(a)] = &PeerPot{
 			NNSet:       nns,
+			PeersPerBin: peersPerBin,
 		}
 	}
 	return ppmap
@@ -695,6 +704,39 @@ func (k *Kademlia) saturation() int {
 	return prev
 }
 
+// isSaturated returns true if the kademlia is considered saturated, or false if not.
+// It checks this by checking an array of ints called unsaturatedBins; each item in that array corresponds
+// to the bin which is unsaturated (number of connections < k.MinBinSize).
+// The bin is considered unsaturated only if there are actual peers in that PeerPot's bin (peersPerBin)
+// (if there is no peer for a given bin, then no connection could ever be established;
+// in a God's view this is relevant as no more peers will ever appear on that bin)
+func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool {
+	// depth could be calculated from k but as this is called from `GetHealthInfo()`,
+	// the depth has already been calculated so we can require it as a parameter
+
+	// early check for depth
+	if depth != len(peersPerBin) {
+		return false
+	}
+	unsaturatedBins := make([]int, 0)
+	k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool {
+
+		if po >= depth {
+			return false
+		}
+		log.Trace("peers per bin", "peersPerBin[po]", peersPerBin[po], "po", po)
+		// if there are actually peers in the PeerPot who can fulfill k.MinBinSize
+		if size < k.MinBinSize && size < peersPerBin[po] {
+			log.Trace("connections for po", "po", po, "size", size)
+			unsaturatedBins = append(unsaturatedBins, po)
+		}
+		return true
+	})
+
+	log.Trace("list of unsaturated bins", "unsaturatedBins", unsaturatedBins)
+	return len(unsaturatedBins) == 0
+}
+
 // knowNeighbours tests if all neighbours in the peerpot
 // are found among the peers known to the kademlia
 // It is used in Healthy function for testing only
@@ -777,11 +819,13 @@ type Health struct {
 	ConnectNN        bool     // whether node is connected to all its neighbours
 	CountConnectNN   int      // amount of neighbours connected to
 	MissingConnectNN [][]byte // which neighbours we should have been connected to but we're not
-	Saturated        bool     // whether we are connected to all the peers we would have liked to
+	// Saturated: if in all bins < depth number of connections >= MinBinsize or,
+	// if number of connections < MinBinSize, to the number of available peers in that bin
+	Saturated bool
 	Hive      string
 }
 
-// Healthy reports the health state of the kademlia connectivity
+// GetHealthInfo reports the health state of the kademlia connectivity
 //
 // The PeerPot argument provides an all-knowing view of the network
 // The resulting Health object is a result of comparisons between
@@ -789,7 +833,7 @@ type Health struct {
 // what SHOULD it have been when we take all we know about the network into consideration.
 //
 // used for testing only
-func (k *Kademlia) Healthy(pp *PeerPot) *Health {
+func (k *Kademlia) GetHealthInfo(pp *PeerPot) *Health {
 	k.lock.RLock()
 	defer k.lock.RUnlock()
 	if len(pp.NNSet) < k.NeighbourhoodSize {
@@ -798,7 +842,10 @@ func (k *Kademlia) Healthy(pp *PeerPot) *Health {
 	gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet)
 	knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet)
 	depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base)
-	saturated := k.saturation() < depth
+
+	// check saturation
+	saturated := k.isSaturated(pp.PeersPerBin, depth)
+
 	log.Trace(fmt.Sprintf("%08x: healthy: knowNNs: %v, gotNNs: %v, saturated: %v\n", k.base, knownn, gotnn, saturated))
 	return &Health{
 		KnowNN:           knownn,
@@ -811,3 +858,13 @@ func (k *Kademlia) Healthy(pp *PeerPot) *Health {
 		Hive:             k.string(),
 	}
 }
+
+// Healthy return the strict interpretation of `Healthy` given a `Health` struct
+// definition of strict health: all conditions must be true:
+// - we at least know one peer
+// - we know all neighbors
+// - we are connected to all known neighbors
+// - it is saturated
+func (h *Health) Healthy() bool {
+	return h.KnowNN && h.ConnectNN && h.CountKnowNN > 0 && h.Saturated
+}

swarm/network/kademlia_test.go

@@ -168,6 +168,46 @@ func TestNeighbourhoodDepth(t *testing.T) {
 	testNum++
 }
 
+// TestHighMinBinSize tests that the saturation function also works
+// if MinBinSize is > 2, the connection count is < k.MinBinSize
+// and there are more peers available than connected
+func TestHighMinBinSize(t *testing.T) {
+	// a function to test for different MinBinSize values
+	testKad := func(minBinSize int) {
+		// create a test kademlia
+		tk := newTestKademlia(t, "11111111")
+		// set its MinBinSize to desired value
+		tk.KadParams.MinBinSize = minBinSize
+
+		// add a couple of peers (so we have NN and depth)
+		tk.On("00000000") // bin 0
+		tk.On("11100000") // bin 3
+		tk.On("11110000") // bin 4
+
+		first := "10000000" // add a first peer at bin 1
+		tk.Register(first)  // register it
+		// we now have one registered peer at bin 1;
+		// iterate and connect one peer at each iteration;
+		// should be unhealthy until at minBinSize - 1
+		// we connect the unconnected but registered peer
+		for i := 1; i < minBinSize; i++ {
+			peer := fmt.Sprintf("1000%b", 8|i)
+			tk.On(peer)
+			if i == minBinSize-1 {
+				tk.On(first)
+				tk.checkHealth(true)
+				return
+			}
+			tk.checkHealth(false)
+		}
+	}
+	// test MinBinSizes of 3 to 5
+	testMinBinSizes := []int{3, 4, 5}
+	for _, k := range testMinBinSizes {
+		testKad(k)
+	}
+}
+
 // TestHealthStrict tests the simplest definition of health
 // Which means whether we are connected to all neighbors we know of
 func TestHealthStrict(t *testing.T) {
@@ -176,60 +216,116 @@ func TestHealthStrict(t *testing.T) {
 	// no peers
 	// unhealthy (and lonely)
 	tk := newTestKademlia(t, "11111111")
-	tk.checkHealth(false, false)
+	tk.checkHealth(false)
 
 	// know one peer but not connected
 	// unhealthy
 	tk.Register("11100000")
-	tk.checkHealth(false, false)
+	tk.checkHealth(false)
 
 	// know one peer and connected
-	// healthy
+	// unhealthy: not saturated
 	tk.On("11100000")
-	tk.checkHealth(true, false)
+	tk.checkHealth(true)
 
 	// know two peers, only one connected
 	// unhealthy
 	tk.Register("11111100")
-	tk.checkHealth(false, false)
+	tk.checkHealth(false)
 
 	// know two peers and connected to both
 	// healthy
 	tk.On("11111100")
-	tk.checkHealth(true, false)
+	tk.checkHealth(true)
 
 	// know three peers, connected to the two deepest
 	// healthy
 	tk.Register("00000000")
-	tk.checkHealth(true, false)
+	tk.checkHealth(false)
 
 	// know three peers, connected to all three
 	// healthy
 	tk.On("00000000")
-	tk.checkHealth(true, false)
+	tk.checkHealth(true)
 
 	// add fourth peer deeper than current depth
 	// unhealthy
 	tk.Register("11110000")
-	tk.checkHealth(false, false)
+	tk.checkHealth(false)
 
 	// connected to three deepest peers
 	// healthy
 	tk.On("11110000")
-	tk.checkHealth(true, false)
+	tk.checkHealth(true)
 
 	// add additional peer in same bin as deepest peer
 	// unhealthy
 	tk.Register("11111101")
-	tk.checkHealth(false, false)
+	tk.checkHealth(false)
 
 	// four deepest of five peers connected
 	// healthy
 	tk.On("11111101")
-	tk.checkHealth(true, false)
+	tk.checkHealth(true)
+
+	// add additional peer in bin 0
+	// unhealthy: unsaturated bin 0, 2 known but 1 connected
+	tk.Register("00000001")
+	tk.checkHealth(false)
+
+	// Connect second in bin 0
+	// healthy
+	tk.On("00000001")
+	tk.checkHealth(true)
+
+	// add peer in bin 1
+	// unhealthy, as it is known but not connected
+	tk.Register("10000000")
+	tk.checkHealth(false)
+
+	// connect  peer in bin 1
+	// depth change, is now 1
+	// healthy, 1 peer in bin 1 known and connected
+	tk.On("10000000")
+	tk.checkHealth(true)
+
+	// add second peer in bin 1
+	// unhealthy, as it is known but not connected
+	tk.Register("10000001")
+	tk.checkHealth(false)
+
+	// connect second peer in bin 1
+	// healthy,
+	tk.On("10000001")
+	tk.checkHealth(true)
+
+	// connect third peer in bin 1
+	// healthy,
+	tk.On("10000011")
+	tk.checkHealth(true)
+
+	// add peer in bin 2
+	// unhealthy, no depth change
+	tk.Register("11000000")
+	tk.checkHealth(false)
+
+	// connect peer in bin 2
+	// depth change - as we already have peers in bin 3 and 4,
+	// we have contiguous bins, no bin < po 5 is empty -> depth 5
+	// healthy, every bin < depth has the max available peers,
+	// even if they are < MinBinSize
+	tk.On("11000000")
+	tk.checkHealth(true)
+
+	// add peer in bin 2
+	// unhealthy, peer bin is below depth 5 but
+	// has more available peers (2) than connected ones (1)
+	// --> unsaturated
+	tk.Register("11000011")
+	tk.checkHealth(false)
 }
 
-func (tk *testKademlia) checkHealth(expectHealthy bool, expectSaturation bool) {
+func (tk *testKademlia) checkHealth(expectHealthy bool) {
 	tk.t.Helper()
 	kid := common.Bytes2Hex(tk.BaseAddr())
 	addrs := [][]byte{tk.BaseAddr()}
@@ -239,13 +335,13 @@ func (tk *testKademlia) checkHealth(expectHealthy bool, expectSaturation bool) {
 	})
 
 	pp := NewPeerPotMap(tk.NeighbourhoodSize, addrs)
-	healthParams := tk.Healthy(pp[kid])
+	healthParams := tk.GetHealthInfo(pp[kid])
 
 	// definition of health, all conditions but be true:
 	// - we at least know one peer
 	// - we know all neighbors
 	// - we are connected to all known neighbors
-	health := healthParams.KnowNN && healthParams.ConnectNN && healthParams.CountKnowNN > 0
+	health := healthParams.Healthy()
 	if expectHealthy != health {
 		tk.t.Fatalf("expected kademlia health %v, is %v\n%v", expectHealthy, health, tk.String())
 	}

swarm/network/simulation/kademlia.go

@@ -64,7 +64,7 @@ func (s *Simulation) WaitTillHealthy(ctx context.Context) (ill map[enode.ID]*net
 				addr := common.Bytes2Hex(k.BaseAddr())
 				pp := ppmap[addr]
 				//call Healthy RPC
-				h := k.Healthy(pp)
+				h := k.GetHealthInfo(pp)
 				//print info
 				log.Debug(k.String())
 				log.Debug("kademlia", "connectNN", h.ConnectNN, "knowNN", h.KnowNN)

swarm/network/simulation/node.go

@@ -44,7 +44,7 @@ func (s *Simulation) NodeIDs() (ids []enode.ID) {
 func (s *Simulation) UpNodeIDs() (ids []enode.ID) {
 	nodes := s.Net.GetNodes()
 	for _, node := range nodes {
-		if node.Up {
+		if node.Up() {
 			ids = append(ids, node.ID())
 		}
 	}
@@ -55,7 +55,7 @@ func (s *Simulation) UpNodeIDs() (ids []enode.ID) {
 func (s *Simulation) DownNodeIDs() (ids []enode.ID) {
 	nodes := s.Net.GetNodes()
 	for _, node := range nodes {
-		if !node.Up {
+		if !node.Up() {
 			ids = append(ids, node.ID())
 		}
 	}

swarm/network/simulation/node_test.go

@@ -54,7 +54,7 @@ func TestUpDownNodeIDs(t *testing.T) {
 	gotIDs = sim.UpNodeIDs()
 
 	for _, id := range gotIDs {
-		if !sim.Net.GetNode(id).Up {
+		if !sim.Net.GetNode(id).Up() {
 			t.Errorf("node %s should not be down", id)
 		}
 	}
@@ -66,7 +66,7 @@ func TestUpDownNodeIDs(t *testing.T) {
 	gotIDs = sim.DownNodeIDs()
 
 	for _, id := range gotIDs {
-		if sim.Net.GetNode(id).Up {
+		if sim.Net.GetNode(id).Up() {
 			t.Errorf("node %s should not be up", id)
 		}
 	}
@@ -112,7 +112,7 @@ func TestAddNode(t *testing.T) {
 		t.Fatal("node not found")
 	}
 
-	if !n.Up {
+	if !n.Up() {
 		t.Error("node not started")
 	}
 }
@@ -327,7 +327,7 @@ func TestStartStopNode(t *testing.T) {
 	if n == nil {
 		t.Fatal("node not found")
 	}
-	if !n.Up {
+	if !n.Up() {
 		t.Error("node not started")
 	}
 
@@ -335,26 +335,17 @@ func TestStartStopNode(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if n.Up {
+	if n.Up() {
 		t.Error("node not stopped")
 	}
 
-	// Sleep here to ensure that Network.watchPeerEvents defer function
+	waitForPeerEventPropagation()
-	// has set the `node.Up = false` before we start the node again.
-	// p2p/simulations/network.go:215
-	//
-	// The same node is stopped and started again, and upon start
-	// watchPeerEvents is started in a goroutine. If the node is stopped
-	// and then very quickly started, that goroutine may be scheduled later
-	// then start and force `node.Up = false` in its defer function.
-	// This will make this test unreliable.
-	time.Sleep(time.Second)
 
 	err = sim.StartNode(id)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !n.Up {
+	if !n.Up() {
 		t.Error("node not started")
 	}
 }
@@ -377,7 +368,7 @@ func TestStartStopRandomNode(t *testing.T) {
 	if n == nil {
 		t.Fatal("node not found")
 	}
-	if n.Up {
+	if n.Up() {
 		t.Error("node not stopped")
 	}
 
@@ -386,16 +377,7 @@ func TestStartStopRandomNode(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Sleep here to ensure that Network.watchPeerEvents defer function
+	waitForPeerEventPropagation()
-	// has set the `node.Up = false` before we start the node again.
-	// p2p/simulations/network.go:215
-	//
-	// The same node is stopped and started again, and upon start
-	// watchPeerEvents is started in a goroutine. If the node is stopped
-	// and then very quickly started, that goroutine may be scheduled later
-	// then start and force `node.Up = false` in its defer function.
-	// This will make this test unreliable.
-	time.Sleep(time.Second)
 
 	idStarted, err := sim.StartRandomNode()
 	if err != nil {
@@ -426,21 +408,12 @@ func TestStartStopRandomNodes(t *testing.T) {
 		if n == nil {
 			t.Fatal("node not found")
 		}
-		if n.Up {
+		if n.Up() {
 			t.Error("node not stopped")
 		}
 	}
 
-	// Sleep here to ensure that Network.watchPeerEvents defer function
+	waitForPeerEventPropagation()
-	// has set the `node.Up = false` before we start the node again.
-	// p2p/simulations/network.go:215
-	//
-	// The same node is stopped and started again, and upon start
-	// watchPeerEvents is started in a goroutine. If the node is stopped
-	// and then very quickly started, that goroutine may be scheduled later
-	// then start and force `node.Up = false` in its defer function.
-	// This will make this test unreliable.
-	time.Sleep(time.Second)
 
 	ids, err = sim.StartRandomNodes(2)
 	if err != nil {
@@ -452,8 +425,20 @@ func TestStartStopRandomNodes(t *testing.T) {
 		if n == nil {
 			t.Fatal("node not found")
 		}
-		if !n.Up {
+		if !n.Up() {
 			t.Error("node not started")
 		}
 	}
 }
+
+func waitForPeerEventPropagation() {
+	// Sleep here to ensure that Network.watchPeerEvents defer function
+	// has set the `node.Up() = false` before we start the node again.
+	//
+	// The same node is stopped and started again, and upon start
+	// watchPeerEvents is started in a goroutine. If the node is stopped
+	// and then very quickly started, that goroutine may be scheduled later
+	// then start and force `node.Up() = false` in its defer function.
+	// This will make this test unreliable.
+	time.Sleep(1 * time.Second)
+}

swarm/network/simulation/service.go

@@ -52,7 +52,7 @@ func (s *Simulation) Services(name string) (services map[enode.ID]node.Service)
 	nodes := s.Net.GetNodes()
 	services = make(map[enode.ID]node.Service)
 	for _, node := range nodes {
-		if !node.Up {
+		if !node.Up() {
 			continue
 		}
 		simNode, ok := node.Node.(*adapters.SimNode)

swarm/network/simulation/simulation_test.go

@@ -124,7 +124,7 @@ func TestClose(t *testing.T) {
 
 	var upNodeCount int
 	for _, n := range sim.Net.GetNodes() {
-		if n.Up {
+		if n.Up() {
 			upNodeCount++
 		}
 	}
@@ -140,7 +140,7 @@ func TestClose(t *testing.T) {
 
 	upNodeCount = 0
 	for _, n := range sim.Net.GetNodes() {
-		if n.Up {
+		if n.Up() {
 			upNodeCount++
 		}
 	}

swarm/network/simulations/discovery/discovery_test.go

@@ -267,7 +267,7 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
 		}
 
 		healthy := &network.Health{}
-		if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
+		if err := client.Call(&healthy, "hive_getHealthInfo", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
 			return false, fmt.Errorf("error getting node health: %s", err)
 		}
 		log.Debug(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive))
@@ -352,7 +352,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt
 				healthy := &network.Health{}
 				addr := id.String()
 				ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
-				if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
+				if err := client.Call(&healthy, "hive_getHealthInfo", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
 					return fmt.Errorf("error getting node health: %s", err)
 				}
 
@@ -422,7 +422,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt
 		healthy := &network.Health{}
 		ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
 
-		if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
+		if err := client.Call(&healthy, "hive_getHealthInfo", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
 			return false, fmt.Errorf("error getting node health: %s", err)
 		}
 		log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v", id, healthy.ConnectNN, healthy.KnowNN))

swarm/network/simulations/overlay_test.go

@@ -178,7 +178,7 @@ func watchSimEvents(net *simulations.Network, ctx context.Context, trigger chan
 		case ev := <-events:
 			//only catch node up events
 			if ev.Type == simulations.EventTypeNode {
-				if ev.Node.Up {
+				if ev.Node.Up() {
 					log.Debug("got node up event", "event", ev, "node", ev.Node.Config.ID)
 					select {
 					case trigger <- ev.Node.Config.ID:

swarm/network/stream/common_test.go

@@ -151,7 +151,7 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
 	// temp datadir
 	datadir, err := ioutil.TempDir("", "streamer")
 	if err != nil {
-		return nil, nil, nil, func() {}, err
+		return nil, nil, nil, nil, err
 	}
 	removeDataDir := func() {
 		os.RemoveAll(datadir)
@@ -163,12 +163,14 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
 
 	localStore, err := storage.NewTestLocalStoreForAddr(params)
 	if err != nil {
-		return nil, nil, nil, removeDataDir, err
+		removeDataDir()
+		return nil, nil, nil, nil, err
 	}
 
 	netStore, err := storage.NewNetStore(localStore, nil)
 	if err != nil {
-		return nil, nil, nil, removeDataDir, err
+		removeDataDir()
+		return nil, nil, nil, nil, err
 	}
 
 	delivery := NewDelivery(to, netStore)
@@ -180,8 +182,9 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
 	}
 	protocolTester := p2ptest.NewProtocolTester(addr.ID(), 1, streamer.runProtocol)
 
-	err = waitForPeers(streamer, 1*time.Second, 1)
+	err = waitForPeers(streamer, 10*time.Second, 1)
 	if err != nil {
+		teardown()
 		return nil, nil, nil, nil, errors.New("timeout: peer is not created")
 	}
 
@@ -317,3 +320,54 @@ func createTestLocalStorageForID(id enode.ID, addr *network.BzzAddr) (storage.Ch
 	}
 	return store, datadir, nil
 }
+
+// watchDisconnections receives simulation peer events in a new goroutine and sets atomic value
+// disconnected to true in case of a disconnect event.
+func watchDisconnections(ctx context.Context, sim *simulation.Simulation) (disconnected *boolean) {
+	log.Debug("Watching for disconnections")
+	disconnections := sim.PeerEvents(
+		ctx,
+		sim.NodeIDs(),
+		simulation.NewPeerEventsFilter().Drop(),
+	)
+	disconnected = new(boolean)
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case d := <-disconnections:
+				if d.Error != nil {
+					log.Error("peer drop event error", "node", d.NodeID, "peer", d.PeerID, "err", d.Error)
+				} else {
+					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
+				}
+				disconnected.set(true)
+			}
+		}
+	}()
+	return disconnected
+}
+
+// boolean is used to concurrently set
+// and read a boolean value.
+type boolean struct {
+	v  bool
+	mu sync.RWMutex
+}
+
+// set sets the value.
+func (b *boolean) set(v bool) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	b.v = v
+}
+
+// bool reads the value.
+func (b *boolean) bool() bool {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	return b.v
+}

swarm/network/stream/delivery_test.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -48,10 +47,10 @@ func TestStreamerRetrieveRequest(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, streamer, _, teardown, err := newStreamerTester(regOpts)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -100,10 +99,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
 		Retrieval: RetrievalEnabled,
 		Syncing:   SyncingDisabled, //do no syncing
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -172,10 +171,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
 		Retrieval: RetrievalEnabled,
 		Syncing:   SyncingDisabled,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -362,10 +361,10 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
 		Retrieval: RetrievalDisabled,
 		Syncing:   SyncingDisabled,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) {
 		return &testClient{
@@ -454,6 +453,8 @@ func TestDeliveryFromNodes(t *testing.T) {
 }
 
 func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) {
+	t.Helper()
+	t.Run(fmt.Sprintf("testDeliveryFromNodes_%d_%d_skipCheck_%t", nodes, chunkCount, skipCheck), func(t *testing.T) {
 		sim := simulation.New(map[string]simulation.ServiceFunc{
 			"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
 				addr, netStore, delivery, clean, err := newNetStoreAndDelivery(ctx, bucket)
@@ -485,7 +486,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 		}
 
 		log.Info("Starting simulation")
-	ctx := context.Background()
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
 		result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
 			nodeIDs := sim.UpNodeIDs()
 			//determine the pivot node to be the first node of the simulation
@@ -548,28 +550,11 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 				retErrC <- err
 			}()
 
-		log.Debug("Watching for disconnections")
+			disconnected := watchDisconnections(ctx, sim)
-		disconnections := sim.PeerEvents(
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
-		var disconnected atomic.Value
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 			defer func() {
-			if err != nil {
+				if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
 					err = errors.New("disconnect events received")
 				}
-			}
 			}()
 
 			//finally check that the pivot node gets all chunks via the root hash
@@ -589,7 +574,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 				return fmt.Errorf("Test failed, chunks not available on all nodes")
 			}
 			if err := <-retErrC; err != nil {
-			t.Fatalf("requesting chunks: %v", err)
+				return fmt.Errorf("requesting chunks: %v", err)
 			}
 			log.Debug("Test terminated successfully")
 			return nil
@@ -597,6 +582,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 		if result.Error != nil {
 			t.Fatal(result.Error)
 		}
+	})
 }
 
 func BenchmarkDeliveryFromNodesWithoutCheck(b *testing.B) {
@@ -657,21 +643,22 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 		b.Fatal(err)
 	}
 
-	ctx := context.Background()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
 		nodeIDs := sim.UpNodeIDs()
 		node := nodeIDs[len(nodeIDs)-1]
 
 		item, ok := sim.NodeItem(node, bucketKeyFileStore)
 		if !ok {
-			b.Fatal("No filestore")
+			return errors.New("No filestore")
 		}
 		remoteFileStore := item.(*storage.FileStore)
 
 		pivotNode := nodeIDs[0]
 		item, ok = sim.NodeItem(pivotNode, bucketKeyNetStore)
 		if !ok {
-			b.Fatal("No filestore")
+			return errors.New("No filestore")
 		}
 		netStore := item.(*storage.NetStore)
 
@@ -679,27 +666,11 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 			return err
 		}
 
-		disconnections := sim.PeerEvents(
+		disconnected := watchDisconnections(ctx, sim)
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
-		var disconnected atomic.Value
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
 				err = errors.New("disconnect events received")
 			}
-			}
 		}()
 		// benchmark loop
 		b.ResetTimer()
@@ -713,12 +684,12 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 				ctx := context.TODO()
 				hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false)
 				if err != nil {
-					b.Fatalf("expected no error. got %v", err)
+					return fmt.Errorf("store: %v", err)
 				}
 				// wait until all chunks stored
 				err = wait(ctx)
 				if err != nil {
-					b.Fatalf("expected no error. got %v", err)
+					return fmt.Errorf("wait store: %v", err)
 				}
 				// collect the hashes
 				hashes[i] = hash
@@ -754,10 +725,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 				break Loop
 			}
 		}
-		if err != nil {
+		return err
-			b.Fatal(err)
-		}
-		return nil
 	})
 	if result.Error != nil {
 		b.Fatal(result.Error)

swarm/network/stream/intervals_test.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -118,13 +117,11 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 
 		_, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
 		if err != nil {
-			log.Error("Store error: %v", "err", err)
+			return fmt.Errorf("store: %v", err)
-			t.Fatal(err)
 		}
 		err = wait(ctx)
 		if err != nil {
-			log.Error("Wait error: %v", "err", err)
+			return fmt.Errorf("wait store: %v", err)
-			t.Fatal(err)
 		}
 
 		item, ok = sim.NodeItem(checker, bucketKeyRegistry)
@@ -136,33 +133,16 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 		liveErrC := make(chan error)
 		historyErrC := make(chan error)
 
-		log.Debug("Watching for disconnections")
-		disconnections := sim.PeerEvents(
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
 		err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
 		if err != nil {
 			return err
 		}
 
-		var disconnected atomic.Value
+		disconnected := watchDisconnections(ctx, sim)
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
 				err = errors.New("disconnect events received")
 			}
-			}
 		}()
 
 		go func() {

swarm/network/stream/lightnode_test.go

@@ -29,10 +29,10 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -68,10 +68,10 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -112,10 +112,10 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) {
 		Syncing:   SyncingRegisterOnly,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -157,10 +157,10 @@ func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 

swarm/network/stream/norace_test.go

@@ -0,0 +1,24 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !race
+
+package stream
+
+// Provide a flag to reduce the scope of tests when running them
+// with race detector. Some of the tests are doing a lot of allocations
+// on the heap, and race detector uses much more memory to track them.
+const raceTest = false

swarm/network/stream/race_test.go

@@ -0,0 +1,23 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build race
+
+package stream
+
+// Reduce the scope of some tests when running with race detector,
+// as it raises the memory consumption significantly.
+const raceTest = true

swarm/network/stream/snapshot_retrieval_test.go

@@ -74,7 +74,7 @@ func TestRetrieval(t *testing.T) {
 	//if nodes/chunks have been provided via commandline,
 	//run the tests with these values
 	if *nodes != 0 && *chunks != 0 {
-		err := runRetrievalTest(*chunks, *nodes)
+		err := runRetrievalTest(t, *chunks, *nodes)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -93,10 +93,12 @@ func TestRetrieval(t *testing.T) {
 		}
 		for _, n := range nodeCnt {
 			for _, c := range chnkCnt {
-				err := runRetrievalTest(c, n)
+				t.Run(fmt.Sprintf("TestRetrieval_%d_%d", n, c), func(t *testing.T) {
+					err := runRetrievalTest(t, c, n)
 					if err != nil {
 						t.Fatal(err)
 					}
+				})
 			}
 		}
 	}
@@ -225,7 +227,8 @@ simulation's `action` function.
 
 The snapshot should have 'streamer' in its service list.
 */
-func runRetrievalTest(chunkCount int, nodeCount int) error {
+func runRetrievalTest(t *testing.T, chunkCount int, nodeCount int) error {
+	t.Helper()
 	sim := simulation.New(retrievalSimServiceMap)
 	defer sim.Close()
 

swarm/network/stream/snapshot_sync_test.go

@@ -17,11 +17,12 @@ package stream
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"runtime"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -92,6 +93,15 @@ func TestSyncingViaGlobalSync(t *testing.T) {
 		if *longrunning {
 			chnkCnt = []int{1, 8, 32, 256, 1024}
 			nodeCnt = []int{16, 32, 64, 128, 256}
+		} else if raceTest {
+			// TestSyncingViaGlobalSync allocates a lot of memory
+			// with race detector. By reducing the number of chunks
+			// and nodes, memory consumption is lower and data races
+			// are still checked, while correctness of syncing is
+			// tested with more chunks and nodes in regular (!race)
+			// tests.
+			chnkCnt = []int{4}
+			nodeCnt = []int{16}
 		} else {
 			//default test
 			chnkCnt = []int{4, 32}
@@ -113,7 +123,23 @@ var simServiceMap = map[string]simulation.ServiceFunc{
 			return nil, nil, err
 		}
 
-		r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+		var dir string
+		var store *state.DBStore
+		if raceTest {
+			// Use on-disk DBStore to reduce memory consumption in race tests.
+			dir, err = ioutil.TempDir("", "swarm-stream-")
+			if err != nil {
+				return nil, nil, err
+			}
+			store, err = state.NewDBStore(dir)
+			if err != nil {
+				return nil, nil, err
+			}
+		} else {
+			store = state.NewInmemoryStore()
+		}
+
+		r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{
 			Retrieval:       RetrievalDisabled,
 			Syncing:         SyncingAutoSubscribe,
 			SyncUpdateDelay: 3 * time.Second,
@@ -156,36 +182,24 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
 		t.Fatal(err)
 	}
 
-	disconnections := sim.PeerEvents(
-		context.Background(),
-		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Drop(),
-	)
-
-	var disconnected atomic.Value
-	go func() {
-		for d := range disconnections {
-			if d.Error != nil {
-				log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-				disconnected.Store(true)
-			}
-		}
-	}()
-
 	result := runSim(conf, ctx, sim, chunkCount)
 
 	if result.Error != nil {
 		t.Fatal(result.Error)
 	}
-	if yes, ok := disconnected.Load().(bool); ok && yes {
-		t.Fatal("disconnect events received")
-	}
 	log.Info("Simulation ended")
 }
 
 func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulation, chunkCount int) simulation.Result {
 
-	return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+	return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
+		disconnected := watchDisconnections(ctx, sim)
+		defer func() {
+			if err != nil && disconnected.bool() {
+				err = errors.New("disconnect events received")
+			}
+		}()
+
 		nodeIDs := sim.UpNodeIDs()
 		for _, n := range nodeIDs {
 			//get the kademlia overlay address from this ID

swarm/network/stream/stream.go

@@ -939,16 +939,22 @@ It returns a map of node IDs with an array of string representations of Stream o
 func (api *API) GetPeerSubscriptions() map[string][]string {
 	//create the empty map
 	pstreams := make(map[string][]string)
+
 	//iterate all streamer peers
+	api.streamer.peersMu.RLock()
+	defer api.streamer.peersMu.RUnlock()
+
 	for id, p := range api.streamer.peers {
 		var streams []string
 		//every peer has a map of stream servers
 		//every stream server represents a subscription
+		p.serverMu.RLock()
 		for s := range p.servers {
 			//append the string representation of the stream
 			//to the list for this peer
 			streams = append(streams, s.String())
 		}
+		p.serverMu.RUnlock()
 		//set the array of stream servers to the map
 		pstreams[id.String()] = streams
 	}

swarm/network/stream/streamer_test.go

@@ -41,10 +41,10 @@ import (
 
 func TestStreamerSubscribe(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 	err = streamer.Subscribe(tester.Nodes[0].ID(), stream, NewRange(0, 0), Top)
@@ -55,10 +55,10 @@ func TestStreamerSubscribe(t *testing.T) {
 
 func TestStreamerRequestSubscription(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", false)
 	err = streamer.RequestSubscription(tester.Nodes[0].ID(), stream, &Range{}, Top)
@@ -146,10 +146,10 @@ func (self *testServer) Close() {
 
 func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) {
 		return newTestClient(t), nil
@@ -239,10 +239,10 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", false)
 
@@ -306,10 +306,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -372,10 +372,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 0), nil
@@ -416,10 +416,10 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -479,10 +479,10 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) {
 
 func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -544,10 +544,10 @@ func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) {
 
 func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -643,10 +643,10 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
 
 func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 10), nil
@@ -780,10 +780,10 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) {
 		Syncing:        SyncingDisabled,
 		MaxPeerServers: maxPeerServers,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 0), nil
@@ -854,10 +854,10 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{
 		MaxPeerServers: maxPeerServers,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 0), nil
@@ -940,10 +940,10 @@ func TestHasPriceImplementation(t *testing.T) {
 		Retrieval: RetrievalDisabled,
 		Syncing:   SyncingDisabled,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	if r.prices == nil {
 		t.Fatal("No prices implementation available for the stream protocol")
@@ -1177,6 +1177,7 @@ starts the simulation, waits for SyncUpdateDelay in order to kick off
 stream registration, then tests that there are subscriptions.
 */
 func TestGetSubscriptionsRPC(t *testing.T) {
+
 	// arbitrarily set to 4
 	nodeCount := 4
 	// run with more nodes if `longrunning` flag is set
@@ -1188,19 +1189,16 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 	// holds the msg code for SubscribeMsg
 	var subscribeMsgCode uint64
 	var ok bool
-	var expectedMsgCount = 0
+	var expectedMsgCount counter
 
 	// this channel signalizes that the expected amount of subscriptiosn is done
 	allSubscriptionsDone := make(chan struct{})
-	lock := sync.RWMutex{}
 	// after the test, we need to reset the subscriptionFunc to the default
 	defer func() { subscriptionFunc = doRequestSubscription }()
 
 	// we use this subscriptionFunc for this test: just increases count and calls the actual subscription
 	subscriptionFunc = func(r *Registry, p *network.Peer, bin uint8, subs map[enode.ID]map[Stream]struct{}) bool {
-		lock.Lock()
+		expectedMsgCount.inc()
-		expectedMsgCount++
-		lock.Unlock()
 		doRequestSubscription(r, p, bin, subs)
 		return true
 	}
@@ -1290,24 +1288,24 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 		select {
 		case <-allSubscriptionsDone:
 		case <-ctx.Done():
-			t.Fatal("Context timed out")
+			return errors.New("Context timed out")
 		}
 
-		log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount)
+		log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount.count())
 		//now iterate again, this time we call each node via RPC to get its subscriptions
 		realCount := 0
 		for _, node := range nodes {
 			//create rpc client
 			client, err := node.Client()
 			if err != nil {
-				t.Fatalf("create node 1 rpc client fail: %v", err)
+				return fmt.Errorf("create node 1 rpc client fail: %v", err)
 			}
 
 			//ask it for subscriptions
 			pstreams := make(map[string][]string)
 			err = client.Call(&pstreams, "stream_getPeerSubscriptions")
 			if err != nil {
-				t.Fatal(err)
+				return fmt.Errorf("client call stream_getPeerSubscriptions: %v", err)
 			}
 			//length of the subscriptions can not be smaller than number of peers
 			log.Debug("node subscriptions", "node", node.String())
@@ -1324,8 +1322,9 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 			}
 		}
 		// every node is mutually subscribed to each other, so the actual count is half of it
-		if realCount/2 != expectedMsgCount {
+		emc := expectedMsgCount.count()
-			return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, expectedMsgCount)
+		if realCount/2 != emc {
+			return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, emc)
 		}
 		return nil
 	})
@@ -1333,3 +1332,26 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 		t.Fatal(result.Error)
 	}
 }
+
+// counter is used to concurrently increment
+// and read an integer value.
+type counter struct {
+	v  int
+	mu sync.RWMutex
+}
+
+// Increment the counter.
+func (c *counter) inc() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.v++
+}
+
+// Read the counter value.
+func (c *counter) count() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.v
+}

swarm/network/stream/syncer_test.go

@@ -22,8 +22,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
+	"os"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -44,10 +44,16 @@ const dataChunkCount = 200
 
 func TestSyncerSimulation(t *testing.T) {
 	testSyncBetweenNodes(t, 2, dataChunkCount, true, 1)
+	// This test uses much more memory when running with
+	// race detector. Allow it to finish successfully by
+	// reducing its scope, and still check for data races
+	// with the smallest number of nodes.
+	if !raceTest {
 		testSyncBetweenNodes(t, 4, dataChunkCount, true, 1)
 		testSyncBetweenNodes(t, 8, dataChunkCount, true, 1)
 		testSyncBetweenNodes(t, 16, dataChunkCount, true, 1)
 	}
+}
 
 func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
 	address := common.BytesToAddress(id.Bytes())
@@ -80,7 +86,23 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 				return nil, nil, err
 			}
 
-			r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+			var dir string
+			var store *state.DBStore
+			if raceTest {
+				// Use on-disk DBStore to reduce memory consumption in race tests.
+				dir, err = ioutil.TempDir("", "swarm-stream-")
+				if err != nil {
+					return nil, nil, err
+				}
+				store, err = state.NewDBStore(dir)
+				if err != nil {
+					return nil, nil, err
+				}
+			} else {
+				store = state.NewInmemoryStore()
+			}
+
+			r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{
 				Retrieval: RetrievalDisabled,
 				Syncing:   SyncingAutoSubscribe,
 				SkipCheck: skipCheck,
@@ -89,6 +111,9 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 			cleanup = func() {
 				r.Close()
 				clean()
+				if dir != "" {
+					os.RemoveAll(dir)
+				}
 			}
 
 			return r, cleanup, nil
@@ -114,27 +139,11 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 			nodeIndex[id] = i
 		}
 
-		disconnections := sim.PeerEvents(
+		disconnected := watchDisconnections(ctx, sim)
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
-		var disconnected atomic.Value
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
 				err = errors.New("disconnect events received")
 			}
-			}
 		}()
 
 		// each node Subscribes to each other's swarmChunkServerStreamName
@@ -142,7 +151,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 			id := nodeIDs[j]
 			client, err := sim.Net.GetNode(id).Client()
 			if err != nil {
-				t.Fatal(err)
+				return fmt.Errorf("node %s client: %v", id, err)
 			}
 			sid := nodeIDs[j+1]
 			client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream("SYNC", FormatSyncBinKey(1), false), NewRange(0, 0), Top)
@@ -158,7 +167,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 				size := chunkCount * chunkSize
 				_, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false)
 				if err != nil {
-					t.Fatal(err.Error())
+					return fmt.Errorf("fileStore.Store: %v", err)
 				}
 				wait(ctx)
 			}
@@ -273,7 +282,7 @@ func TestSameVersionID(t *testing.T) {
 
 		//the peers should connect, thus getting the peer should not return nil
 		if registry.getPeer(nodes[1]) == nil {
-			t.Fatal("Expected the peer to not be nil, but it is")
+			return errors.New("Expected the peer to not be nil, but it is")
 		}
 		return nil
 	})
@@ -338,7 +347,7 @@ func TestDifferentVersionID(t *testing.T) {
 
 		//getting the other peer should fail due to the different version numbers
 		if registry.getPeer(nodes[1]) != nil {
-			t.Fatal("Expected the peer to be nil, but it is not")
+			return errors.New("Expected the peer to be nil, but it is not")
 		}
 		return nil
 	})

swarm/network/stream/visualized_snapshot_sync_sim_test.go

@@ -66,31 +66,6 @@ func setupSim(serviceMap map[string]simulation.ServiceFunc) (int, int, *simulati
 	return nodeCount, chunkCount, sim
 }
 
-//watch for disconnections and wait for healthy
-func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) {
-	ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute)
-
-	if _, err := sim.WaitTillHealthy(ctx); err != nil {
-		panic(err)
-	}
-
-	disconnections := sim.PeerEvents(
-		context.Background(),
-		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Drop(),
-	)
-
-	go func() {
-		for d := range disconnections {
-			log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-			panic("unexpected disconnect")
-			cancelSimRun()
-		}
-	}()
-
-	return ctx, cancelSimRun
-}
-
 //This test requests bogus hashes into the network
 func TestNonExistingHashesWithServer(t *testing.T) {
 
@@ -102,19 +77,25 @@ func TestNonExistingHashesWithServer(t *testing.T) {
 		panic(err)
 	}
 
-	ctx, cancelSimRun := watchSim(sim)
-	defer cancelSimRun()
-
 	//in order to get some meaningful visualization, it is beneficial
 	//to define a minimum duration of this test
 	testDuration := 20 * time.Second
 
-	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
+		disconnected := watchDisconnections(ctx, sim)
+		defer func() {
+			if err != nil {
+				if yes, ok := disconnected.Load().(bool); ok && yes {
+					err = errors.New("disconnect events received")
+				}
+			}
+		}()
+
 		//check on the node's FileStore (netstore)
 		id := sim.Net.GetRandomUpNode().ID()
 		item, ok := sim.NodeItem(id, bucketKeyFileStore)
 		if !ok {
-			t.Fatalf("No filestore")
+			return errors.New("No filestore")
 		}
 		fileStore := item.(*storage.FileStore)
 		//create a bogus hash
@@ -213,9 +194,6 @@ func TestSnapshotSyncWithServer(t *testing.T) {
 		panic(err)
 	}
 
-	ctx, cancelSimRun := watchSim(sim)
-	defer cancelSimRun()
-
 	//run the sim
 	result := runSim(conf, ctx, sim, chunkCount)
 

swarm/pss/keystore.go

@@ -0,0 +1,281 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package pss
+
+import (
+	"crypto/ecdsa"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/log"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
+)
+
+type KeyStore struct {
+	w *whisper.Whisper // key and encryption backend
+
+	mx                       sync.RWMutex
+	pubKeyPool               map[string]map[Topic]*pssPeer // mapping of hex public keys to peer address by topic.
+	symKeyPool               map[string]map[Topic]*pssPeer // mapping of symkeyids to peer address by topic.
+	symKeyDecryptCache       []*string                     // fast lookup of symkeys recently used for decryption; last used is on top of stack
+	symKeyDecryptCacheCursor int                           // modular cursor pointing to last used, wraps on symKeyDecryptCache array
+}
+
+func loadKeyStore() *KeyStore {
+	return &KeyStore{
+		w: whisper.New(&whisper.DefaultConfig),
+
+		pubKeyPool:         make(map[string]map[Topic]*pssPeer),
+		symKeyPool:         make(map[string]map[Topic]*pssPeer),
+		symKeyDecryptCache: make([]*string, defaultSymKeyCacheCapacity),
+	}
+}
+
+func (ks *KeyStore) isSymKeyStored(key string) bool {
+	ks.mx.RLock()
+	defer ks.mx.RUnlock()
+	var ok bool
+	_, ok = ks.symKeyPool[key]
+	return ok
+}
+
+func (ks *KeyStore) isPubKeyStored(key string) bool {
+	ks.mx.RLock()
+	defer ks.mx.RUnlock()
+	var ok bool
+	_, ok = ks.pubKeyPool[key]
+	return ok
+}
+
+func (ks *KeyStore) getPeerSym(symkeyid string, topic Topic) (*pssPeer, bool) {
+	ks.mx.RLock()
+	defer ks.mx.RUnlock()
+	psp, ok := ks.symKeyPool[symkeyid][topic]
+	return psp, ok
+}
+
+func (ks *KeyStore) getPeerPub(pubkeyid string, topic Topic) (*pssPeer, bool) {
+	ks.mx.RLock()
+	defer ks.mx.RUnlock()
+	psp, ok := ks.pubKeyPool[pubkeyid][topic]
+	return psp, ok
+}
+
+// Links a peer ECDSA public key to a topic.
+// This is required for asymmetric message exchange on the given topic.
+// The value in `address` will be used as a routing hint for the public key / topic association.
+func (ks *KeyStore) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address PssAddress) error {
+	if err := validateAddress(address); err != nil {
+		return err
+	}
+	pubkeybytes := crypto.FromECDSAPub(pubkey)
+	if len(pubkeybytes) == 0 {
+		return fmt.Errorf("invalid public key: %v", pubkey)
+	}
+	pubkeyid := common.ToHex(pubkeybytes)
+	psp := &pssPeer{
+		address: address,
+	}
+	ks.mx.Lock()
+	if _, ok := ks.pubKeyPool[pubkeyid]; !ok {
+		ks.pubKeyPool[pubkeyid] = make(map[Topic]*pssPeer)
+	}
+	ks.pubKeyPool[pubkeyid][topic] = psp
+	ks.mx.Unlock()
+	log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", address)
+	return nil
+}
+
+// adds a symmetric key to the pss key pool, and optionally adds the key to the
+// collection of keys used to attempt symmetric decryption of incoming messages
+func (ks *KeyStore) addSymmetricKeyToPool(keyid string, topic Topic, address PssAddress, addtocache bool, protected bool) {
+	psp := &pssPeer{
+		address:   address,
+		protected: protected,
+	}
+	ks.mx.Lock()
+	if _, ok := ks.symKeyPool[keyid]; !ok {
+		ks.symKeyPool[keyid] = make(map[Topic]*pssPeer)
+	}
+	ks.symKeyPool[keyid][topic] = psp
+	ks.mx.Unlock()
+	if addtocache {
+		ks.symKeyDecryptCacheCursor++
+		ks.symKeyDecryptCache[ks.symKeyDecryptCacheCursor%cap(ks.symKeyDecryptCache)] = &keyid
+	}
+}
+
+// Returns all recorded topic and address combination for a specific public key
+func (ks *KeyStore) GetPublickeyPeers(keyid string) (topic []Topic, address []PssAddress, err error) {
+	ks.mx.RLock()
+	defer ks.mx.RUnlock()
+	for t, peer := range ks.pubKeyPool[keyid] {
+		topic = append(topic, t)
+		address = append(address, peer.address)
+	}
+	return topic, address, nil
+}
+
+func (ks *KeyStore) getPeerAddress(keyid string, topic Topic) (PssAddress, error) {
+	ks.mx.RLock()
+	defer ks.mx.RUnlock()
+	if peers, ok := ks.pubKeyPool[keyid]; ok {
+		if t, ok := peers[topic]; ok {
+			return t.address, nil
+		}
+	}
+	return nil, fmt.Errorf("peer with pubkey %s, topic %x not found", keyid, topic)
+}
+
+// Attempt to decrypt, validate and unpack a symmetrically encrypted message.
+// If successful, returns the unpacked whisper ReceivedMessage struct
+// encapsulating the decrypted message, and the whisper backend id
+// of the symmetric key used to decrypt the message.
+// It fails if decryption of the message fails or if the message is corrupted.
+func (ks *KeyStore) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
+	metrics.GetOrRegisterCounter("pss.process.sym", nil).Inc(1)
+
+	for i := ks.symKeyDecryptCacheCursor; i > ks.symKeyDecryptCacheCursor-cap(ks.symKeyDecryptCache) && i > 0; i-- {
+		symkeyid := ks.symKeyDecryptCache[i%cap(ks.symKeyDecryptCache)]
+		symkey, err := ks.w.GetSymKey(*symkeyid)
+		if err != nil {
+			continue
+		}
+		recvmsg, err := envelope.OpenSymmetric(symkey)
+		if err != nil {
+			continue
+		}
+		if !recvmsg.ValidateAndParse() {
+			return nil, "", nil, errors.New("symmetrically encrypted message has invalid signature or is corrupt")
+		}
+		var from PssAddress
+		ks.mx.RLock()
+		if ks.symKeyPool[*symkeyid][Topic(envelope.Topic)] != nil {
+			from = ks.symKeyPool[*symkeyid][Topic(envelope.Topic)].address
+		}
+		ks.mx.RUnlock()
+		ks.symKeyDecryptCacheCursor++
+		ks.symKeyDecryptCache[ks.symKeyDecryptCacheCursor%cap(ks.symKeyDecryptCache)] = symkeyid
+		return recvmsg, *symkeyid, from, nil
+	}
+	return nil, "", nil, errors.New("could not decrypt message")
+}
+
+// Attempt to decrypt, validate and unpack an asymmetrically encrypted message.
+// If successful, returns the unpacked whisper ReceivedMessage struct
+// encapsulating the decrypted message, and the byte representation of
+// the public key used to decrypt the message.
+// It fails if decryption of message fails, or if the message is corrupted.
+func (ks *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
+	metrics.GetOrRegisterCounter("pss.process.asym", nil).Inc(1)
+
+	recvmsg, err := envelope.OpenAsymmetric(ks.privateKey)
+	if err != nil {
+		return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err)
+	}
+	// check signature (if signed), strip padding
+	if !recvmsg.ValidateAndParse() {
+		return nil, "", nil, errors.New("invalid message")
+	}
+	pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src))
+	var from PssAddress
+	ks.mx.RLock()
+	if ks.pubKeyPool[pubkeyid][Topic(envelope.Topic)] != nil {
+		from = ks.pubKeyPool[pubkeyid][Topic(envelope.Topic)].address
+	}
+	ks.mx.RUnlock()
+	return recvmsg, pubkeyid, from, nil
+}
+
+// Symkey garbage collection
+// a key is removed if:
+// - it is not marked as protected
+// - it is not in the incoming decryption cache
+func (ks *Pss) cleanKeys() (count int) {
+	for keyid, peertopics := range ks.symKeyPool {
+		var expiredtopics []Topic
+		for topic, psp := range peertopics {
+			if psp.protected {
+				continue
+			}
+
+			var match bool
+			for i := ks.symKeyDecryptCacheCursor; i > ks.symKeyDecryptCacheCursor-cap(ks.symKeyDecryptCache) && i > 0; i-- {
+				cacheid := ks.symKeyDecryptCache[i%cap(ks.symKeyDecryptCache)]
+				if *cacheid == keyid {
+					match = true
+				}
+			}
+			if !match {
+				expiredtopics = append(expiredtopics, topic)
+			}
+		}
+		for _, topic := range expiredtopics {
+			ks.mx.Lock()
+			delete(ks.symKeyPool[keyid], topic)
+			log.Trace("symkey cleanup deletion", "symkeyid", keyid, "topic", topic, "val", ks.symKeyPool[keyid])
+			ks.mx.Unlock()
+			count++
+		}
+	}
+	return count
+}
+
+// Automatically generate a new symkey for a topic and address hint
+func (ks *KeyStore) GenerateSymmetricKey(topic Topic, address PssAddress, addToCache bool) (string, error) {
+	keyid, err := ks.w.GenerateSymKey()
+	if err == nil {
+		ks.addSymmetricKeyToPool(keyid, topic, address, addToCache, false)
+	}
+	return keyid, err
+}
+
+// Returns a symmetric key byte sequence stored in the whisper backend by its unique id.
+// Passes on the error value from the whisper backend.
+func (ks *KeyStore) GetSymmetricKey(symkeyid string) ([]byte, error) {
+	return ks.w.GetSymKey(symkeyid)
+}
+
+// Links a peer symmetric key (arbitrary byte sequence) to a topic.
+//
+// This is required for symmetrically encrypted message exchange on the given topic.
+//
+// The key is stored in the whisper backend.
+//
+// If addtocache is set to true, the key will be added to the cache of keys
+// used to attempt symmetric decryption of incoming messages.
+//
+// Returns a string id that can be used to retrieve the key bytes
+// from the whisper backend (see pss.GetSymmetricKey())
+func (ks *KeyStore) SetSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool) (string, error) {
+	if err := validateAddress(address); err != nil {
+		return "", err
+	}
+	return ks.setSymmetricKey(key, topic, address, addtocache, true)
+}
+
+func (ks *KeyStore) setSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool, protected bool) (string, error) {
+	keyid, err := ks.w.AddSymKeyDirect(key)
+	if err == nil {
+		ks.addSymmetricKeyToPool(keyid, topic, address, addtocache, protected)
+	}
+	return keyid, err
+}

swarm/pss/protocol.go

@@ -241,20 +241,16 @@ func (p *Protocol) AddPeer(peer *p2p.Peer, topic Topic, asymmetric bool, key str
 		rw.sendFunc = p.Pss.SendSym
 	}
 	if asymmetric {
-		p.Pss.pubKeyPoolMu.Lock()
+		if !p.Pss.isPubKeyStored(key) {
-		if _, ok := p.Pss.pubKeyPool[key]; !ok {
 			return nil, fmt.Errorf("asym key does not exist: %s", key)
 		}
-		p.Pss.pubKeyPoolMu.Unlock()
 		p.RWPoolMu.Lock()
 		p.pubKeyRWPool[key] = rw
 		p.RWPoolMu.Unlock()
 	} else {
-		p.Pss.symKeyPoolMu.Lock()
+		if !p.Pss.isSymKeyStored(key) {
-		if _, ok := p.Pss.symKeyPool[key]; !ok {
 			return nil, fmt.Errorf("symkey does not exist: %s", key)
 		}
-		p.Pss.symKeyPoolMu.Unlock()
 		p.RWPoolMu.Lock()
 		p.symKeyRWPool[key] = rw
 		p.RWPoolMu.Unlock()

swarm/pss/pss.go

@@ -113,8 +113,9 @@ func (params *PssParams) WithPrivateKey(privatekey *ecdsa.PrivateKey) *PssParams
 // Implements node.Service
 type Pss struct {
 	*network.Kademlia // we can get the Kademlia address from this
+	*KeyStore
+
 	privateKey *ecdsa.PrivateKey // pss can have it's own independent key
-	w                 *whisper.Whisper  // key and encryption backend
 	auxAPIs    []rpc.API         // builtins (handshake, test) can add APIs
 
 	// sending and forwarding
@@ -128,15 +129,6 @@ type Pss struct {
 	capstring       string
 	outbox          chan *PssMsg
 
-	// keys and peers
-	pubKeyPool                 map[string]map[Topic]*pssPeer // mapping of hex public keys to peer address by topic.
-	pubKeyPoolMu               sync.RWMutex
-	symKeyPool                 map[string]map[Topic]*pssPeer // mapping of symkeyids to peer address by topic.
-	symKeyPoolMu               sync.RWMutex
-	symKeyDecryptCache         []*string // fast lookup of symkeys recently used for decryption; last used is on top of stack
-	symKeyDecryptCacheCursor   int       // modular cursor pointing to last used, wraps on symKeyDecryptCache array
-	symKeyDecryptCacheCapacity int       // max amount of symkeys to keep.
-
 	// message handling
 	handlers           map[Topic]map[*handler]bool // topic and version based pss payload handlers. See pss.Handle()
 	handlersMu         sync.RWMutex
@@ -166,8 +158,9 @@ func NewPss(k *network.Kademlia, params *PssParams) (*Pss, error) {
 	}
 	ps := &Pss{
 		Kademlia: k,
+		KeyStore: loadKeyStore(),
+
 		privateKey: params.privateKey,
-		w:          whisper.New(&whisper.DefaultConfig),
 		quitC:      make(chan struct{}),
 
 		fwdPool:         make(map[string]*protocols.Peer),
@@ -178,11 +171,6 @@ func NewPss(k *network.Kademlia, params *PssParams) (*Pss, error) {
 		capstring:       cap.String(),
 		outbox:          make(chan *PssMsg, defaultOutboxCapacity),
 
-		pubKeyPool:                 make(map[string]map[Topic]*pssPeer),
-		symKeyPool:                 make(map[string]map[Topic]*pssPeer),
-		symKeyDecryptCache:         make([]*string, params.SymKeyCacheCapacity),
-		symKeyDecryptCacheCapacity: params.SymKeyCacheCapacity,
-
 		handlers:         make(map[Topic]map[*handler]bool),
 		topicHandlerCaps: make(map[Topic]*handlerCaps),
 
@@ -539,227 +527,6 @@ func (p *Pss) isSelfPossibleRecipient(msg *PssMsg, prox bool) bool {
 	return depth <= po
 }
 
-/////////////////////////////////////////////////////////////////////
-// SECTION: Encryption
-/////////////////////////////////////////////////////////////////////
-
-// Links a peer ECDSA public key to a topic
-//
-// This is required for asymmetric message exchange
-// on the given topic
-//
-// The value in `address` will be used as a routing hint for the
-// public key / topic association
-func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address PssAddress) error {
-	if err := validateAddress(address); err != nil {
-		return err
-	}
-	pubkeybytes := crypto.FromECDSAPub(pubkey)
-	if len(pubkeybytes) == 0 {
-		return fmt.Errorf("invalid public key: %v", pubkey)
-	}
-	pubkeyid := common.ToHex(pubkeybytes)
-	psp := &pssPeer{
-		address: address,
-	}
-	p.pubKeyPoolMu.Lock()
-	if _, ok := p.pubKeyPool[pubkeyid]; !ok {
-		p.pubKeyPool[pubkeyid] = make(map[Topic]*pssPeer)
-	}
-	p.pubKeyPool[pubkeyid][topic] = psp
-	p.pubKeyPoolMu.Unlock()
-	log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", address)
-	return nil
-}
-
-// Automatically generate a new symkey for a topic and address hint
-func (p *Pss) GenerateSymmetricKey(topic Topic, address PssAddress, addToCache bool) (string, error) {
-	keyid, err := p.w.GenerateSymKey()
-	if err != nil {
-		return "", err
-	}
-	p.addSymmetricKeyToPool(keyid, topic, address, addToCache, false)
-	return keyid, nil
-}
-
-// Links a peer symmetric key (arbitrary byte sequence) to a topic
-//
-// This is required for symmetrically encrypted message exchange
-// on the given topic
-//
-// The key is stored in the whisper backend.
-//
-// If addtocache is set to true, the key will be added to the cache of keys
-// used to attempt symmetric decryption of incoming messages.
-//
-// Returns a string id that can be used to retrieve the key bytes
-// from the whisper backend (see pss.GetSymmetricKey())
-func (p *Pss) SetSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool) (string, error) {
-	if err := validateAddress(address); err != nil {
-		return "", err
-	}
-	return p.setSymmetricKey(key, topic, address, addtocache, true)
-}
-
-func (p *Pss) setSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool, protected bool) (string, error) {
-	keyid, err := p.w.AddSymKeyDirect(key)
-	if err != nil {
-		return "", err
-	}
-	p.addSymmetricKeyToPool(keyid, topic, address, addtocache, protected)
-	return keyid, nil
-}
-
-// adds a symmetric key to the pss key pool, and optionally adds the key
-// to the collection of keys used to attempt symmetric decryption of
-// incoming messages
-func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address PssAddress, addtocache bool, protected bool) {
-	psp := &pssPeer{
-		address:   address,
-		protected: protected,
-	}
-	p.symKeyPoolMu.Lock()
-	if _, ok := p.symKeyPool[keyid]; !ok {
-		p.symKeyPool[keyid] = make(map[Topic]*pssPeer)
-	}
-	p.symKeyPool[keyid][topic] = psp
-	p.symKeyPoolMu.Unlock()
-	if addtocache {
-		p.symKeyDecryptCacheCursor++
-		p.symKeyDecryptCache[p.symKeyDecryptCacheCursor%cap(p.symKeyDecryptCache)] = &keyid
-	}
-	key, _ := p.GetSymmetricKey(keyid)
-	log.Trace("added symkey", "symkeyid", keyid, "symkey", common.ToHex(key), "topic", topic, "address", address, "cache", addtocache)
-}
-
-// Returns a symmetric key byte seqyence stored in the whisper backend
-// by its unique id
-//
-// Passes on the error value from the whisper backend
-func (p *Pss) GetSymmetricKey(symkeyid string) ([]byte, error) {
-	symkey, err := p.w.GetSymKey(symkeyid)
-	if err != nil {
-		return nil, err
-	}
-	return symkey, nil
-}
-
-// Returns all recorded topic and address combination for a specific public key
-func (p *Pss) GetPublickeyPeers(keyid string) (topic []Topic, address []PssAddress, err error) {
-	p.pubKeyPoolMu.RLock()
-	defer p.pubKeyPoolMu.RUnlock()
-	for t, peer := range p.pubKeyPool[keyid] {
-		topic = append(topic, t)
-		address = append(address, peer.address)
-	}
-
-	return topic, address, nil
-}
-
-func (p *Pss) getPeerAddress(keyid string, topic Topic) (PssAddress, error) {
-	p.pubKeyPoolMu.RLock()
-	defer p.pubKeyPoolMu.RUnlock()
-	if peers, ok := p.pubKeyPool[keyid]; ok {
-		if t, ok := peers[topic]; ok {
-			return t.address, nil
-		}
-	}
-	return nil, fmt.Errorf("peer with pubkey %s, topic %x not found", keyid, topic)
-}
-
-// Attempt to decrypt, validate and unpack a
-// symmetrically encrypted message
-// If successful, returns the unpacked whisper ReceivedMessage struct
-// encapsulating the decrypted message, and the whisper backend id
-// of the symmetric key used to decrypt the message.
-// It fails if decryption of the message fails or if the message is corrupted
-func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
-	metrics.GetOrRegisterCounter("pss.process.sym", nil).Inc(1)
-
-	for i := p.symKeyDecryptCacheCursor; i > p.symKeyDecryptCacheCursor-cap(p.symKeyDecryptCache) && i > 0; i-- {
-		symkeyid := p.symKeyDecryptCache[i%cap(p.symKeyDecryptCache)]
-		symkey, err := p.w.GetSymKey(*symkeyid)
-		if err != nil {
-			continue
-		}
-		recvmsg, err := envelope.OpenSymmetric(symkey)
-		if err != nil {
-			continue
-		}
-		if !recvmsg.ValidateAndParse() {
-			return nil, "", nil, fmt.Errorf("symmetrically encrypted message has invalid signature or is corrupt")
-		}
-		p.symKeyPoolMu.Lock()
-		from := p.symKeyPool[*symkeyid][Topic(envelope.Topic)].address
-		p.symKeyPoolMu.Unlock()
-		p.symKeyDecryptCacheCursor++
-		p.symKeyDecryptCache[p.symKeyDecryptCacheCursor%cap(p.symKeyDecryptCache)] = symkeyid
-		return recvmsg, *symkeyid, from, nil
-	}
-	return nil, "", nil, fmt.Errorf("could not decrypt message")
-}
-
-// Attempt to decrypt, validate and unpack an
-// asymmetrically encrypted message
-// If successful, returns the unpacked whisper ReceivedMessage struct
-// encapsulating the decrypted message, and the byte representation of
-// the public key used to decrypt the message.
-// It fails if decryption of message fails, or if the message is corrupted
-func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
-	metrics.GetOrRegisterCounter("pss.process.asym", nil).Inc(1)
-
-	recvmsg, err := envelope.OpenAsymmetric(p.privateKey)
-	if err != nil {
-		return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err)
-	}
-	// check signature (if signed), strip padding
-	if !recvmsg.ValidateAndParse() {
-		return nil, "", nil, fmt.Errorf("invalid message")
-	}
-	pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src))
-	var from PssAddress
-	p.pubKeyPoolMu.Lock()
-	if p.pubKeyPool[pubkeyid][Topic(envelope.Topic)] != nil {
-		from = p.pubKeyPool[pubkeyid][Topic(envelope.Topic)].address
-	}
-	p.pubKeyPoolMu.Unlock()
-	return recvmsg, pubkeyid, from, nil
-}
-
-// Symkey garbage collection
-// a key is removed if:
-// - it is not marked as protected
-// - it is not in the incoming decryption cache
-func (p *Pss) cleanKeys() (count int) {
-	for keyid, peertopics := range p.symKeyPool {
-		var expiredtopics []Topic
-		for topic, psp := range peertopics {
-			if psp.protected {
-				continue
-			}
-
-			var match bool
-			for i := p.symKeyDecryptCacheCursor; i > p.symKeyDecryptCacheCursor-cap(p.symKeyDecryptCache) && i > 0; i-- {
-				cacheid := p.symKeyDecryptCache[i%cap(p.symKeyDecryptCache)]
-				if *cacheid == keyid {
-					match = true
-				}
-			}
-			if !match {
-				expiredtopics = append(expiredtopics, topic)
-			}
-		}
-		for _, topic := range expiredtopics {
-			p.symKeyPoolMu.Lock()
-			delete(p.symKeyPool[keyid], topic)
-			log.Trace("symkey cleanup deletion", "symkeyid", keyid, "topic", topic, "val", p.symKeyPool[keyid])
-			p.symKeyPoolMu.Unlock()
-			count++
-		}
-	}
-	return
-}
-
 /////////////////////////////////////////////////////////////////////
 // SECTION: Message sending
 /////////////////////////////////////////////////////////////////////
@@ -817,9 +584,7 @@ func (p *Pss) SendSym(symkeyid string, topic Topic, msg []byte) error {
 	if err != nil {
 		return fmt.Errorf("missing valid send symkey %s: %v", symkeyid, err)
 	}
-	p.symKeyPoolMu.Lock()
+	psp, ok := p.getPeerSym(symkeyid, topic)
-	psp, ok := p.symKeyPool[symkeyid][topic]
-	p.symKeyPoolMu.Unlock()
 	if !ok {
 		return fmt.Errorf("invalid topic '%s' for symkey '%s'", topic.String(), symkeyid)
 	}
@@ -833,9 +598,7 @@ func (p *Pss) SendAsym(pubkeyid string, topic Topic, msg []byte) error {
 	if _, err := crypto.UnmarshalPubkey(common.FromHex(pubkeyid)); err != nil {
 		return fmt.Errorf("Cannot unmarshal pubkey: %x", pubkeyid)
 	}
-	p.pubKeyPoolMu.Lock()
+	psp, ok := p.getPeerPub(pubkeyid, topic)
-	psp, ok := p.pubKeyPool[pubkeyid][topic]
-	p.pubKeyPoolMu.Unlock()
 	if !ok {
 		return fmt.Errorf("invalid topic '%s' for pubkey '%s'", topic.String(), pubkeyid)
 	}

swarm/state/dbstore.go

@@ -88,18 +88,15 @@ func (s *DBStore) Get(key string, i interface{}) (err error) {
 // Put stores an object that implements Binary for a specific key.
 func (s *DBStore) Put(key string, i interface{}) (err error) {
 	var bytes []byte
-
+	if marshaler, ok := i.(encoding.BinaryMarshaler); ok {
-	marshaler, ok := i.(encoding.BinaryMarshaler)
-	if !ok {
-		if bytes, err = json.Marshal(i); err != nil {
-			return err
-		}
-	} else {
 		if bytes, err = marshaler.MarshalBinary(); err != nil {
 			return err
 		}
+	} else {
+		if bytes, err = json.Marshal(i); err != nil {
+			return err
+		}
 	}
-
 	return s.db.Put([]byte(key), bytes, nil)
 }
 

swarm/storage/filestore.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"io"
 	"sort"
+	"sync"
 )
 
 /*
@@ -101,38 +102,45 @@ func (f *FileStore) HashSize() int {
 // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file
 func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) {
 	// create a special kind of putter, which only will store the references
-	putter := &HashExplorer{
+	putter := &hashExplorer{
 		hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt),
-		References:  make([]Reference, 0),
 	}
 	// do the actual splitting anyway, no way around it
-	_, _, err = PyramidSplit(ctx, data, putter, putter)
+	_, wait, err := PyramidSplit(ctx, data, putter, putter)
+	if err != nil {
+		return nil, err
+	}
+	// wait for splitting to be complete and all chunks processed
+	err = wait(ctx)
 	if err != nil {
 		return nil, err
 	}
 	// collect all references
 	addrs = NewAddressCollection(0)
-	for _, ref := range putter.References {
+	for _, ref := range putter.references {
 		addrs = append(addrs, Address(ref))
 	}
 	sort.Sort(addrs)
 	return addrs, nil
 }
 
-// HashExplorer is a special kind of putter which will only store chunk references
+// hashExplorer is a special kind of putter which will only store chunk references
-type HashExplorer struct {
+type hashExplorer struct {
 	*hasherStore
-	References []Reference
+	references []Reference
+	lock       sync.Mutex
 }
 
 // HashExplorer's Put will add just the chunk hashes to its `References`
-func (he *HashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
+func (he *hashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
 	// Need to do the actual Put, which returns the references
 	ref, err := he.hasherStore.Put(ctx, chunkData)
 	if err != nil {
 		return nil, err
 	}
 	// internally store the reference
-	he.References = append(he.References, ref)
+	he.lock.Lock()
+	he.references = append(he.references, ref)
+	he.lock.Unlock()
 	return ref, nil
 }

swarm/storage/ldbstore.go

@@ -317,7 +317,6 @@ func decodeData(addr Address, data []byte) (*chunk, error) {
 }
 
 func (s *LDBStore) collectGarbage() error {
-
 	// prevent duplicate gc from starting when one is already running
 	select {
 	case <-s.gc.runC:
@@ -335,7 +334,6 @@ func (s *LDBStore) collectGarbage() error {
 	s.startGC(int(entryCnt))
 	log.Debug("collectGarbage", "target", s.gc.target, "entryCnt", entryCnt)
 
-	var totalDeleted int
 	for s.gc.count < s.gc.target {
 		it := s.db.NewIterator()
 		ok := it.Seek([]byte{keyGCIdx})
@@ -371,15 +369,15 @@ func (s *LDBStore) collectGarbage() error {
 		}
 
 		s.writeBatch(s.gc.batch, wEntryCnt)
+		log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count)
 		s.lock.Unlock()
 		it.Release()
-		log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count)
 	}
 
-	s.gc.runC <- struct{}{}
+	metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(s.gc.count))
 	log.Debug("garbage collect done", "c", s.gc.count)
+	s.gc.runC <- struct{}{}
 
-	metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(totalDeleted))
 	return nil
 }
 
@@ -1049,7 +1047,6 @@ func (s *LDBStore) Close() {
 	s.lock.Unlock()
 	// force writing out current batch
 	s.writeCurrentBatch()
-	close(s.batchesC)
 	s.db.Close()
 }
 

swarm/storage/netstore.go

@@ -128,7 +128,25 @@ func (n *NetStore) FetchFunc(ctx context.Context, ref Address) func(context.Cont
 func (n *NetStore) Close() {
 	close(n.closeC)
 	n.store.Close()
-	// TODO: loop through fetchers to cancel them
+
+	wg := sync.WaitGroup{}
+	for _, key := range n.fetchers.Keys() {
+		if f, ok := n.fetchers.Get(key); ok {
+			if fetch, ok := f.(*fetcher); ok {
+				wg.Add(1)
+				go func(fetch *fetcher) {
+					defer wg.Done()
+					fetch.cancel()
+
+					select {
+					case <-fetch.deliveredC:
+					case <-fetch.cancelledC:
+					}
+				}(fetch)
+			}
+		}
+	}
+	wg.Wait()
 }
 
 // get attempts at retrieving the chunk from LocalStore

swarm/swarm.go

@@ -79,7 +79,7 @@ type Swarm struct {
 	swap              *swap.Swap
 	stateStore        *state.DBStore
 	accountingMetrics *protocols.AccountingMetrics
-	startTime         time.Time
+	cleanupFuncs      []func() error
 
 	tracerClose io.Closer
 }
@@ -109,6 +109,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 		config:       config,
 		backend:      backend,
 		privateKey:   config.ShiftPrivateKey(),
+		cleanupFuncs: []func() error{},
 	}
 	log.Debug("Setting up Swarm service components")
 
@@ -343,51 +344,51 @@ Start is called when the stack is started
 * TODO: start subservices like sword, swear, swarmdns
 */
 // implements the node.Service interface
-func (self *Swarm) Start(srv *p2p.Server) error {
+func (s *Swarm) Start(srv *p2p.Server) error {
-	self.startTime = time.Now()
+	startTime := time.Now()
 
-	self.tracerClose = tracing.Closer
+	s.tracerClose = tracing.Closer
 
 	// update uaddr to correct enode
-	newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String()))
+	newaddr := s.bzz.UpdateLocalAddr([]byte(srv.Self().String()))
 	log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr))
 	// set chequebook
 	//TODO: Currently if swap is enabled and no chequebook (or inexistent) contract is provided, the node would crash.
 	//Once we integrate back the contracts, this check MUST be revisited
-	if self.config.SwapEnabled && self.config.SwapAPI != "" {
+	if s.config.SwapEnabled && s.config.SwapAPI != "" {
 		ctx := context.Background() // The initial setup has no deadline.
-		err := self.SetChequebook(ctx)
+		err := s.SetChequebook(ctx)
 		if err != nil {
 			return fmt.Errorf("Unable to set chequebook for SWAP: %v", err)
 		}
-		log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", self.config.Swap.Chequebook()))
+		log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", s.config.Swap.Chequebook()))
 	} else {
 		log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set"))
 	}
 
 	log.Info("Starting bzz service")
 
-	err := self.bzz.Start(srv)
+	err := s.bzz.Start(srv)
 	if err != nil {
 		log.Error("bzz failed", "err", err)
 		return err
 	}
-	log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", self.bzz.Hive.BaseAddr()))
+	log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", s.bzz.Hive.BaseAddr()))
 
-	if self.ps != nil {
+	if s.ps != nil {
-		self.ps.Start(srv)
+		s.ps.Start(srv)
 	}
 
 	// start swarm http proxy server
-	if self.config.Port != "" {
+	if s.config.Port != "" {
-		addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port)
+		addr := net.JoinHostPort(s.config.ListenAddr, s.config.Port)
-		server := httpapi.NewServer(self.api, self.config.Cors)
+		server := httpapi.NewServer(s.api, s.config.Cors)
 
-		if self.config.Cors != "" {
+		if s.config.Cors != "" {
-			log.Debug("Swarm HTTP proxy CORS headers", "allowedOrigins", self.config.Cors)
+			log.Debug("Swarm HTTP proxy CORS headers", "allowedOrigins", s.config.Cors)
 		}
 
-		log.Debug("Starting Swarm HTTP proxy", "port", self.config.Port)
+		log.Debug("Starting Swarm HTTP proxy", "port", s.config.Port)
 		go func() {
 			err := server.ListenAndServe(addr)
 			if err != nil {
@@ -396,61 +397,71 @@ func (self *Swarm) Start(srv *p2p.Server) error {
 		}()
 	}
 
-	self.periodicallyUpdateGauges()
+	doneC := make(chan struct{})
+
+	s.cleanupFuncs = append(s.cleanupFuncs, func() error {
+		close(doneC)
+		return nil
+	})
+
+	go func(time.Time) {
+		for {
+			select {
+			case <-time.After(updateGaugesPeriod):
+				uptimeGauge.Update(time.Since(startTime).Nanoseconds())
+				requestsCacheGauge.Update(int64(s.netStore.RequestsCacheLen()))
+			case <-doneC:
+				return
+			}
+		}
+	}(startTime)
 
 	startCounter.Inc(1)
-	self.streamer.Start(srv)
+	s.streamer.Start(srv)
 	return nil
 }
 
-func (self *Swarm) periodicallyUpdateGauges() {
-	ticker := time.NewTicker(updateGaugesPeriod)
-
-	go func() {
-		for range ticker.C {
-			self.updateGauges()
-		}
-	}()
-}
-
-func (self *Swarm) updateGauges() {
-	uptimeGauge.Update(time.Since(self.startTime).Nanoseconds())
-	requestsCacheGauge.Update(int64(self.netStore.RequestsCacheLen()))
-}
-
 // implements the node.Service interface
 // stops all component services.
-func (self *Swarm) Stop() error {
+func (s *Swarm) Stop() error {
-	if self.tracerClose != nil {
+	if s.tracerClose != nil {
-		err := self.tracerClose.Close()
+		err := s.tracerClose.Close()
 		if err != nil {
 			return err
 		}
 	}
 
-	if self.ps != nil {
+	if s.ps != nil {
-		self.ps.Stop()
+		s.ps.Stop()
 	}
-	if ch := self.config.Swap.Chequebook(); ch != nil {
+	if ch := s.config.Swap.Chequebook(); ch != nil {
 		ch.Stop()
 		ch.Save()
 	}
-	if self.swap != nil {
+	if s.swap != nil {
-		self.swap.Close()
+		s.swap.Close()
 	}
-	if self.accountingMetrics != nil {
+	if s.accountingMetrics != nil {
-		self.accountingMetrics.Close()
+		s.accountingMetrics.Close()
 	}
-	if self.netStore != nil {
+	if s.netStore != nil {
-		self.netStore.Close()
+		s.netStore.Close()
 	}
-	self.sfs.Stop()
+	s.sfs.Stop()
 	stopCounter.Inc(1)
-	self.streamer.Stop()
+	s.streamer.Stop()
 
-	err := self.bzz.Stop()
+	err := s.bzz.Stop()
-	if self.stateStore != nil {
+	if s.stateStore != nil {
-		self.stateStore.Close()
+		s.stateStore.Close()
+	}
+
+	for _, cleanF := range s.cleanupFuncs {
+		err = cleanF()
+		if err != nil {
+			log.Error("encountered an error while running cleanup function", "err", err)
+			break
+		}
 	}
 	return err
 }
@@ -471,68 +482,73 @@ func (s *Swarm) Protocols() (protos []p2p.Protocol) {
 
 // implements node.Service
 // APIs returns the RPC API descriptors the Swarm implementation offers
-func (self *Swarm) APIs() []rpc.API {
+func (s *Swarm) APIs() []rpc.API {
 
 	apis := []rpc.API{
 		// public APIs
 		{
 			Namespace: "bzz",
 			Version:   "3.0",
-			Service:   &Info{self.config, chequebook.ContractParams},
+			Service:   &Info{s.config, chequebook.ContractParams},
 			Public:    true,
 		},
 		// admin APIs
 		{
 			Namespace: "bzz",
 			Version:   "3.0",
-			Service:   api.NewInspector(self.api, self.bzz.Hive, self.netStore),
+			Service:   api.NewInspector(s.api, s.bzz.Hive, s.netStore),
 			Public:    false,
 		},
 		{
 			Namespace: "chequebook",
 			Version:   chequebook.Version,
-			Service:   chequebook.NewAPI(self.config.Swap.Chequebook),
+			Service:   chequebook.NewAPI(s.config.Swap.Chequebook),
 			Public:    false,
 		},
 		{
 			Namespace: "swarmfs",
 			Version:   fuse.Swarmfs_Version,
-			Service:   self.sfs,
+			Service:   s.sfs,
 			Public:    false,
 		},
 		{
 			Namespace: "accounting",
 			Version:   protocols.AccountingVersion,
-			Service:   protocols.NewAccountingApi(self.accountingMetrics),
+			Service:   protocols.NewAccountingApi(s.accountingMetrics),
 			Public:    false,
 		},
 	}
 
-	apis = append(apis, self.bzz.APIs()...)
+	apis = append(apis, s.bzz.APIs()...)
 
-	if self.ps != nil {
+	if s.ps != nil {
-		apis = append(apis, self.ps.APIs()...)
+		apis = append(apis, s.ps.APIs()...)
 	}
 
 	return apis
 }
 
 // SetChequebook ensures that the local checquebook is set up on chain.
-func (self *Swarm) SetChequebook(ctx context.Context) error {
+func (s *Swarm) SetChequebook(ctx context.Context) error {
-	err := self.config.Swap.SetChequebook(ctx, self.backend, self.config.Path)
+	err := s.config.Swap.SetChequebook(ctx, s.backend, s.config.Path)
 	if err != nil {
 		return err
 	}
-	log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex()))
+	log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", s.config.Swap.Contract.Hex()))
 	return nil
 }
 
+// RegisterPssProtocol adds a devp2p protocol to the swarm node's Pss instance
+func (s *Swarm) RegisterPssProtocol(topic *pss.Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol, options *pss.ProtocolParams) (*pss.Protocol, error) {
+	return pss.RegisterProtocol(s.ps, topic, spec, targetprotocol, options)
+}
+
 // serialisable info about swarm
 type Info struct {
 	*api.Config
 	*chequebook.Params
 }
 
-func (self *Info) Info() *Info {
+func (s *Info) Info() *Info {
-	return self
+	return s
 }

trie/encoding.go

@@ -52,6 +52,9 @@ func hexToCompact(hex []byte) []byte {
 }
 
 func compactToHex(compact []byte) []byte {
+	if len(compact) == 0 {
+		return compact
+	}
 	base := keybytesToHex(compact)
 	// delete terminator flag
 	if base[0] < 2 {

trie/trie_test.go

@@ -614,3 +614,16 @@ func updateString(trie *Trie, k, v string) {
 func deleteString(trie *Trie, k string) {
 	trie.Delete([]byte(k))
 }
+
+func TestDecodeNode(t *testing.T) {
+	t.Parallel()
+	var (
+		hash  = make([]byte, 20)
+		elems = make([]byte, 20)
+	)
+	for i := 0; i < 5000000; i++ {
+		rand.Read(hash)
+		rand.Read(elems)
+		decodeNode(hash, elems, 1)
+	}
+}

vendor/github.com/allegro/bigcache/bigcache.go

@@ -102,7 +102,7 @@ func (c *BigCache) Close() error {
 }
 
 // Get reads entry for the key.
-// It returns an EntryNotFoundError when
+// It returns an ErrEntryNotFound when
 // no entry exists for the given key.
 func (c *BigCache) Get(key string) ([]byte, error) {
 	hashedKey := c.hash.Sum64(key)

vendor/github.com/allegro/bigcache/entry_not_found_error.go

@@ -1,17 +1,6 @@
 package bigcache
 
-import "fmt"
+import "errors"
 
-// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key
+// ErrEntryNotFound is an error type struct which is returned when entry was not found for provided key
-type EntryNotFoundError struct {
+var ErrEntryNotFound = errors.New("Entry not found")
-	key string
-}
-
-func notFound(key string) error {
-	return &EntryNotFoundError{key}
-}
-
-// Error returned when entry does not exist.
-func (e EntryNotFoundError) Error() string {
-	return fmt.Sprintf("Entry %q not found", e.key)
-}

vendor/github.com/allegro/bigcache/queue/bytes_queue.go

@@ -16,6 +16,12 @@ const (
 	minimumEmptyBlobSize = 32 + headerEntrySize
 )
 
+var (
+	errEmptyQueue       = &queueError{"Empty queue"}
+	errInvalidIndex     = &queueError{"Index must be greater than zero. Invalid index."}
+	errIndexOutOfBounds = &queueError{"Index out of range"}
+)
+
 // BytesQueue is a non-thread safe queue type of fifo based on bytes array.
 // For every push operation index of entry is returned. It can be used to read the entry later
 type BytesQueue struct {
@@ -162,6 +168,11 @@ func (q *BytesQueue) Get(index int) ([]byte, error) {
 	return data, err
 }
 
+// CheckGet checks if an entry can be read from index
+func (q *BytesQueue) CheckGet(index int) error {
+	return q.peekCheckErr(index)
+}
+
 // Capacity returns number of allocated bytes for queue
 func (q *BytesQueue) Capacity() int {
 	return q.capacity
@@ -177,18 +188,35 @@ func (e *queueError) Error() string {
 	return e.message
 }
 
-func (q *BytesQueue) peek(index int) ([]byte, int, error) {
+// peekCheckErr is identical to peek, but does not actually return any data
+func (q *BytesQueue) peekCheckErr(index int) error {
 
 	if q.count == 0 {
-		return nil, 0, &queueError{"Empty queue"}
+		return errEmptyQueue
 	}
 
 	if index <= 0 {
-		return nil, 0, &queueError{"Index must be grater than zero. Invalid index."}
+		return errInvalidIndex
 	}
 
 	if index+headerEntrySize >= len(q.array) {
-		return nil, 0, &queueError{"Index out of range"}
+		return errIndexOutOfBounds
+	}
+	return nil
+}
+
+func (q *BytesQueue) peek(index int) ([]byte, int, error) {
+
+	if q.count == 0 {
+		return nil, 0, errEmptyQueue
+	}
+
+	if index <= 0 {
+		return nil, 0, errInvalidIndex
+	}
+
+	if index+headerEntrySize >= len(q.array) {
+		return nil, 0, errIndexOutOfBounds
 	}
 
 	blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize]))