swarm: merge branch 'master' into mock-store-listings

.travis.yml

@@ -68,8 +68,10 @@ matrix:
             - debhelper
             - dput
             - fakeroot
+            - python-bzrlib
+            - python-paramiko
       script:
-        - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - if: type = push

accounts/abi/bind/backends/simulated.go

@@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
 	return receipt, nil
 }
 
+// TransactionByHash checks the pool of pending transactions in addition to the
+// blockchain. The isPending return value indicates whether the transaction has been
+// mined yet. Note that the transaction may not be part of the canonical chain even if
+// it's not pending.
+func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) {
+
+	tx = b.pendingBlock.Transaction(txHash)
+	if tx != nil {
+		return tx, true, nil
+	}
+
+	tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
+	if tx != nil {
+		return tx, false, nil
+	}
+
+	return nil, false, ethereum.NotFound
+}
+
 // PendingCodeAt returns the code associated with an account in the pending state.
 func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
 	b.mu.Lock()

accounts/abi/bind/backends/simulated_test.go

@@ -0,0 +1,66 @@
+package backends_test
+
+import (
+	"context"
+	"math/big"
+	"testing"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+func TestSimulatedBackend(t *testing.T) {
+	var gasLimit uint64 = 8000029
+	key, _ := crypto.GenerateKey() // nolint: gosec
+	auth := bind.NewKeyedTransactor(key)
+	genAlloc := make(core.GenesisAlloc)
+	genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
+
+	sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
+
+	// should return an error if the tx is not found
+	txHash := common.HexToHash("2")
+	_, isPending, err := sim.TransactionByHash(context.Background(), txHash)
+
+	if isPending {
+		t.Fatal("transaction should not be pending")
+	}
+	if err != ethereum.NotFound {
+		t.Fatalf("err should be `ethereum.NotFound` but received %v", err)
+	}
+
+	// generate a transaction and confirm you can retrieve it
+	code := `6060604052600a8060106000396000f360606040526008565b00`
+	var gas uint64 = 3000000
+	tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code))
+	tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key)
+
+	err = sim.SendTransaction(context.Background(), tx)
+	if err != nil {
+		t.Fatal("error sending transaction")
+	}
+
+	txHash = tx.Hash()
+	_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if !isPending {
+		t.Fatal("transaction should have pending status")
+	}
+
+	sim.Commit()
+	tx, isPending, err = sim.TransactionByHash(context.Background(), txHash)
+	if err != nil {
+		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+	}
+	if isPending {
+		t.Fatal("transaction should not have pending status")
+	}
+
+}

accounts/external/backend.go

@@ -26,7 +26,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
@@ -154,13 +153,31 @@ func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]by
 
 // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
 func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
-	// TODO! Replace this with a call to clef SignData with correct mime-type for Clique, once we
+	var res hexutil.Bytes
-	// have that in place
+	var signAddress = common.NewMixedcaseAddress(account.Address)
-	return api.signHash(account, crypto.Keccak256(data))
+	if err := api.client.Call(&res, "account_signData",
+		mimeType,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(data)); err != nil {
+		return nil, err
+	}
+	// If V is on 27/28-form, convert to to 0/1 for Clique
+	if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) {
+		res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
+	}
+	return res, nil
 }
 
 func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
-	return api.signHash(account, accounts.TextHash(text))
+	var res hexutil.Bytes
+	var signAddress = common.NewMixedcaseAddress(account.Address)
+	if err := api.client.Call(&res, "account_signData",
+		accounts.MimetypeTextPlain,
+		&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
+		hexutil.Encode(text)); err != nil {
+		return nil, err
+	}
+	return res, nil
 }
 
 func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
@@ -202,18 +219,6 @@ func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
 	return res, nil
 }
 
-func (api *ExternalSigner) signCliqueBlock(a common.Address, rlpBlock hexutil.Bytes) (hexutil.Bytes, error) {
-	var sig hexutil.Bytes
-	if err := api.client.Call(&sig, "account_signData", core.ApplicationClique.Mime, a, rlpBlock); err != nil {
-		return nil, err
-	}
-	if sig[64] != 27 && sig[64] != 28 {
-		return nil, fmt.Errorf("invalid Ethereum signature (V is not 27 or 28)")
-	}
-	sig[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
-	return sig, nil
-}
-
 func (api *ExternalSigner) pingVersion() (string, error) {
 	var v string
 	if err := api.client.Call(&v, "account_version"); err != nil {

build/ci-notes.md

@@ -7,11 +7,18 @@ Canonical.
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
 
+## Launchpad
+
 The packages are built and served by launchpad.net. We generate a Debian source package
 for each distribution and upload it. Their builder picks up the source package, builds it
 and installs the new version into the PPA repository. Launchpad requires a valid signature
-by a team member for source package uploads. The signing key is stored in an environment
+by a team member for source package uploads.
-variable which Travis CI makes available to certain builds.
+
+The signing key is stored in an environment variable which Travis CI makes available to
+certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
+packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
+and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
+variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
@@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 

build/ci.go

@@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
 func archiveUpload(archive string, blobstore string, signer string) error {
 	// If signing was requested, generate the signature files
 	if signer != "" {
-		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
+		key := getenvBase64(signer)
-		if err != nil {
+		if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
-			return fmt.Errorf("invalid base64 %s", signer)
-		}
-		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
 			return err
 		}
 	}
@@ -488,7 +485,8 @@ func maybeSkipArchive(env build.Environment) {
 func doDebianSource(cmdline []string) {
 	var (
 		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+		upload  = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+		sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
 		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
 		now     = time.Now()
 	)
@@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
 	maybeSkipArchive(env)
 
 	// Import the signing key.
-	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
+	if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatal("invalid base64 PPA_SIGNING_KEY")
-		}
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
@@ -523,12 +517,45 @@ func doDebianSource(cmdline []string) {
 				build.MustRunCommand("debsign", changes)
 			}
 			if *upload != "" {
-				build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes)
+				uploadDebianSource(*workdir, *upload, *sshUser, changes)
 			}
 		}
 	}
 }
 
+func uploadDebianSource(workdir, ppa, sshUser, changes string) {
+	// Create the dput config file.
+	dputConfig := filepath.Join(workdir, "dput.cf")
+	p := strings.Split(ppa, "/")
+	if len(p) != 2 {
+		log.Fatal("-upload PPA name must contain single /")
+	}
+	templateData := map[string]string{
+		"LaunchpadUser": p[0],
+		"LaunchpadPPA":  p[1],
+		"LaunchpadSSH":  sshUser,
+	}
+	if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
+		idfile := filepath.Join(workdir, "sshkey")
+		ioutil.WriteFile(idfile, sshkey, 0600)
+		templateData["IdentityFile"] = idfile
+	}
+	build.Render("build/dput-launchpad.cf", dputConfig, 0644, templateData)
+
+	// Run dput to do the upload.
+	dput := exec.Command("dput", "-c", dputConfig, "--no-upload-log", ppa, changes)
+	dput.Stdin = strings.NewReader("Yes\n") // accept SSH host key
+	build.MustRun(dput)
+}
+
+func getenvBase64(variable string) []byte {
+	dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
+	if err != nil {
+		log.Fatal("invalid base64 " + variable)
+	}
+	return []byte(dec)
+}
+
 func makeWorkdir(wdflag string) string {
 	var err error
 	if wdflag != "" {
@@ -800,15 +827,10 @@ func doAndroidArchive(cmdline []string) {
 	os.Rename(archive, meta.Package+".aar")
 	if *signer != "" && *deploy != "" {
 		// Import the signing key into the local GPG instance
-		b64key := os.Getenv(*signer)
+		key := getenvBase64(*signer)
-		key, err := base64.StdEncoding.DecodeString(b64key)
-		if err != nil {
-			log.Fatalf("invalid base64 %s", *signer)
-		}
 		gpg := exec.Command("gpg", "--import")
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
-
 		keyID, err := build.PGPKeyID(string(key))
 		if err != nil {
 			log.Fatal(err)

build/dput-launchpad.cf

@@ -0,0 +1,8 @@
+[{{.LaunchpadUser}}/{{.LaunchpadPPA}}]
+fqdn = ppa.launchpad.net
+method = sftp
+incoming = ~{{.LaunchpadUser}}/ubuntu/{{.LaunchpadPPA}}/
+login = {{.LaunchpadSSH}}
+{{ if .IdentityFile }}
+  ssh_options = IdentityFile {{.IdentityFile}}
+{{ end }}

cmd/clef/extapi_changelog.md

@@ -1,5 +1,10 @@
 ### Changelog for external API
 
+### 6.0.0
+
+* `New` was changed to deliver only an address, not the full `Account` data
+* `Export` was moved from External API to the UI Server API
+
 #### 5.0.0
 
 * The external `account_EcRecover`-method was reimplemented.

cmd/clef/intapi_changelog.md

@@ -1,8 +1,39 @@
 ### Changelog for internal API (ui-api)
 
+### 4.0.0
+
+* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
+  - `clef_listWallets` 
+  - `clef_listAccounts`
+  - `clef_listWallets`
+  - `clef_deriveAccount`
+  - `clef_importRawKey`
+  - `clef_openWallet`
+  - `clef_chainId`
+  - `clef_setChainId`
+  - `clef_export`
+  - `clef_import`
+ 
+* The type `Account` was modified (the json-field `type` was removed), to consist of 
+
+```golang
+type Account struct {
+	Address common.Address `json:"address"` // Ethereum account address derived from the key
+	URL     URL            `json:"url"`     // Optional resource locator within a backend
+}
+```
+
+
+### 3.2.0
+
+* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
+
+> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
+> 
+>  Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
 ### 3.1.0
 
-* Add `ContentType string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
+* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
 
 ### 3.0.0
 

cmd/clef/main.go

@@ -44,6 +44,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/signer/core"
@@ -83,6 +84,11 @@ var (
 		Value: DefaultConfigDir(),
 		Usage: "Directory for Clef configuration",
 	}
+	chainIdFlag = cli.Int64Flag{
+		Name:  "chainid",
+		Value: params.MainnetChainConfig.ChainID.Int64(),
+		Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
+	}
 	rpcPortFlag = cli.IntFlag{
 		Name:  "rpcport",
 		Usage: "HTTP-RPC server listening port",
@@ -178,7 +184,7 @@ func init() {
 		logLevelFlag,
 		keystoreFlag,
 		configdirFlag,
-		utils.NetworkIdFlag,
+		chainIdFlag,
 		utils.LightKDFFlag,
 		utils.NoUSBFlag,
 		utils.RPCListenAddrFlag,
@@ -338,7 +344,7 @@ func signer(c *cli.Context) error {
 		return err
 	}
 	var (
-		ui core.SignerUI
+		ui core.UIClientAPI
 	)
 	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
@@ -402,14 +408,21 @@ func signer(c *cli.Context) error {
 			}
 		}
 	}
+	var (
+		chainId  = c.GlobalInt64(chainIdFlag.Name)
+		ksLoc    = c.GlobalString(keystoreFlag.Name)
+		lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
+		advanced = c.GlobalBool(advancedMode.Name)
+		nousb    = c.GlobalBool(utils.NoUSBFlag.Name)
+	)
+	log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
+		"light-kdf", lightKdf, "advanced", advanced)
+	am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
+	apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced)
 
-	apiImpl := core.NewSignerAPI(
+	// Establish the bidirectional communication, by creating a new UI backend and registering
-		c.GlobalInt64(utils.NetworkIdFlag.Name),
+	// it with the UI.
-		c.GlobalString(keystoreFlag.Name),
+	ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
-		c.GlobalBool(utils.NoUSBFlag.Name),
-		ui, db,
-		c.GlobalBool(utils.LightKDFFlag.Name),
-		c.GlobalBool(advancedMode.Name))
 	api = apiImpl
 	// Audit logging
 	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
@@ -529,7 +542,7 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	var (
 		file      string
 		configDir = ctx.GlobalString(configdirFlag.Name)
@@ -664,10 +677,6 @@ func testExternalUI(api *core.SignerAPI) {
 	checkErr("List", err)
 	_, err = api.New(ctx)
 	checkErr("New", err)
-	_, err = api.Export(ctx, common.Address{})
-	checkErr("Export", err)
-	_, err = api.Import(ctx, json.RawMessage{})
-	checkErr("Import", err)
 
 	api.UI.ShowInfo("Tests completed")
 

cmd/clef/tests/testsigner.js

@@ -0,0 +1,73 @@
+// This file is a test-utility for testing clef-functionality
+//
+// Start clef with
+//
+// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
+//
+// Start geth with
+//
+// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
+//
+// and in the console simply invoke
+//
+// > test()
+//
+// You can reload the file via `reload()`
+
+function reload(){
+	loadScript("./cmd/clef/tests/testsigner.js");
+}
+
+function init(){
+    if (typeof accts == 'undefined' || accts.length == 0){
+        accts = eth.accounts
+        console.log("Got accounts ", accts);
+    }
+}
+init()
+function testTx(){
+    if( accts && accts.length > 0) {
+        var a = accts[0]
+        var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
+        var v = parseInt(txdata.tx.v)
+        console.log("V value: ", v)
+        if (v == 37 || v == 38){
+            console.log("Mainnet 155-protected chainid was used")
+        }
+        if (v == 27 || v == 28){
+            throw new Error("Mainnet chainid was used, but without replay protection!")
+        }
+    }
+}
+function testSignText(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
+        console.log("signing response",  r)
+    }
+}
+function testClique(){
+    if( accts && accts.length > 0){
+        var a = accts[0]
+        var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
+        console.log("signing response",  r)
+        if( a != r){
+            throw new Error("Requested signing by "+a+ " but got sealer "+r)
+        }
+    }
+}
+
+function test(){
+    var tests = [
+        testTx,
+        testSignText,
+        testClique,
+    ]
+    for( i in tests){
+        try{
+            tests[i]()
+        }catch(err){
+            console.log(err)
+        }
+    }
+ }

cmd/swarm/config.go

@@ -82,6 +82,7 @@ const (
 	SWARM_ENV_BOOTNODE_MODE           = "SWARM_BOOTNODE_MODE"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
 	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
+	SWARM_GLOBALSTORE_API             = "SWARM_GLOBALSTORE_API"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 
@@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name)
 	}
 
+	if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) {
+		currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name)
+	}
+
 	return currentConfig
 
 }
@@ -375,6 +380,10 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		currentConfig.BootnodeMode = bootnodeMode
 	}
 
+	if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" {
+		currentConfig.GlobalStoreAPI = api
+	}
+
 	return currentConfig
 }
 

cmd/swarm/flags.go

@@ -176,4 +176,9 @@ var (
 		Name:  "user",
 		Usage: "Indicates the user who updates the feed",
 	}
+	SwarmGlobalStoreAPIFlag = cli.StringFlag{
+		Name:   "globalstore-api",
+		Usage:  "URL of the Global Store API provider (only for testing)",
+		EnvVar: SWARM_GLOBALSTORE_API,
+	}
 )

cmd/swarm/global-store/global_store.go

@@ -0,0 +1,100 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+// startHTTP starts a global store with HTTP RPC server.
+// It is used for "http" cli command.
+func startHTTP(ctx *cli.Context) (err error) {
+	server, cleanup, err := newServer(ctx)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	listener, err := net.Listen("tcp", ctx.String("addr"))
+	if err != nil {
+		return err
+	}
+	log.Info("http", "address", listener.Addr().String())
+
+	return http.Serve(listener, server)
+}
+
+// startWS starts a global store with WebSocket RPC server.
+// It is used for "websocket" cli command.
+func startWS(ctx *cli.Context) (err error) {
+	server, cleanup, err := newServer(ctx)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	listener, err := net.Listen("tcp", ctx.String("addr"))
+	if err != nil {
+		return err
+	}
+	origins := ctx.StringSlice("origins")
+	log.Info("websocket", "address", listener.Addr().String(), "origins", origins)
+
+	return http.Serve(listener, server.WebsocketHandler(origins))
+}
+
+// newServer creates a global store and returns its RPC server.
+// Returned cleanup function should be called only if err is nil.
+func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
+
+	cleanup = func() {}
+	var globalStore mock.GlobalStorer
+	dir := ctx.String("dir")
+	if dir != "" {
+		dbStore, err := db.NewGlobalStore(dir)
+		if err != nil {
+			return nil, nil, err
+		}
+		cleanup = func() {
+			dbStore.Close()
+		}
+		globalStore = dbStore
+		log.Info("database global store", "dir", dir)
+	} else {
+		globalStore = mem.NewGlobalStore()
+		log.Info("in-memory global store")
+	}
+
+	server = rpc.NewServer()
+	if err := server.RegisterName("mockStore", globalStore); err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+
+	return server, cleanup, nil
+}

cmd/swarm/global-store/global_store_test.go

@@ -0,0 +1,191 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
+)
+
+// TestHTTP_InMemory tests in-memory global store that exposes
+// HTTP server.
+func TestHTTP_InMemory(t *testing.T) {
+	testHTTP(t, true)
+}
+
+// TestHTTP_Database tests global store with persisted database
+// that exposes HTTP server.
+func TestHTTP_Database(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-global-store-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// create a fresh global store
+	testHTTP(t, true, "--dir", dir)
+
+	// check if data saved by the previous global store instance
+	testHTTP(t, false, "--dir", dir)
+}
+
+// testWebsocket starts global store binary with HTTP server
+// and validates that it can store and retrieve data.
+// If put is false, no data will be stored, only retrieved,
+// giving the possibility to check if data is present in the
+// storage directory.
+func testHTTP(t *testing.T, put bool, args ...string) {
+	addr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...)
+	defer testCmd.Interrupt()
+
+	client, err := rpc.DialHTTP("http://" + addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// wait until global store process is started as
+	// rpc.DialHTTP is actually not connecting
+	for i := 0; i < 1000; i++ {
+		_, err = http.DefaultClient.Get("http://" + addr)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	node := store.NewNodeStore(common.HexToAddress("123abc"))
+
+	wantKey := "key"
+	wantValue := "value"
+
+	if put {
+		err = node.Put([]byte(wantKey), []byte(wantValue))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	gotValue, err := node.Get([]byte(wantKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(gotValue) != wantValue {
+		t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
+	}
+}
+
+// TestWebsocket_InMemory tests in-memory global store that exposes
+// WebSocket server.
+func TestWebsocket_InMemory(t *testing.T) {
+	testWebsocket(t, true)
+}
+
+// TestWebsocket_Database tests global store with persisted database
+// that exposes HTTP server.
+func TestWebsocket_Database(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-global-store-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// create a fresh global store
+	testWebsocket(t, true, "--dir", dir)
+
+	// check if data saved by the previous global store instance
+	testWebsocket(t, false, "--dir", dir)
+}
+
+// testWebsocket starts global store binary with WebSocket server
+// and validates that it can store and retrieve data.
+// If put is false, no data will be stored, only retrieved,
+// giving the possibility to check if data is present in the
+// storage directory.
+func testWebsocket(t *testing.T, put bool, args ...string) {
+	addr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
+	defer testCmd.Interrupt()
+
+	var client *rpc.Client
+	var err error
+	// wait until global store process is started
+	for i := 0; i < 1000; i++ {
+		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	node := store.NewNodeStore(common.HexToAddress("123abc"))
+
+	wantKey := "key"
+	wantValue := "value"
+
+	if put {
+		err = node.Put([]byte(wantKey), []byte(wantValue))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	gotValue, err := node.Get([]byte(wantKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(gotValue) != wantValue {
+		t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
+	}
+}
+
+// findFreeTCPAddress returns a local address (IP:Port) to which
+// global store can listen on.
+func findFreeTCPAddress(t *testing.T) (addr string) {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer listener.Close()
+
+	return listener.Addr().String()
+}

cmd/swarm/global-store/main.go

@@ -0,0 +1,104 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/log"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+
+func main() {
+	err := newApp().Run(os.Args)
+	if err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+// newApp construct a new instance of Swarm Global Store.
+// Method Run is called on it in the main function and in tests.
+func newApp() (app *cli.App) {
+	app = utils.NewApp(gitCommit, "Swarm Global Store")
+
+	app.Name = "global-store"
+
+	// app flags (for all commands)
+	app.Flags = []cli.Flag{
+		cli.IntFlag{
+			Name:  "verbosity",
+			Value: 3,
+			Usage: "verbosity level",
+		},
+	}
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "http",
+			Aliases: []string{"h"},
+			Usage:   "start swarm global store with http server",
+			Action:  startHTTP,
+			// Flags only for "start" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.StringFlag{
+					Name:  "dir",
+					Value: "",
+					Usage: "data directory",
+				},
+				cli.StringFlag{
+					Name:  "addr",
+					Value: "0.0.0.0:3033",
+					Usage: "address to listen for http connection",
+				},
+			),
+		},
+		{
+			Name:    "websocket",
+			Aliases: []string{"ws"},
+			Usage:   "start swarm global store with websocket server",
+			Action:  startWS,
+			// Flags only for "start" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.StringFlag{
+					Name:  "dir",
+					Value: "",
+					Usage: "data directory",
+				},
+				cli.StringFlag{
+					Name:  "addr",
+					Value: "0.0.0.0:3033",
+					Usage: "address to listen for websocket connection",
+				},
+				cli.StringSliceFlag{
+					Name:  "origins",
+					Value: &cli.StringSlice{"*"},
+					Usage: "websocket origins",
+				},
+			),
+		},
+	}
+
+	return app
+}

cmd/swarm/global-store/run_test.go

@@ -0,0 +1,49 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
+)
+
+func init() {
+	reexec.Register("swarm-global-store", func() {
+		if err := newApp().Run(os.Args); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	})
+}
+
+func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd {
+	tt := cmdtest.NewTestCmd(t, nil)
+	tt.Run("swarm-global-store", args...)
+	return tt
+}
+
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
+}

cmd/swarm/main.go

@@ -39,13 +39,16 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
 	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
 	"github.com/ethereum/go-ethereum/swarm/tracing"
 	sv "github.com/ethereum/go-ethereum/swarm/version"
 
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 const clientIdentifier = "swarm"
@@ -196,6 +199,7 @@ func init() {
 		SwarmStorePath,
 		SwarmStoreCapacity,
 		SwarmStoreCacheCapacity,
+		SwarmGlobalStoreAPIFlag,
 	}
 	rpcFlags := []cli.Flag{
 		utils.WSEnabledFlag,
@@ -325,8 +329,18 @@ func bzzd(ctx *cli.Context) error {
 func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
 	//define the swarm service boot function
 	boot := func(_ *node.ServiceContext) (node.Service, error) {
-		// In production, mockStore must be always nil.
+		var nodeStore *mock.NodeStore
-		return swarm.NewSwarm(bzzconfig, nil)
+		if bzzconfig.GlobalStoreAPI != "" {
+			// connect to global store
+			client, err := rpc.Dial(bzzconfig.GlobalStoreAPI)
+			if err != nil {
+				return nil, fmt.Errorf("global store: %v", err)
+			}
+			globalStore := mockrpc.NewGlobalStore(client)
+			// create a node store for this swarm key on global store
+			nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey))
+		}
+		return swarm.NewSwarm(bzzconfig, nodeStore)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {
@@ -454,5 +468,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
 		}
 		cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node)
 	}
-	log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes))
+
 }

cmd/utils/flags.go

@@ -332,12 +332,12 @@ var (
 	}
 	CacheTrieFlag = cli.IntFlag{
 		Name:  "cache.trie",
-		Usage: "Percentage of cache memory allowance to use for trie caching",
+		Usage: "Percentage of cache memory allowance to use for trie caching (default = 25% full mode, 50% archive mode)",
 		Value: 25,
 	}
 	CacheGCFlag = cli.IntFlag{
 		Name:  "cache.gc",
-		Usage: "Percentage of cache memory allowance to use for trie pruning",
+		Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)",
 		Value: 25,
 	}
 	TrieCacheGenFlag = cli.IntFlag{
@@ -950,10 +950,11 @@ func makeDatabaseHandles() int {
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if err := fdlimit.Raise(uint64(limit)); err != nil {
+	raised, err := fdlimit.Raise(uint64(limit))
+	if err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
-	return limit / 2 // Leave half for networking and other stuff
+	return int(raised / 2) // Leave half for networking and other stuff
 }
 
 // MakeAddress converts an account specified directly as a hex encoded string or

common/fdlimit/fdlimit_freebsd.go

@@ -26,11 +26,11 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
@@ -38,9 +38,12 @@ func Raise(max uint64) error {
 		limit.Cur = int64(max)
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
-	return nil
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_test.go

@@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
 	if limit, err := Current(); err != nil || limit <= 0 {
 		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
 	}
-	if err := Raise(uint64(target)); err != nil {
+	if _, err := Raise(uint64(target)); err != nil {
 		t.Fatalf("failed to raise file allowance")
 	}
 	if limit, err := Current(); err != nil || limit < target {

common/fdlimit/fdlimit_unix.go

@@ -22,11 +22,12 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
@@ -34,9 +35,13 @@ func Raise(max uint64) error {
 		limit.Cur = max
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
-	return nil
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_windows.go

@@ -20,7 +20,7 @@ import "errors"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// This method is NOP by design:
 	//  * Linux/Darwin counterparts need to manually increase per process limits
 	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
@@ -30,7 +30,7 @@ func Raise(max uint64) error {
 	if max > 16384 {
 		return errors.New("file descriptor limit (16384) reached")
 	}
-	return nil
+	return max, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/size.go

@@ -26,10 +26,10 @@ type StorageSize float64
 
 // String implements the stringer interface.
 func (s StorageSize) String() string {
-	if s > 1000000 {
+	if s > 1048576 {
-		return fmt.Sprintf("%.2f mB", s/1000000)
+		return fmt.Sprintf("%.2f MiB", s/1048576)
-	} else if s > 1000 {
+	} else if s > 1024 {
-		return fmt.Sprintf("%.2f kB", s/1000)
+		return fmt.Sprintf("%.2f KiB", s/1024)
 	} else {
 		return fmt.Sprintf("%.2f B", s)
 	}
@@ -38,10 +38,10 @@ func (s StorageSize) String() string {
 // TerminalString implements log.TerminalStringer, formatting a string for console
 // output during logging.
 func (s StorageSize) TerminalString() string {
-	if s > 1000000 {
+	if s > 1048576 {
-		return fmt.Sprintf("%.2fmB", s/1000000)
+		return fmt.Sprintf("%.2fMiB", s/1048576)
-	} else if s > 1000 {
+	} else if s > 1024 {
-		return fmt.Sprintf("%.2fkB", s/1000)
+		return fmt.Sprintf("%.2fKiB", s/1024)
 	} else {
 		return fmt.Sprintf("%.2fB", s)
 	}

common/size_test.go

@@ -25,8 +25,8 @@ func TestStorageSizeString(t *testing.T) {
 		size StorageSize
 		str  string
 	}{
-		{2381273, "2.38 mB"},
+		{2381273, "2.27 MiB"},
-		{2192, "2.19 kB"},
+		{2192, "2.14 KiB"},
 		{12, "12.00 B"},
 	}
 

core/blockchain.go

@@ -979,20 +979,26 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 				triedb.Cap(limit - ethdb.IdealBatchSize)
 			}
 			// Find the next state trie we need to commit
-			header := bc.GetHeaderByNumber(current - triesInMemory)
+			chosen := current - triesInMemory
-			chosen := header.Number.Uint64()
 
 			// If we exceeded out time allowance, flush an entire trie to disk
 			if bc.gcproc > bc.cacheConfig.TrieTimeLimit {
-				// If we're exceeding limits but haven't reached a large enough memory gap,
+				// If the header is missing (canonical chain behind), we're reorging a low
-				// warn the user that the system is becoming unstable.
+				// diff sidechain. Suspend committing until this operation is completed.
-				if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
+				header := bc.GetHeaderByNumber(chosen)
-					log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+				if header == nil {
+					log.Warn("Reorg in progress, trie commit postponed", "number", chosen)
+				} else {
+					// If we're exceeding limits but haven't reached a large enough memory gap,
+					// warn the user that the system is becoming unstable.
+					if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
+						log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+					}
+					// Flush an entire trie and restart the counters
+					triedb.Commit(header.Root, true)
+					lastWrite = chosen
+					bc.gcproc = 0
 				}
-				// Flush an entire trie and restart the counters
-				triedb.Commit(header.Root, true)
-				lastWrite = chosen
-				bc.gcproc = 0
 			}
 			// Garbage collect anything below our required write retention
 			for !bc.triegc.Empty() {
@@ -1253,8 +1259,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 		stats.processed++
 		stats.usedGas += usedGas
 
-		cache, _ := bc.stateCache.TrieDB().Size()
+		dirty, _ := bc.stateCache.TrieDB().Size()
-		stats.report(chain, it.index, cache)
+		stats.report(chain, it.index, dirty)
 	}
 	// Any blocks remaining here? The only ones we care about are the future ones
 	if block != nil && err == consensus.ErrFutureBlock {
@@ -1324,7 +1330,7 @@ func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (i
 			if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
 				return it.index, nil, nil, err
 			}
-			log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
+			log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(),
 				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
 				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
 				"root", block.Root())

core/blockchain_insert.go

@@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second
 
 // report prints statistics if some number of blocks have been processed
 // or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
+func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize) {
 	// Fetch the timings for the batch
 	var (
 		now     = mclock.Now()
@@ -63,7 +63,7 @@ func (st *insertStats) report(chain []*types.Block, index int, cache common.Stor
 		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
 		}
-		context = append(context, []interface{}{"cache", cache}...)
+		context = append(context, []interface{}{"dirty", dirty}...)
 
 		if st.queued > 0 {
 			context = append(context, []interface{}{"queued", st.queued}...)

core/blockchain_test.go

@@ -1483,3 +1483,58 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
 
 	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
 }
+
+// Tests that importing a very large side fork, which is larger than the canon chain,
+// but where the difficulty per block is kept low: this means that it will not
+// overtake the 'canon' chain until after it's passed canon by about 200 blocks.
+//
+// Details at:
+//  - https://github.com/ethereum/go-ethereum/issues/18977
+//  - https://github.com/ethereum/go-ethereum/pull/18988
+func TestLowDiffLongChain(t *testing.T) {
+	// Generate a canonical chain to act as the main dataset
+	engine := ethash.NewFaker()
+	db := ethdb.NewMemDatabase()
+	genesis := new(Genesis).MustCommit(db)
+
+	// We must use a pretty long chain to ensure that the fork doesn't overtake us
+	// until after at least 128 blocks post tip
+	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*triesInMemory, func(i int, b *BlockGen) {
+		b.SetCoinbase(common.Address{1})
+		b.OffsetTime(-9)
+	})
+
+	// Import the canonical chain
+	diskdb := ethdb.NewMemDatabase()
+	new(Genesis).MustCommit(diskdb)
+
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	if n, err := chain.InsertChain(blocks); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+	}
+	// Generate fork chain, starting from an early block
+	parent := blocks[10]
+	fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*triesInMemory, func(i int, b *BlockGen) {
+		b.SetCoinbase(common.Address{2})
+	})
+
+	// And now import the fork
+	if i, err := chain.InsertChain(fork); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", i, err)
+	}
+	head := chain.CurrentBlock()
+	if got := fork[len(fork)-1].Hash(); got != head.Hash() {
+		t.Fatalf("head wrong, expected %x got %x", head.Hash(), got)
+	}
+	// Sanity check that all the canonical numbers are present
+	header := chain.CurrentHeader()
+	for number := head.NumberU64(); number > 0; number-- {
+		if hash := chain.GetHeaderByNumber(number).Hash(); hash != header.Hash() {
+			t.Fatalf("header %d: canonical hash mismatch: have %x, want %x", number, hash, header.Hash())
+		}
+		header = chain.GetHeader(header.ParentHash, number-1)
+	}
+}

core/chain_makers.go

@@ -149,7 +149,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block {
 // associated difficulty. It's useful to test scenarios where forking is not
 // tied to chain length directly.
 func (b *BlockGen) OffsetTime(seconds int64) {
-	b.header.Time.Add(b.header.Time, new(big.Int).SetInt64(seconds))
+	b.header.Time.Add(b.header.Time, big.NewInt(seconds))
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}

eth/backend.go

@@ -113,6 +113,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		log.Warn("Sanitizing invalid miner gas price", "provided", config.MinerGasPrice, "updated", DefaultConfig.MinerGasPrice)
 		config.MinerGasPrice = new(big.Int).Set(DefaultConfig.MinerGasPrice)
 	}
+	if config.NoPruning && config.TrieDirtyCache > 0 {
+		config.TrieCleanCache += config.TrieDirtyCache
+		config.TrieDirtyCache = 0
+	}
+	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
+
 	// Assemble the Ethereum object
 	chainDb, err := CreateDB(ctx, config, "chaindata")
 	if err != nil {

ethdb/database.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -70,7 +71,7 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
 	if handles < 16 {
 		handles = 16
 	}
-	logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles)
+	logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles)
 
 	// Open the db and recover any potential corruptions
 	db, err := leveldb.OpenFile(file, &opt.Options{

internal/ethapi/api.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -356,11 +357,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArg
 	// Assemble the transaction and sign with the wallet
 	tx := args.toTransaction()
 
-	var chainID *big.Int
+	return wallet.SignTxWithPassphrase(account, passwd, tx, s.b.ChainConfig().ChainID)
-	if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
-		chainID = config.ChainID
-	}
-	return wallet.SignTxWithPassphrase(account, passwd, tx, chainID)
 }
 
 // SendTransaction will create a transaction from the given arguments and
@@ -1186,11 +1183,7 @@ func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transacti
 		return nil, err
 	}
 	// Request the wallet to sign the transaction
-	var chainID *big.Int
+	return wallet.SignTx(account, tx, s.b.ChainConfig().ChainID)
-	if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
-		chainID = config.ChainID
-	}
-	return wallet.SignTx(account, tx, chainID)
 }
 
 // SendTxArgs represents the arguments to sumbit a new transaction into the transaction pool.
@@ -1306,11 +1299,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen
 	// Assemble the transaction and sign with the wallet
 	tx := args.toTransaction()
 
-	var chainID *big.Int
+	signed, err := wallet.SignTx(account, tx, s.b.ChainConfig().ChainID)
-	if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
-		chainID = config.ChainID
-	}
-	signed, err := wallet.SignTx(account, tx, chainID)
 	if err != nil {
 		return common.Hash{}, err
 	}
@@ -1481,6 +1470,45 @@ func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (stri
 	return fmt.Sprintf("%x", encoded), nil
 }
 
+// TestSignCliqueBlock fetches the given block number, and attempts to sign it as a clique header with the
+// given address, returning the address of the recovered signature
+//
+// This is a temporary method to debug the externalsigner integration,
+// TODO: Remove this method when the integration is mature
+func (api *PublicDebugAPI) TestSignCliqueBlock(ctx context.Context, address common.Address, number uint64) (common.Address, error) {
+	block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))
+	if block == nil {
+		return common.Address{}, fmt.Errorf("block #%d not found", number)
+	}
+	header := block.Header()
+	header.Extra = make([]byte, 32+65)
+	encoded := clique.CliqueRLP(header)
+
+	// Look up the wallet containing the requested signer
+	account := accounts.Account{Address: address}
+	wallet, err := api.b.AccountManager().Find(account)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	signature, err := wallet.SignData(account, accounts.MimetypeClique, encoded)
+	if err != nil {
+		return common.Address{}, err
+	}
+	sealHash := clique.SealHash(header).Bytes()
+	log.Info("test signing of clique block",
+		"Sealhash", fmt.Sprintf("%x", sealHash),
+		"signature", fmt.Sprintf("%x", signature))
+	pubkey, err := crypto.Ecrecover(sealHash, signature)
+	if err != nil {
+		return common.Address{}, err
+	}
+	var signer common.Address
+	copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
+
+	return signer, nil
+}
+
 // PrintBlock retrieves a block and returns its pretty printed form.
 func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) {
 	block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))

internal/web3ext/web3ext.go

@@ -231,6 +231,12 @@ web3._extend({
 			call: 'debug_getBlockRlp',
 			params: 1
 		}),
+		new web3._extend.Method({
+			name: 'testSignCliqueBlock',
+			call: 'debug_testSignCliqueBlock',
+			params: 2, 
+			inputFormatters: [web3._extend.formatters.inputAddressFormatter, null],
+		}),
 		new web3._extend.Method({
 			name: 'setHead',
 			call: 'debug_setHead',

p2p/simulations/network.go

@@ -461,7 +461,7 @@ func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node {
 	if l == 0 {
 		return nil
 	}
-	return net.GetNode(filtered[rand.Intn(l)])
+	return net.getNode(filtered[rand.Intn(l)])
 }
 
 func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID {

rpc/stdio.go

@@ -19,6 +19,7 @@ package rpc
 import (
 	"context"
 	"errors"
+	"io"
 	"net"
 	"os"
 	"time"
@@ -26,19 +27,30 @@ import (
 
 // DialStdIO creates a client on stdin/stdout.
 func DialStdIO(ctx context.Context) (*Client, error) {
+	return DialIO(ctx, os.Stdin, os.Stdout)
+}
+
+// DialIO creates a client which uses the given IO channels
+func DialIO(ctx context.Context, in io.Reader, out io.Writer) (*Client, error) {
 	return newClient(ctx, func(_ context.Context) (ServerCodec, error) {
-		return NewJSONCodec(stdioConn{}), nil
+		return NewJSONCodec(stdioConn{
+			in:  in,
+			out: out,
+		}), nil
 	})
 }
 
-type stdioConn struct{}
+type stdioConn struct {
+	in  io.Reader
+	out io.Writer
+}
 
 func (io stdioConn) Read(b []byte) (n int, err error) {
-	return os.Stdin.Read(b)
+	return io.in.Read(b)
 }
 
 func (io stdioConn) Write(b []byte) (n int, err error) {
-	return os.Stdout.Write(b)
+	return io.out.Write(b)
 }
 
 func (io stdioConn) Close() error {

signer/core/api.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"reflect"
 
@@ -39,9 +38,9 @@ const (
 	// numberOfAccountsToDerive For hardware wallets, the number of accounts to derive
 	numberOfAccountsToDerive = 10
 	// ExternalAPIVersion -- see extapi_changelog.md
-	ExternalAPIVersion = "5.0.0"
+	ExternalAPIVersion = "6.0.0"
 	// InternalAPIVersion -- see intapi_changelog.md
-	InternalAPIVersion = "3.1.0"
+	InternalAPIVersion = "4.0.0"
 )
 
 // ExternalAPI defines the external API through which signing requests are made.
@@ -49,7 +48,7 @@ type ExternalAPI interface {
 	// List available accounts
 	List(ctx context.Context) ([]common.Address, error)
 	// New request to create a new account
-	New(ctx context.Context) (accounts.Account, error)
+	New(ctx context.Context) (common.Address, error)
 	// SignTransaction request to sign the specified transaction
 	SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error)
 	// SignData - request to sign the given data (plus prefix)
@@ -58,17 +57,13 @@ type ExternalAPI interface {
 	SignTypedData(ctx context.Context, addr common.MixedcaseAddress, data TypedData) (hexutil.Bytes, error)
 	// EcRecover - recover public key from given message and signature
 	EcRecover(ctx context.Context, data hexutil.Bytes, sig hexutil.Bytes) (common.Address, error)
-	// Export - request to export an account
+	// Version info about the APIs
-	Export(ctx context.Context, addr common.Address) (json.RawMessage, error)
-	// Import - request to import an account
-	// Should be moved to Internal API, in next phase when we have
-	// bi-directional communication
-	//Import(ctx context.Context, keyJSON json.RawMessage) (Account, error)
 	Version(ctx context.Context) (string, error)
 }
 
-// SignerUI specifies what method a UI needs to implement to be able to be used as a UI for the signer
+// UIClientAPI specifies what method a UI needs to implement to be able to be used as a
-type SignerUI interface {
+// UI for the signer
+type UIClientAPI interface {
 	// ApproveTx prompt the user for confirmation to request to sign Transaction
 	ApproveTx(request *SignTxRequest) (SignTxResponse, error)
 	// ApproveSignData prompt the user for confirmation to request to sign data
@@ -95,13 +90,15 @@ type SignerUI interface {
 	// OnInputRequired is invoked when clef requires user input, for example master password or
 	// pin-code for unlocking hardware wallets
 	OnInputRequired(info UserInputRequest) (UserInputResponse, error)
+	// RegisterUIServer tells the UI to use the given UIServerAPI for ui->clef communication
+	RegisterUIServer(api *UIServerAPI)
 }
 
 // SignerAPI defines the actual implementation of ExternalAPI
 type SignerAPI struct {
 	chainID    *big.Int
 	am         *accounts.Manager
-	UI         SignerUI
+	UI         UIClientAPI
 	validator  *Validator
 	rejectMode bool
 }
@@ -115,6 +112,37 @@ type Metadata struct {
 	Origin    string `json:"Origin"`
 }
 
+func StartClefAccountManager(ksLocation string, nousb, lightKDF bool) *accounts.Manager {
+	var (
+		backends []accounts.Backend
+		n, p     = keystore.StandardScryptN, keystore.StandardScryptP
+	)
+	if lightKDF {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	// support password based accounts
+	if len(ksLocation) > 0 {
+		backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
+	}
+	if !nousb {
+		// Start a USB hub for Ledger hardware wallets
+		if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
+			log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
+		} else {
+			backends = append(backends, ledgerhub)
+			log.Debug("Ledger support enabled")
+		}
+		// Start a USB hub for Trezor hardware wallets
+		if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
+			log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
+		} else {
+			backends = append(backends, trezorhub)
+			log.Debug("Trezor support enabled")
+		}
+	}
+	return accounts.NewManager(backends...)
+}
+
 // MetadataFromContext extracts Metadata from a given context.Context
 func MetadataFromContext(ctx context.Context) Metadata {
 	m := Metadata{"NA", "NA", "NA", "", ""} // batman
@@ -199,11 +227,11 @@ type (
 		Password string `json:"password"`
 	}
 	ListRequest struct {
-		Accounts []Account `json:"accounts"`
+		Accounts []accounts.Account `json:"accounts"`
-		Meta     Metadata  `json:"meta"`
+		Meta     Metadata           `json:"meta"`
 	}
 	ListResponse struct {
-		Accounts []Account `json:"accounts"`
+		Accounts []accounts.Account `json:"accounts"`
 	}
 	Message struct {
 		Text string `json:"text"`
@@ -234,38 +262,11 @@ var ErrRequestDenied = errors.New("Request denied")
 // key that is generated when a new Account is created.
 // noUSB disables USB support that is required to support hardware devices such as
 // ledger and trezor.
-func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abidb *AbiDb, lightKDF bool, advancedMode bool) *SignerAPI {
+func NewSignerAPI(am *accounts.Manager, chainID int64, noUSB bool, ui UIClientAPI, abidb *AbiDb, advancedMode bool) *SignerAPI {
-	var (
-		backends []accounts.Backend
-		n, p     = keystore.StandardScryptN, keystore.StandardScryptP
-	)
-	if lightKDF {
-		n, p = keystore.LightScryptN, keystore.LightScryptP
-	}
-	// support password based accounts
-	if len(ksLocation) > 0 {
-		backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
-	}
 	if advancedMode {
 		log.Info("Clef is in advanced mode: will warn instead of reject")
 	}
-	if !noUSB {
+	signer := &SignerAPI{big.NewInt(chainID), am, ui, NewValidator(abidb), !advancedMode}
-		// Start a USB hub for Ledger hardware wallets
-		if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
-			log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
-		} else {
-			backends = append(backends, ledgerhub)
-			log.Debug("Ledger support enabled")
-		}
-		// Start a USB hub for Trezor hardware wallets
-		if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
-			log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
-		} else {
-			backends = append(backends, trezorhub)
-			log.Debug("Trezor support enabled")
-		}
-	}
-	signer := &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb), !advancedMode}
 	if !noUSB {
 		signer.startUSBListener()
 	}
@@ -358,12 +359,9 @@ func (api *SignerAPI) startUSBListener() {
 // List returns the set of wallet this signer manages. Each wallet can contain
 // multiple accounts.
 func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
-	var accs []Account
+	var accs []accounts.Account
 	for _, wallet := range api.am.Wallets() {
-		for _, acc := range wallet.Accounts() {
+		accs = append(accs, wallet.Accounts()...)
-			acc := Account{Typ: "Account", URL: wallet.URL(), Address: acc.Address}
-			accs = append(accs, acc)
-		}
 	}
 	result, err := api.UI.ApproveListing(&ListRequest{Accounts: accs, Meta: MetadataFromContext(ctx)})
 	if err != nil {
@@ -373,7 +371,6 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
 		return nil, ErrRequestDenied
 
 	}
-
 	addresses := make([]common.Address, 0)
 	for _, acc := range result.Accounts {
 		addresses = append(addresses, acc.Address)
@@ -385,10 +382,10 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
 // New creates a new password protected Account. The private key is protected with
 // the given password. Users are responsible to backup the private key that is stored
 // in the keystore location thas was specified when this API was created.
-func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
+func (api *SignerAPI) New(ctx context.Context) (common.Address, error) {
 	be := api.am.Backends(keystore.KeyStoreType)
 	if len(be) == 0 {
-		return accounts.Account{}, errors.New("password based accounts not supported")
+		return common.Address{}, errors.New("password based accounts not supported")
 	}
 	var (
 		resp NewAccountResponse
@@ -398,20 +395,21 @@ func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
 	for i := 0; i < 3; i++ {
 		resp, err = api.UI.ApproveNewAccount(&NewAccountRequest{MetadataFromContext(ctx)})
 		if err != nil {
-			return accounts.Account{}, err
+			return common.Address{}, err
 		}
 		if !resp.Approved {
-			return accounts.Account{}, ErrRequestDenied
+			return common.Address{}, ErrRequestDenied
 		}
 		if pwErr := ValidatePasswordFormat(resp.Password); pwErr != nil {
 			api.UI.ShowError(fmt.Sprintf("Account creation attempt #%d failed due to password requirements: %v", (i + 1), pwErr))
 		} else {
 			// No error
-			return be[0].(*keystore.KeyStore).NewAccount(resp.Password)
+			acc, err := be[0].(*keystore.KeyStore).NewAccount(resp.Password)
+			return acc.Address, err
 		}
 	}
 	// Otherwise fail, with generic error message
-	return accounts.Account{}, errors.New("account creation failed")
+	return common.Address{}, errors.New("account creation failed")
 }
 
 // logDiff logs the difference between the incoming (original) transaction and the one returned from the signer.
@@ -521,57 +519,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth
 
 }
 
-// Export returns encrypted private key associated with the given address in web3 keystore format.
-func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
-	res, err := api.UI.ApproveExport(&ExportRequest{Address: addr, Meta: MetadataFromContext(ctx)})
-
-	if err != nil {
-		return nil, err
-	}
-	if !res.Approved {
-		return nil, ErrRequestDenied
-	}
-	// Look up the wallet containing the requested signer
-	wallet, err := api.am.Find(accounts.Account{Address: addr})
-	if err != nil {
-		return nil, err
-	}
-	if wallet.URL().Scheme != keystore.KeyStoreScheme {
-		return nil, fmt.Errorf("Account is not a keystore-account")
-	}
-	return ioutil.ReadFile(wallet.URL().Path)
-}
-
-// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
-// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
-// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
-// OBS! This method is removed from the public API. It should not be exposed on the external API
-// for a couple of reasons:
-// 1. Even though it is encrypted, it should still be seen as sensitive data
-// 2. It can be used to DoS clef, by using malicious data with e.g. extreme large
-// values for the kdfparams.
-func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) {
-	be := api.am.Backends(keystore.KeyStoreType)
-
-	if len(be) == 0 {
-		return Account{}, errors.New("password based accounts not supported")
-	}
-	res, err := api.UI.ApproveImport(&ImportRequest{Meta: MetadataFromContext(ctx)})
-
-	if err != nil {
-		return Account{}, err
-	}
-	if !res.Approved {
-		return Account{}, ErrRequestDenied
-	}
-	acc, err := be[0].(*keystore.KeyStore).Import(keyJSON, res.OldPassword, res.NewPassword)
-	if err != nil {
-		api.UI.ShowError(err.Error())
-		return Account{}, err
-	}
-	return Account{Typ: "Account", URL: acc.URL, Address: acc.Address}, nil
-}
-
 // Returns the external api version. This method does not require user acceptance. Available methods are
 // available via enumeration anyway, and this info does not contain user-specific data
 func (api *SignerAPI) Version(ctx context.Context) (string, error) {

signer/core/api_test.go

@@ -28,6 +28,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -47,6 +48,8 @@ func (ui *HeadlessUI) OnInputRequired(info UserInputRequest) (UserInputResponse,
 
 func (ui *HeadlessUI) OnSignerStartup(info StartupInfo) {
 }
+func (ui *HeadlessUI) RegisterUIServer(api *UIServerAPI) {
+}
 
 func (ui *HeadlessUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
 	fmt.Printf("OnApproved()\n")
@@ -91,7 +94,7 @@ func (ui *HeadlessUI) ApproveListing(request *ListRequest) (ListResponse, error)
 	case "A":
 		return ListResponse{request.Accounts}, nil
 	case "1":
-		l := make([]Account, 1)
+		l := make([]accounts.Account, 1)
 		l[0] = request.Accounts[1]
 		return ListResponse{l}, nil
 	default:
@@ -138,13 +141,8 @@ func setup(t *testing.T) (*SignerAPI, chan string) {
 	}
 	var (
 		ui  = &HeadlessUI{controller}
-		api = NewSignerAPI(
+		am  = StartClefAccountManager(tmpDirName(t), true, true)
-			1,
+		api = NewSignerAPI(am, 1337, true, ui, db, true)
-			tmpDirName(t),
-			true,
-			ui,
-			db,
-			true, true)
 	)
 	return api, controller
 }
@@ -169,22 +167,22 @@ func failCreateAccountWithPassword(control chan string, api *SignerAPI, password
 	control <- "Y"
 	control <- password
 
-	acc, err := api.New(context.Background())
+	addr, err := api.New(context.Background())
 	if err == nil {
 		t.Fatal("Should have returned an error")
 	}
-	if acc.Address != (common.Address{}) {
+	if addr != (common.Address{}) {
 		t.Fatal("Empty address should be returned")
 	}
 }
 
 func failCreateAccount(control chan string, api *SignerAPI, t *testing.T) {
 	control <- "N"
-	acc, err := api.New(context.Background())
+	addr, err := api.New(context.Background())
 	if err != ErrRequestDenied {
 		t.Fatal(err)
 	}
-	if acc.Address != (common.Address{}) {
+	if addr != (common.Address{}) {
 		t.Fatal("Empty address should be returned")
 	}
 }

signer/core/auditlog.go

@@ -18,9 +18,7 @@ package core
 
 import (
 	"context"
-	"encoding/json"
 
-	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
@@ -40,7 +38,7 @@ func (l *AuditLogger) List(ctx context.Context) ([]common.Address, error) {
 	return res, e
 }
 
-func (l *AuditLogger) New(ctx context.Context) (accounts.Account, error) {
+func (l *AuditLogger) New(ctx context.Context) (common.Address, error) {
 	return l.api.New(ctx)
 }
 
@@ -86,15 +84,6 @@ func (l *AuditLogger) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex
 	return b, e
 }
 
-func (l *AuditLogger) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
-	l.log.Info("Export", "type", "request", "metadata", MetadataFromContext(ctx).String(),
-		"addr", addr.Hex())
-	j, e := l.api.Export(ctx, addr)
-	// In this case, we don't actually log the json-response, which may be extra sensitive
-	l.log.Info("Export", "type", "response", "json response size", len(j), "error", e)
-	return j, e
-}
-
 func (l *AuditLogger) Version(ctx context.Context) (string, error) {
 	l.log.Info("Version", "type", "request", "metadata", MetadataFromContext(ctx).String())
 	data, err := l.api.Version(ctx)

signer/core/cliui.go

@@ -18,12 +18,12 @@ package core
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
 	"sync"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
@@ -39,6 +39,10 @@ func NewCommandlineUI() *CommandlineUI {
 	return &CommandlineUI{in: bufio.NewReader(os.Stdin)}
 }
 
+func (ui *CommandlineUI) RegisterUIServer(api *UIServerAPI) {
+	// noop
+}
+
 // readString reads a single line from stdin, trimming if from spaces, enforcing
 // non-emptyness.
 func (ui *CommandlineUI) readString() string {
@@ -223,7 +227,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err
 	for _, account := range request.Accounts {
 		fmt.Printf("  [x] %v\n", account.Address.Hex())
 		fmt.Printf("    URL: %v\n", account.URL)
-		fmt.Printf("    Type: %v\n", account.Typ)
 	}
 	fmt.Printf("-------------------------------------------\n")
 	showMetadata(request.Meta)
@@ -264,7 +267,11 @@ func (ui *CommandlineUI) ShowInfo(message string) {
 
 func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
 	fmt.Printf("Transaction signed:\n ")
-	spew.Dump(tx.Tx)
+	if jsn, err := json.MarshalIndent(tx.Tx, "  ", "  "); err != nil {
+		fmt.Printf("WARN: marshalling error %v\n", err)
+	} else {
+		fmt.Println(string(jsn))
+	}
 }
 
 func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) {

signer/core/signed_data.go

@@ -121,8 +121,8 @@ var typedDataReferenceTypeRegexp = regexp.MustCompile(`^[A-Z](\w*)(\[\])?$`)
 // sign receives a request and produces a signature
 
 // Note, the produced signature conforms to the secp256k1 curve R, S and V values,
-// where the V value will be 27 or 28 for legacy reasons.
+// where the V value will be 27 or 28 for legacy reasons, if legacyV==true.
-func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (hexutil.Bytes, error) {
+func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest, legacyV bool) (hexutil.Bytes, error) {
 
 	// We make the request prior to looking up if we actually have the account, to prevent
 	// account-enumeration via the API
@@ -140,11 +140,13 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (
 		return nil, err
 	}
 	// Sign the data with the wallet
-	signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Hash)
+	signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Rawdata)
 	if err != nil {
 		return nil, err
 	}
-	signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
+	if legacyV {
+		signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
+	}
 	return signature, nil
 }
 
@@ -153,17 +155,16 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (
 //
 // Different types of validation occur.
 func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (hexutil.Bytes, error) {
-	var req, err = api.determineSignatureFormat(ctx, contentType, addr, data)
+	var req, transformV, err = api.determineSignatureFormat(ctx, contentType, addr, data)
 	if err != nil {
 		return nil, err
 	}
 
-	signature, err := api.sign(addr, req)
+	signature, err := api.sign(addr, req, transformV)
 	if err != nil {
 		api.UI.ShowError(err.Error())
 		return nil, err
 	}
-
 	return signature, nil
 }
 
@@ -173,12 +174,14 @@ func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr com
 // charset, ok := params["charset"]
 // As it is now, we accept any charset and just treat it as 'raw'.
 // This method returns the mimetype for signing along with the request
-func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, error) {
+func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, bool, error) {
-	var req *SignDataRequest
+	var (
-
+		req          *SignDataRequest
+		useEthereumV = true // Default to use V = 27 or 28, the legacy Ethereum format
+	)
 	mediaType, _, err := mime.ParseMediaType(contentType)
 	if err != nil {
-		return nil, err
+		return nil, useEthereumV, err
 	}
 
 	switch mediaType {
@@ -186,7 +189,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType
 		// Data with an intended validator
 		validatorData, err := UnmarshalValidatorData(data)
 		if err != nil {
-			return nil, err
+			return nil, useEthereumV, err
 		}
 		sighash, msg := SignTextValidator(validatorData)
 		message := []*NameValueType{
@@ -201,55 +204,63 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType
 		// Clique is the Ethereum PoA standard
 		stringData, ok := data.(string)
 		if !ok {
-			return nil, fmt.Errorf("input for %v plain must be an hex-encoded string", ApplicationClique.Mime)
+			return nil, useEthereumV, fmt.Errorf("input for %v must be an hex-encoded string", ApplicationClique.Mime)
 		}
 		cliqueData, err := hexutil.Decode(stringData)
 		if err != nil {
-			return nil, err
+			return nil, useEthereumV, err
 		}
 		header := &types.Header{}
 		if err := rlp.DecodeBytes(cliqueData, header); err != nil {
-			return nil, err
+			return nil, useEthereumV, err
+		}
+		// The incoming clique header is already truncated, sent to us with a extradata already shortened
+		if len(header.Extra) < 65 {
+			// Need to add it back, to get a suitable length for hashing
+			newExtra := make([]byte, len(header.Extra)+65)
+			copy(newExtra, header.Extra)
+			header.Extra = newExtra
 		}
 		// Get back the rlp data, encoded by us
-		cliqueData = clique.CliqueRLP(header)
+		sighash, cliqueRlp, err := cliqueHeaderHashAndRlp(header)
-		sighash, err := SignCliqueHeader(header)
 		if err != nil {
-			return nil, err
+			return nil, useEthereumV, err
 		}
 		message := []*NameValueType{
 			{
-				Name:  "Clique block",
+				Name:  "Clique header",
 				Typ:   "clique",
-				Value: fmt.Sprintf("clique block %d [0x%x]", header.Number, header.Hash()),
+				Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()),
 			},
 		}
-		req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueData, Message: message, Hash: sighash}
+		// Clique uses V on the form 0 or 1
+		useEthereumV = false
+		req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Message: message, Hash: sighash}
 	default: // also case TextPlain.Mime:
 		// Calculates an Ethereum ECDSA signature for:
 		// hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}")
 		// We expect it to be a string
-		stringData, ok := data.(string)
+		if stringData, ok := data.(string); !ok {
-		if !ok {
+			return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string")
-			return nil, fmt.Errorf("input for text/plain must be a string")
+		} else {
+			if textData, err := hexutil.Decode(stringData); err != nil {
+				return nil, useEthereumV, err
+			} else {
+				sighash, msg := accounts.TextAndHash(textData)
+				message := []*NameValueType{
+					{
+						Name:  "message",
+						Typ:   "text/plain",
+						Value: msg,
+					},
+				}
+				req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash}
+			}
 		}
-		//plainData, err := hexutil.Decode(stringdata)
-		//if err != nil {
-		//	return nil, err
-		//}
-		sighash, msg := accounts.TextAndHash([]byte(stringData))
-		message := []*NameValueType{
-			{
-				Name:  "message",
-				Typ:   "text/plain",
-				Value: msg,
-			},
-		}
-		req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash}
 	}
 	req.Address = addr
 	req.Meta = MetadataFromContext(ctx)
-	return req, nil
+	return req, useEthereumV, nil
 
 }
 
@@ -262,20 +273,21 @@ func SignTextValidator(validatorData ValidatorData) (hexutil.Bytes, string) {
 	return crypto.Keccak256([]byte(msg)), msg
 }
 
-// SignCliqueHeader returns the hash which is used as input for the proof-of-authority
+// cliqueHeaderHashAndRlp returns the hash which is used as input for the proof-of-authority
 // signing. It is the hash of the entire header apart from the 65 byte signature
 // contained at the end of the extra data.
 //
 // The method requires the extra data to be at least 65 bytes -- the original implementation
 // in clique.go panics if this is the case, thus it's been reimplemented here to avoid the panic
 // and simply return an error instead
-func SignCliqueHeader(header *types.Header) (hexutil.Bytes, error) {
+func cliqueHeaderHashAndRlp(header *types.Header) (hash, rlp []byte, err error) {
-	//hash := common.Hash{}
 	if len(header.Extra) < 65 {
-		return nil, fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra))
+		err = fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra))
+		return
 	}
-	hash := clique.SealHash(header)
+	rlp = clique.CliqueRLP(header)
-	return hash.Bytes(), nil
+	hash = clique.SealHash(header).Bytes()
+	return hash, rlp, err
 }
 
 // SignTypedData signs EIP-712 conformant typed data
@@ -293,7 +305,7 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd
 	sighash := crypto.Keccak256(rawData)
 	message := typedData.Format()
 	req := &SignDataRequest{ContentType: DataTyped.Mime, Rawdata: rawData, Message: message, Hash: sighash}
-	signature, err := api.sign(addr, req)
+	signature, err := api.sign(addr, req, true)
 	if err != nil {
 		api.UI.ShowError(err.Error())
 		return nil, err
@@ -722,7 +734,7 @@ func (nvt *NameValueType) Pprint(depth int) string {
 			output.WriteString(sublevel)
 		}
 	} else {
-		output.WriteString(fmt.Sprintf("%s\n", nvt.Value))
+		output.WriteString(fmt.Sprintf("%q\n", nvt.Value))
 	}
 	return output.String()
 }

signer/core/stdioui.go

@@ -32,12 +32,16 @@ type StdIOUI struct {
 }
 
 func NewStdIOUI() *StdIOUI {
-	log.Info("NewStdIOUI")
 	client, err := rpc.DialContext(context.Background(), "stdio://")
 	if err != nil {
 		log.Crit("Could not create stdio client", "err", err)
 	}
-	return &StdIOUI{client: *client}
+	ui := &StdIOUI{client: *client}
+	return ui
+}
+
+func (ui *StdIOUI) RegisterUIServer(api *UIServerAPI) {
+	ui.client.RegisterName("clef", api)
 }
 
 // dispatch sends a request over the stdio
@@ -49,6 +53,16 @@ func (ui *StdIOUI) dispatch(serviceMethod string, args interface{}, reply interf
 	return err
 }
 
+// notify sends a request over the stdio, and does not listen for a response
+func (ui *StdIOUI) notify(serviceMethod string, args interface{}) error {
+	ctx := context.Background()
+	err := ui.client.Notify(ctx, serviceMethod, args)
+	if err != nil {
+		log.Info("Error", "exc", err.Error())
+	}
+	return err
+}
+
 func (ui *StdIOUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) {
 	var result SignTxResponse
 	err := ui.dispatch("ApproveTx", request, &result)
@@ -86,27 +100,27 @@ func (ui *StdIOUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResp
 }
 
 func (ui *StdIOUI) ShowError(message string) {
-	err := ui.dispatch("ShowError", &Message{message}, nil)
+	err := ui.notify("ShowError", &Message{message})
 	if err != nil {
 		log.Info("Error calling 'ShowError'", "exc", err.Error(), "msg", message)
 	}
 }
 
 func (ui *StdIOUI) ShowInfo(message string) {
-	err := ui.dispatch("ShowInfo", Message{message}, nil)
+	err := ui.notify("ShowInfo", Message{message})
 	if err != nil {
 		log.Info("Error calling 'ShowInfo'", "exc", err.Error(), "msg", message)
 	}
 }
 func (ui *StdIOUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
-	err := ui.dispatch("OnApprovedTx", tx, nil)
+	err := ui.notify("OnApprovedTx", tx)
 	if err != nil {
 		log.Info("Error calling 'OnApprovedTx'", "exc", err.Error(), "tx", tx)
 	}
 }
 
 func (ui *StdIOUI) OnSignerStartup(info StartupInfo) {
-	err := ui.dispatch("OnSignerStartup", info, nil)
+	err := ui.notify("OnSignerStartup", info)
 	if err != nil {
 		log.Info("Error calling 'OnSignerStartup'", "exc", err.Error(), "info", info)
 	}

signer/core/types.go

@@ -22,36 +22,11 @@ import (
 	"math/big"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
-type Accounts []Account
-
-func (as Accounts) String() string {
-	var output []string
-	for _, a := range as {
-		output = append(output, a.String())
-	}
-	return strings.Join(output, "\n")
-}
-
-type Account struct {
-	Typ     string         `json:"type"`
-	URL     accounts.URL   `json:"url"`
-	Address common.Address `json:"address"`
-}
-
-func (a Account) String() string {
-	s, err := json.Marshal(a)
-	if err == nil {
-		return string(s)
-	}
-	return err.Error()
-}
-
 type ValidationInfo struct {
 	Typ     string `json:"type"`
 	Message string `json:"message"`

signer/core/uiapi.go

@@ -0,0 +1,201 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package core
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// SignerUIAPI implements methods Clef provides for a UI to query, in the bidirectional communication
+// channel.
+// This API is considered secure, since a request can only
+// ever arrive from the UI -- and the UI is capable of approving any action, thus we can consider these
+// requests pre-approved.
+// NB: It's very important that these methods are not ever exposed on the external service
+// registry.
+type UIServerAPI struct {
+	extApi *SignerAPI
+	am     *accounts.Manager
+}
+
+// NewUIServerAPI creates a new UIServerAPI
+func NewUIServerAPI(extapi *SignerAPI) *UIServerAPI {
+	return &UIServerAPI{extapi, extapi.am}
+}
+
+// List available accounts. As opposed to the external API definition, this method delivers
+// the full Account object and not only Address.
+// Example call
+// {"jsonrpc":"2.0","method":"clef_listAccounts","params":[], "id":4}
+func (s *UIServerAPI) ListAccounts(ctx context.Context) ([]accounts.Account, error) {
+	var accs []accounts.Account
+	for _, wallet := range s.am.Wallets() {
+		accs = append(accs, wallet.Accounts()...)
+	}
+	return accs, nil
+}
+
+// rawWallet is a JSON representation of an accounts.Wallet interface, with its
+// data contents extracted into plain fields.
+type rawWallet struct {
+	URL      string             `json:"url"`
+	Status   string             `json:"status"`
+	Failure  string             `json:"failure,omitempty"`
+	Accounts []accounts.Account `json:"accounts,omitempty"`
+}
+
+// ListWallets will return a list of wallets that clef manages
+// Example call
+// {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5}
+func (s *UIServerAPI) ListWallets() []rawWallet {
+	wallets := make([]rawWallet, 0) // return [] instead of nil if empty
+	for _, wallet := range s.am.Wallets() {
+		status, failure := wallet.Status()
+
+		raw := rawWallet{
+			URL:      wallet.URL().String(),
+			Status:   status,
+			Accounts: wallet.Accounts(),
+		}
+		if failure != nil {
+			raw.Failure = failure.Error()
+		}
+		wallets = append(wallets, raw)
+	}
+	return wallets
+}
+
+// DeriveAccount requests a HD wallet to derive a new account, optionally pinning
+// it for later reuse.
+// Example call
+// {"jsonrpc":"2.0","method":"clef_deriveAccount","params":["ledger://","m/44'/60'/0'", false], "id":6}
+func (s *UIServerAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) {
+	wallet, err := s.am.Wallet(url)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	derivPath, err := accounts.ParseDerivationPath(path)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	if pin == nil {
+		pin = new(bool)
+	}
+	return wallet.Derive(derivPath, *pin)
+}
+
+// fetchKeystore retrives the encrypted keystore from the account manager.
+func fetchKeystore(am *accounts.Manager) *keystore.KeyStore {
+	return am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+}
+
+// ImportRawKey stores the given hex encoded ECDSA key into the key directory,
+// encrypting it with the passphrase.
+// Example call (should fail on password too short)
+// {"jsonrpc":"2.0","method":"clef_importRawKey","params":["1111111111111111111111111111111111111111111111111111111111111111","test"], "id":6}
+func (s *UIServerAPI) ImportRawKey(privkey string, password string) (accounts.Account, error) {
+	key, err := crypto.HexToECDSA(privkey)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	if err := ValidatePasswordFormat(password); err != nil {
+		return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err)
+	}
+	// No error
+	return fetchKeystore(s.am).ImportECDSA(key, password)
+}
+
+// OpenWallet initiates a hardware wallet opening procedure, establishing a USB
+// connection and attempting to authenticate via the provided passphrase. Note,
+// the method may return an extra challenge requiring a second open (e.g. the
+// Trezor PIN matrix challenge).
+// Example
+// {"jsonrpc":"2.0","method":"clef_openWallet","params":["ledger://",""], "id":6}
+func (s *UIServerAPI) OpenWallet(url string, passphrase *string) error {
+	wallet, err := s.am.Wallet(url)
+	if err != nil {
+		return err
+	}
+	pass := ""
+	if passphrase != nil {
+		pass = *passphrase
+	}
+	return wallet.Open(pass)
+}
+
+// ChainId returns the chainid in use for Eip-155 replay protection
+// Example call
+// {"jsonrpc":"2.0","method":"clef_chainId","params":[], "id":8}
+func (s *UIServerAPI) ChainId() math.HexOrDecimal64 {
+	return (math.HexOrDecimal64)(s.extApi.chainID.Uint64())
+}
+
+// SetChainId sets the chain id to use when signing transactions.
+// Example call to set Ropsten:
+// {"jsonrpc":"2.0","method":"clef_setChainId","params":["3"], "id":8}
+func (s *UIServerAPI) SetChainId(id math.HexOrDecimal64) math.HexOrDecimal64 {
+	s.extApi.chainID = new(big.Int).SetUint64(uint64(id))
+	return s.ChainId()
+}
+
+// Export returns encrypted private key associated with the given address in web3 keystore format.
+// Example
+// {"jsonrpc":"2.0","method":"clef_export","params":["0x19e7e376e7c213b7e7e7e46cc70a5dd086daff2a"], "id":4}
+func (s *UIServerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
+	// Look up the wallet containing the requested signer
+	wallet, err := s.am.Find(accounts.Account{Address: addr})
+	if err != nil {
+		return nil, err
+	}
+	if wallet.URL().Scheme != keystore.KeyStoreScheme {
+		return nil, fmt.Errorf("Account is not a keystore-account")
+	}
+	return ioutil.ReadFile(wallet.URL().Path)
+}
+
+// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
+// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
+// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
+// Example (the address in question has privkey `11...11`):
+// {"jsonrpc":"2.0","method":"clef_import","params":[{"address":"19e7e376e7c213b7e7e7e46cc70a5dd086daff2a","crypto":{"cipher":"aes-128-ctr","ciphertext":"33e4cd3756091d037862bb7295e9552424a391a6e003272180a455ca2a9fb332","cipherparams":{"iv":"b54b263e8f89c42bb219b6279fba5cce"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"e4ca94644fd30569c1b1afbbc851729953c92637b7fe4bb9840bbb31ffbc64a5"},"mac":"f4092a445c2b21c0ef34f17c9cd0d873702b2869ec5df4439a0c2505823217e7"},"id":"216c7eac-e8c1-49af-a215-fa0036f29141","version":3},"test","yaddayadda"], "id":4}
+func (api *UIServerAPI) Import(ctx context.Context, keyJSON json.RawMessage, oldPassphrase, newPassphrase string) (accounts.Account, error) {
+	be := api.am.Backends(keystore.KeyStoreType)
+
+	if len(be) == 0 {
+		return accounts.Account{}, errors.New("password based accounts not supported")
+	}
+	if err := ValidatePasswordFormat(newPassphrase); err != nil {
+		return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err)
+	}
+	return be[0].(*keystore.KeyStore).Import(keyJSON, oldPassphrase, newPassphrase)
+}
+
+// Other methods to be added, not yet implemented are:
+// - Ruleset interaction: add rules, attest rulefiles
+// - Store metadata about accounts, e.g. naming of accounts

signer/rules/rules.go

@@ -46,16 +46,16 @@ func consoleOutput(call otto.FunctionCall) otto.Value {
 	return otto.Value{}
 }
 
-// rulesetUI provides an implementation of SignerUI that evaluates a javascript
+// rulesetUI provides an implementation of UIClientAPI that evaluates a javascript
 // file for each defined UI-method
 type rulesetUI struct {
-	next        core.SignerUI // The next handler, for manual processing
+	next        core.UIClientAPI // The next handler, for manual processing
 	storage     storage.Storage
 	credentials storage.Storage
 	jsRules     string // The rules to use
 }
 
-func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) {
+func NewRuleEvaluator(next core.UIClientAPI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) {
 	c := &rulesetUI{
 		next:        next,
 		storage:     jsbackend,
@@ -65,6 +65,9 @@ func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.
 
 	return c, nil
 }
+func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) {
+	// TODO, make it possible to query from js
+}
 
 func (r *rulesetUI) Init(javascriptRules string) error {
 	r.jsRules = javascriptRules

signer/rules/rules_test.go

@@ -77,6 +77,8 @@ type alwaysDenyUI struct{}
 func (alwaysDenyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) {
 	return core.UserInputResponse{}, nil
 }
+func (alwaysDenyUI) RegisterUIServer(api *core.UIServerAPI) {
+}
 
 func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) {
 }
@@ -133,11 +135,11 @@ func initRuleEngine(js string) (*rulesetUI, error) {
 }
 
 func TestListRequest(t *testing.T) {
-	accs := make([]core.Account, 5)
+	accs := make([]accounts.Account, 5)
 
 	for i := range accs {
 		addr := fmt.Sprintf("000000000000000000000000000000000000000%x", i)
-		acc := core.Account{
+		acc := accounts.Account{
 			Address: common.BytesToAddress(common.Hex2Bytes(addr)),
 			URL:     accounts.URL{Scheme: "test", Path: fmt.Sprintf("acc-%d", i)},
 		}
@@ -208,6 +210,10 @@ type dummyUI struct {
 	calls []string
 }
 
+func (d *dummyUI) RegisterUIServer(api *core.UIServerAPI) {
+	panic("implement me")
+}
+
 func (d *dummyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) {
 	d.calls = append(d.calls, "OnInputRequired")
 	return core.UserInputResponse{}, nil
@@ -531,6 +537,8 @@ func (d *dontCallMe) OnInputRequired(info core.UserInputRequest) (core.UserInput
 	d.t.Fatalf("Did not expect next-handler to be called")
 	return core.UserInputResponse{}, nil
 }
+func (d *dontCallMe) RegisterUIServer(api *core.UIServerAPI) {
+}
 
 func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) {
 }

swarm/api/config.go

@@ -71,6 +71,7 @@ type Config struct {
 	SwapAPI              string
 	Cors                 string
 	BzzAccount           string
+	GlobalStoreAPI       string
 	privateKey           *ecdsa.PrivateKey
 }
 

swarm/api/inspector.go

@@ -0,0 +1,58 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+type Inspector struct {
+	api      *API
+	hive     *network.Hive
+	netStore *storage.NetStore
+}
+
+func NewInspector(api *API, hive *network.Hive, netStore *storage.NetStore) *Inspector {
+	return &Inspector{api, hive, netStore}
+}
+
+// Hive prints the kademlia table
+func (inspector *Inspector) Hive() string {
+	return inspector.hive.String()
+}
+
+type HasInfo struct {
+	Addr string `json:"address"`
+	Has  bool   `json:"has"`
+}
+
+// Has checks whether each chunk address is present in the underlying datastore,
+// the bool in the returned structs indicates if the underlying datastore has
+// the chunk stored with the given address (true), or not (false)
+func (inspector *Inspector) Has(chunkAddresses []storage.Address) []HasInfo {
+	results := make([]HasInfo, 0)
+	for _, addr := range chunkAddresses {
+		res := HasInfo{}
+		res.Addr = addr.String()
+		res.Has = inspector.netStore.Has(context.Background(), addr)
+		results = append(results, res)
+	}
+	return results
+}

swarm/docker/Dockerfile

@@ -10,14 +10,23 @@ RUN mkdir -p $GOPATH/src/github.com/ethereum && \
     git checkout ${VERSION} && \
     go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm && \
     go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/swarm-smoke && \
-    go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth && \
+    go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/global-store && \
-    cp $GOPATH/bin/swarm /swarm && cp $GOPATH/bin/geth /geth && cp $GOPATH/bin/swarm-smoke /swarm-smoke
+    go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth
 
 
-# Release image with the required binaries and scripts
+FROM alpine:3.8 as swarm-smoke
-FROM alpine:3.8
 WORKDIR /
-COPY --from=builder /swarm /geth /swarm-smoke /
+COPY --from=builder /go/bin/swarm-smoke /
-ADD run.sh /run.sh
 ADD run-smoke.sh /run-smoke.sh
+ENTRYPOINT ["/run-smoke.sh"]
+
+FROM alpine:3.8 as swarm-global-store
+WORKDIR /
+COPY --from=builder /go/bin/global-store /
+ENTRYPOINT ["/global-store"]
+
+FROM alpine:3.8 as swarm
+WORKDIR /
+COPY --from=builder /go/bin/swarm /go/bin/geth /
+ADD run.sh /run.sh
 ENTRYPOINT ["/run.sh"]

swarm/network/fetcher.go

@@ -52,6 +52,7 @@ type Fetcher struct {
 	requestC         chan uint8      // channel for incoming requests (with the hopCount value in it)
 	searchTimeout    time.Duration
 	skipCheck        bool
+	ctx              context.Context
 }
 
 type Request struct {
@@ -109,14 +110,14 @@ func NewFetcherFactory(request RequestFunc, skipCheck bool) *FetcherFactory {
 // contain the peers which are actively requesting this chunk, to make sure we
 // don't request back the chunks from them.
 // The created Fetcher is started and returned.
-func (f *FetcherFactory) New(ctx context.Context, source storage.Address, peersToSkip *sync.Map) storage.NetFetcher {
+func (f *FetcherFactory) New(ctx context.Context, source storage.Address, peers *sync.Map) storage.NetFetcher {
-	fetcher := NewFetcher(source, f.request, f.skipCheck)
+	fetcher := NewFetcher(ctx, source, f.request, f.skipCheck)
-	go fetcher.run(ctx, peersToSkip)
+	go fetcher.run(peers)
 	return fetcher
 }
 
 // NewFetcher creates a new Fetcher for the given chunk address using the given request function.
-func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher {
+func NewFetcher(ctx context.Context, addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher {
 	return &Fetcher{
 		addr:             addr,
 		protoRequestFunc: rf,
@@ -124,14 +125,15 @@ func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher {
 		requestC:         make(chan uint8),
 		searchTimeout:    defaultSearchTimeout,
 		skipCheck:        skipCheck,
+		ctx:              ctx,
 	}
 }
 
 // Offer is called when an upstream peer offers the chunk via syncing as part of `OfferedHashesMsg` and the node does not have the chunk locally.
-func (f *Fetcher) Offer(ctx context.Context, source *enode.ID) {
+func (f *Fetcher) Offer(source *enode.ID) {
 	// First we need to have this select to make sure that we return if context is done
 	select {
-	case <-ctx.Done():
+	case <-f.ctx.Done():
 		return
 	default:
 	}
@@ -140,15 +142,15 @@ func (f *Fetcher) Offer(ctx context.Context, source *enode.ID) {
 	// push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements)
 	select {
 	case f.offerC <- source:
-	case <-ctx.Done():
+	case <-f.ctx.Done():
 	}
 }
 
 // Request is called when an upstream peer request the chunk as part of `RetrieveRequestMsg`, or from a local request through FileStore, and the node does not have the chunk locally.
-func (f *Fetcher) Request(ctx context.Context, hopCount uint8) {
+func (f *Fetcher) Request(hopCount uint8) {
 	// First we need to have this select to make sure that we return if context is done
 	select {
-	case <-ctx.Done():
+	case <-f.ctx.Done():
 		return
 	default:
 	}
@@ -162,13 +164,13 @@ func (f *Fetcher) Request(ctx context.Context, hopCount uint8) {
 	// push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements)
 	select {
 	case f.requestC <- hopCount + 1:
-	case <-ctx.Done():
+	case <-f.ctx.Done():
 	}
 }
 
 // start prepares the Fetcher
 // it keeps the Fetcher alive within the lifecycle of the passed context
-func (f *Fetcher) run(ctx context.Context, peers *sync.Map) {
+func (f *Fetcher) run(peers *sync.Map) {
 	var (
 		doRequest bool             // determines if retrieval is initiated in the current iteration
 		wait      *time.Timer      // timer for search timeout
@@ -219,7 +221,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) {
 			doRequest = requested
 
 			// all Fetcher context closed, can quit
-		case <-ctx.Done():
+		case <-f.ctx.Done():
 			log.Trace("terminate fetcher", "request addr", f.addr)
 			// TODO: send cancellations to all peers left over in peers map (i.e., those we requested from)
 			return
@@ -228,7 +230,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) {
 		// need to issue a new request
 		if doRequest {
 			var err error
-			sources, err = f.doRequest(ctx, gone, peers, sources, hopCount)
+			sources, err = f.doRequest(gone, peers, sources, hopCount)
 			if err != nil {
 				log.Info("unable to request", "request addr", f.addr, "err", err)
 			}
@@ -266,7 +268,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) {
 // * the peer's address is added to the set of peers to skip
 // * the peer's address is removed from prospective sources, and
 // * a go routine is started that reports on the gone channel if the peer is disconnected (or terminated their streamer)
-func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSkip *sync.Map, sources []*enode.ID, hopCount uint8) ([]*enode.ID, error) {
+func (f *Fetcher) doRequest(gone chan *enode.ID, peersToSkip *sync.Map, sources []*enode.ID, hopCount uint8) ([]*enode.ID, error) {
 	var i int
 	var sourceID *enode.ID
 	var quit chan struct{}
@@ -283,7 +285,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki
 	for i = 0; i < len(sources); i++ {
 		req.Source = sources[i]
 		var err error
-		sourceID, quit, err = f.protoRequestFunc(ctx, req)
+		sourceID, quit, err = f.protoRequestFunc(f.ctx, req)
 		if err == nil {
 			// remove the peer from known sources
 			// Note: we can modify the source although we are looping on it, because we break from the loop immediately
@@ -297,7 +299,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki
 	if !foundSource {
 		req.Source = nil
 		var err error
-		sourceID, quit, err = f.protoRequestFunc(ctx, req)
+		sourceID, quit, err = f.protoRequestFunc(f.ctx, req)
 		if err != nil {
 			// if no peers found to request from
 			return sources, err
@@ -314,7 +316,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki
 		select {
 		case <-quit:
 			gone <- sourceID
-		case <-ctx.Done():
+		case <-f.ctx.Done():
 		}
 	}()
 	return sources, nil

swarm/network/fetcher_test.go

@@ -69,7 +69,11 @@ func (m *mockRequester) doRequest(ctx context.Context, request *Request) (*enode
 func TestFetcherSingleRequest(t *testing.T) {
 	requester := newMockRequester()
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
 
 	peers := []string{"a", "b", "c", "d"}
 	peersToSkip := &sync.Map{}
@@ -77,13 +81,9 @@ func TestFetcherSingleRequest(t *testing.T) {
 		peersToSkip.Store(p, time.Now())
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
+	go fetcher.run(peersToSkip)
-	defer cancel()
 
-	go fetcher.run(ctx, peersToSkip)
+	fetcher.Request(0)
-
-	rctx := context.Background()
-	fetcher.Request(rctx, 0)
 
 	select {
 	case request := <-requester.requestC:
@@ -115,20 +115,19 @@ func TestFetcherSingleRequest(t *testing.T) {
 func TestFetcherCancelStopsFetcher(t *testing.T) {
 	requester := newMockRequester()
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
-
-	peersToSkip := &sync.Map{}
 
 	ctx, cancel := context.WithCancel(context.Background())
 
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
+
+	peersToSkip := &sync.Map{}
+
 	// we start the fetcher, and then we immediately cancel the context
-	go fetcher.run(ctx, peersToSkip)
+	go fetcher.run(peersToSkip)
 	cancel()
 
-	rctx, rcancel := context.WithTimeout(ctx, 100*time.Millisecond)
-	defer rcancel()
 	// we call Request with an active context
-	fetcher.Request(rctx, 0)
+	fetcher.Request(0)
 
 	// fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening
 	select {
@@ -140,23 +139,23 @@ func TestFetcherCancelStopsFetcher(t *testing.T) {
 
 // TestFetchCancelStopsRequest tests that calling a Request function with a cancelled context does not initiate a request
 func TestFetcherCancelStopsRequest(t *testing.T) {
+	t.Skip("since context is now per fetcher, this test is likely redundant")
+
 	requester := newMockRequester(100 * time.Millisecond)
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
-
-	peersToSkip := &sync.Map{}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// we start the fetcher with an active context
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
-	go fetcher.run(ctx, peersToSkip)
 
-	rctx, rcancel := context.WithCancel(context.Background())
+	peersToSkip := &sync.Map{}
-	rcancel()
+
+	// we start the fetcher with an active context
+	go fetcher.run(peersToSkip)
 
 	// we call Request with a cancelled context
-	fetcher.Request(rctx, 0)
+	fetcher.Request(0)
 
 	// fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening
 	select {
@@ -166,8 +165,7 @@ func TestFetcherCancelStopsRequest(t *testing.T) {
 	}
 
 	// if there is another Request with active context, there should be a request, because the fetcher itself is not cancelled
-	rctx = context.Background()
+	fetcher.Request(0)
-	fetcher.Request(rctx, 0)
 
 	select {
 	case <-requester.requestC:
@@ -182,19 +180,19 @@ func TestFetcherCancelStopsRequest(t *testing.T) {
 func TestFetcherOfferUsesSource(t *testing.T) {
 	requester := newMockRequester(100 * time.Millisecond)
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
-
-	peersToSkip := &sync.Map{}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// start the fetcher
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
-	go fetcher.run(ctx, peersToSkip)
+
+	peersToSkip := &sync.Map{}
+
+	// start the fetcher
+	go fetcher.run(peersToSkip)
 
-	rctx := context.Background()
 	// call the Offer function with the source peer
-	fetcher.Offer(rctx, &sourcePeerID)
+	fetcher.Offer(&sourcePeerID)
 
 	// fetcher should not initiate request
 	select {
@@ -204,8 +202,7 @@ func TestFetcherOfferUsesSource(t *testing.T) {
 	}
 
 	// call Request after the Offer
-	rctx = context.Background()
+	fetcher.Request(0)
-	fetcher.Request(rctx, 0)
 
 	// there should be exactly 1 request coming from fetcher
 	var request *Request
@@ -234,19 +231,19 @@ func TestFetcherOfferUsesSource(t *testing.T) {
 func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) {
 	requester := newMockRequester(100 * time.Millisecond)
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
-
-	peersToSkip := &sync.Map{}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
+
+	peersToSkip := &sync.Map{}
+
 	// start the fetcher
-	go fetcher.run(ctx, peersToSkip)
+	go fetcher.run(peersToSkip)
 
 	// call Request first
-	rctx := context.Background()
+	fetcher.Request(0)
-	fetcher.Request(rctx, 0)
 
 	// there should be a request coming from fetcher
 	var request *Request
@@ -260,7 +257,7 @@ func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) {
 	}
 
 	// after the Request call Offer
-	fetcher.Offer(context.Background(), &sourcePeerID)
+	fetcher.Offer(&sourcePeerID)
 
 	// there should be a request coming from fetcher
 	select {
@@ -283,21 +280,21 @@ func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) {
 func TestFetcherRetryOnTimeout(t *testing.T) {
 	requester := newMockRequester()
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
 	// set searchTimeOut to low value so the test is quicker
 	fetcher.searchTimeout = 250 * time.Millisecond
 
 	peersToSkip := &sync.Map{}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	// start the fetcher
-	go fetcher.run(ctx, peersToSkip)
+	go fetcher.run(peersToSkip)
 
 	// call the fetch function with an active context
-	rctx := context.Background()
+	fetcher.Request(0)
-	fetcher.Request(rctx, 0)
 
 	// after 100ms the first request should be initiated
 	time.Sleep(100 * time.Millisecond)
@@ -339,7 +336,7 @@ func TestFetcherFactory(t *testing.T) {
 
 	fetcher := fetcherFactory.New(context.Background(), addr, peersToSkip)
 
-	fetcher.Request(context.Background(), 0)
+	fetcher.Request(0)
 
 	// check if the created fetchFunction really starts a fetcher and initiates a request
 	select {
@@ -353,7 +350,11 @@ func TestFetcherFactory(t *testing.T) {
 func TestFetcherRequestQuitRetriesRequest(t *testing.T) {
 	requester := newMockRequester()
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
 
 	// make sure the searchTimeout is long so it is sure the request is not
 	// retried because of timeout
@@ -361,13 +362,9 @@ func TestFetcherRequestQuitRetriesRequest(t *testing.T) {
 
 	peersToSkip := &sync.Map{}
 
-	ctx, cancel := context.WithCancel(context.Background())
+	go fetcher.run(peersToSkip)
-	defer cancel()
 
-	go fetcher.run(ctx, peersToSkip)
+	fetcher.Request(0)
-
-	rctx := context.Background()
-	fetcher.Request(rctx, 0)
 
 	select {
 	case <-requester.requestC:
@@ -460,17 +457,15 @@ func TestRequestSkipPeerPermanent(t *testing.T) {
 func TestFetcherMaxHopCount(t *testing.T) {
 	requester := newMockRequester()
 	addr := make([]byte, 32)
-	fetcher := NewFetcher(addr, requester.doRequest, true)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	fetcher := NewFetcher(ctx, addr, requester.doRequest, true)
+
 	peersToSkip := &sync.Map{}
 
-	go fetcher.run(ctx, peersToSkip)
+	go fetcher.run(peersToSkip)
-
-	rctx := context.Background()
-	fetcher.Request(rctx, maxHopCount)
 
 	// if hopCount is already at max no request should be initiated
 	select {

swarm/network/kademlia.go

@@ -353,6 +353,9 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() {
 // Not receiving from the returned channel will block Register function
 // when address count value changes.
 func (k *Kademlia) AddrCountC() <-chan int {
+	k.lock.Lock()
+	defer k.lock.Unlock()
+
 	if k.addrCountC == nil {
 		k.addrCountC = make(chan int)
 	}

swarm/network/networkid_test.go

@@ -44,7 +44,7 @@ var (
 
 const (
 	NumberOfNets = 4
-	MaxTimeout   = 6
+	MaxTimeout   = 15 * time.Second
 )
 
 func init() {
@@ -146,7 +146,7 @@ func setupNetwork(numnodes int) (net *simulations.Network, err error) {
 			return nil, fmt.Errorf("create node %d rpc client fail: %v", i, err)
 		}
 		//now setup and start event watching in order to know when we can upload
-		ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout*time.Second)
+		ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout)
 		defer watchCancel()
 		watchSubscriptionEvents(ctx, nodes[i].ID(), client, errc, quitC)
 		//on every iteration we connect to all previous ones

swarm/network/simulations/overlay_test.go

@@ -32,7 +32,7 @@ import (
 )
 
 var (
-	nodeCount = 16
+	nodeCount = 10
 )
 
 //This test is used to test the overlay simulation.

swarm/network/stream/common_test.go

@@ -151,7 +151,7 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
 	// temp datadir
 	datadir, err := ioutil.TempDir("", "streamer")
 	if err != nil {
-		return nil, nil, nil, func() {}, err
+		return nil, nil, nil, nil, err
 	}
 	removeDataDir := func() {
 		os.RemoveAll(datadir)
@@ -163,12 +163,14 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
 
 	localStore, err := storage.NewTestLocalStoreForAddr(params)
 	if err != nil {
-		return nil, nil, nil, removeDataDir, err
+		removeDataDir()
+		return nil, nil, nil, nil, err
 	}
 
 	netStore, err := storage.NewNetStore(localStore, nil)
 	if err != nil {
-		return nil, nil, nil, removeDataDir, err
+		removeDataDir()
+		return nil, nil, nil, nil, err
 	}
 
 	delivery := NewDelivery(to, netStore)
@@ -180,8 +182,9 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
 	}
 	protocolTester := p2ptest.NewProtocolTester(addr.ID(), 1, streamer.runProtocol)
 
-	err = waitForPeers(streamer, 1*time.Second, 1)
+	err = waitForPeers(streamer, 10*time.Second, 1)
 	if err != nil {
+		teardown()
 		return nil, nil, nil, nil, errors.New("timeout: peer is not created")
 	}
 
@@ -214,6 +217,11 @@ func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore {
 	}
 }
 
+// not used in this context, only to fulfill ChunkStore interface
+func (rrs *roundRobinStore) Has(ctx context.Context, addr storage.Address) bool {
+	panic("RoundRobinStor doesn't support HasChunk")
+}
+
 func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (storage.Chunk, error) {
 	return nil, errors.New("get not well defined on round robin store")
 }
@@ -312,3 +320,54 @@ func createTestLocalStorageForID(id enode.ID, addr *network.BzzAddr) (storage.Ch
 	}
 	return store, datadir, nil
 }
+
+// watchDisconnections receives simulation peer events in a new goroutine and sets atomic value
+// disconnected to true in case of a disconnect event.
+func watchDisconnections(ctx context.Context, sim *simulation.Simulation) (disconnected *boolean) {
+	log.Debug("Watching for disconnections")
+	disconnections := sim.PeerEvents(
+		ctx,
+		sim.NodeIDs(),
+		simulation.NewPeerEventsFilter().Drop(),
+	)
+	disconnected = new(boolean)
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case d := <-disconnections:
+				if d.Error != nil {
+					log.Error("peer drop event error", "node", d.NodeID, "peer", d.PeerID, "err", d.Error)
+				} else {
+					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
+				}
+				disconnected.set(true)
+			}
+		}
+	}()
+	return disconnected
+}
+
+// boolean is used to concurrently set
+// and read a boolean value.
+type boolean struct {
+	v  bool
+	mu sync.RWMutex
+}
+
+// set sets the value.
+func (b *boolean) set(v bool) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	b.v = v
+}
+
+// bool reads the value.
+func (b *boolean) bool() bool {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	return b.v
+}

swarm/network/stream/delivery.go

@@ -144,7 +144,6 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req *
 	ctx, osp = spancontext.StartSpan(
 		ctx,
 		"retrieve.request")
-	defer osp.Finish()
 
 	s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", true))
 	if err != nil {
@@ -167,6 +166,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req *
 	}()
 
 	go func() {
+		defer osp.Finish()
 		chunk, err := d.chunkStore.Get(ctx, req.Addr)
 		if err != nil {
 			retrieveChunkFail.Inc(1)
@@ -213,11 +213,12 @@ func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *Ch
 	ctx, osp = spancontext.StartSpan(
 		ctx,
 		"chunk.delivery")
-	defer osp.Finish()
 
 	processReceivedChunksCount.Inc(1)
 
 	go func() {
+		defer osp.Finish()
+
 		req.peer = sp
 		err := d.chunkStore.Put(ctx, storage.NewChunk(req.Addr, req.SData))
 		if err != nil {
@@ -271,7 +272,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) (
 		Addr:      req.Addr,
 		SkipCheck: req.SkipCheck,
 		HopCount:  req.HopCount,
-	}, Top)
+	}, Top, "request.from.peers")
 	if err != nil {
 		return nil, nil, err
 	}

swarm/network/stream/delivery_test.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -48,10 +47,10 @@ func TestStreamerRetrieveRequest(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, streamer, _, teardown, err := newStreamerTester(regOpts)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -100,10 +99,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
 		Retrieval: RetrievalEnabled,
 		Syncing:   SyncingDisabled, //do no syncing
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -172,10 +171,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
 		Retrieval: RetrievalEnabled,
 		Syncing:   SyncingDisabled,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -362,10 +361,10 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
 		Retrieval: RetrievalDisabled,
 		Syncing:   SyncingDisabled,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) {
 		return &testClient{
@@ -485,7 +484,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 	}
 
 	log.Info("Starting simulation")
-	ctx := context.Background()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
 		nodeIDs := sim.UpNodeIDs()
 		//determine the pivot node to be the first node of the simulation
@@ -548,27 +548,10 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 			retErrC <- err
 		}()
 
-		log.Debug("Watching for disconnections")
+		disconnected := watchDisconnections(ctx, sim)
-		disconnections := sim.PeerEvents(
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
-		var disconnected atomic.Value
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
+				err = errors.New("disconnect events received")
-					err = errors.New("disconnect events received")
-				}
 			}
 		}()
 
@@ -589,7 +572,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool)
 			return fmt.Errorf("Test failed, chunks not available on all nodes")
 		}
 		if err := <-retErrC; err != nil {
-			t.Fatalf("requesting chunks: %v", err)
+			return fmt.Errorf("requesting chunks: %v", err)
 		}
 		log.Debug("Test terminated successfully")
 		return nil
@@ -657,21 +640,22 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 		b.Fatal(err)
 	}
 
-	ctx := context.Background()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
 		nodeIDs := sim.UpNodeIDs()
 		node := nodeIDs[len(nodeIDs)-1]
 
 		item, ok := sim.NodeItem(node, bucketKeyFileStore)
 		if !ok {
-			b.Fatal("No filestore")
+			return errors.New("No filestore")
 		}
 		remoteFileStore := item.(*storage.FileStore)
 
 		pivotNode := nodeIDs[0]
 		item, ok = sim.NodeItem(pivotNode, bucketKeyNetStore)
 		if !ok {
-			b.Fatal("No filestore")
+			return errors.New("No filestore")
 		}
 		netStore := item.(*storage.NetStore)
 
@@ -679,26 +663,10 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 			return err
 		}
 
-		disconnections := sim.PeerEvents(
+		disconnected := watchDisconnections(ctx, sim)
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
-		var disconnected atomic.Value
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
+				err = errors.New("disconnect events received")
-					err = errors.New("disconnect events received")
-				}
 			}
 		}()
 		// benchmark loop
@@ -713,12 +681,12 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 				ctx := context.TODO()
 				hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false)
 				if err != nil {
-					b.Fatalf("expected no error. got %v", err)
+					return fmt.Errorf("store: %v", err)
 				}
 				// wait until all chunks stored
 				err = wait(ctx)
 				if err != nil {
-					b.Fatalf("expected no error. got %v", err)
+					return fmt.Errorf("wait store: %v", err)
 				}
 				// collect the hashes
 				hashes[i] = hash
@@ -754,10 +722,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
 				break Loop
 			}
 		}
-		if err != nil {
+		return err
-			b.Fatal(err)
-		}
-		return nil
 	})
 	if result.Error != nil {
 		b.Fatal(result.Error)

swarm/network/stream/intervals_test.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -118,13 +117,11 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 
 		_, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
 		if err != nil {
-			log.Error("Store error: %v", "err", err)
+			return fmt.Errorf("store: %v", err)
-			t.Fatal(err)
 		}
 		err = wait(ctx)
 		if err != nil {
-			log.Error("Wait error: %v", "err", err)
+			return fmt.Errorf("wait store: %v", err)
-			t.Fatal(err)
 		}
 
 		item, ok = sim.NodeItem(checker, bucketKeyRegistry)
@@ -136,32 +133,15 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 		liveErrC := make(chan error)
 		historyErrC := make(chan error)
 
-		log.Debug("Watching for disconnections")
-		disconnections := sim.PeerEvents(
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
 		err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
 		if err != nil {
 			return err
 		}
 
-		var disconnected atomic.Value
+		disconnected := watchDisconnections(ctx, sim)
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
+				err = errors.New("disconnect events received")
-					err = errors.New("disconnect events received")
-				}
 			}
 		}()
 

swarm/network/stream/lightnode_test.go

@@ -29,10 +29,10 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -68,10 +68,10 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -112,10 +112,10 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) {
 		Syncing:   SyncingRegisterOnly,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 
@@ -157,10 +157,10 @@ func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) {
 		Syncing:   SyncingDisabled,
 	}
 	tester, _, _, teardown, err := newStreamerTester(registryOptions)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	node := tester.Nodes[0]
 

swarm/network/stream/messages.go

@@ -300,7 +300,7 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg
 			return
 		}
 		log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "from", msg.From, "to", msg.To)
-		err := p.SendPriority(ctx, msg, c.priority)
+		err := p.SendPriority(ctx, msg, c.priority, "")
 		if err != nil {
 			log.Warn("SendPriority error", "err", err)
 		}

swarm/network/stream/norace_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,21 +14,11 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package api
+// +build !race
 
-import (
+package stream
-	"github.com/ethereum/go-ethereum/swarm/network"
-)
 
-type Control struct {
+// Provide a flag to reduce the scope of tests when running them
-	api  *API
+// with race detector. Some of the tests are doing a lot of allocations
-	hive *network.Hive
+// on the heap, and race detector uses much more memory to track them.
-}
+const raceTest = false
-
-func NewControl(api *API, hive *network.Hive) *Control {
-	return &Control{api, hive}
-}
-
-func (c *Control) Hive() string {
-	return c.hive.String()
-}

swarm/network/stream/peer.go

@@ -65,6 +65,7 @@ type Peer struct {
 	// on creating a new client in offered hashes handler.
 	clientParams map[Stream]*clientParams
 	quit         chan struct{}
+	spans        sync.Map
 }
 
 type WrappedPriorityMsg struct {
@@ -82,10 +83,16 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer {
 		clients:      make(map[Stream]*client),
 		clientParams: make(map[Stream]*clientParams),
 		quit:         make(chan struct{}),
+		spans:        sync.Map{},
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	go p.pq.Run(ctx, func(i interface{}) {
 		wmsg := i.(WrappedPriorityMsg)
+		defer p.spans.Delete(wmsg.Context)
+		sp, ok := p.spans.Load(wmsg.Context)
+		if ok {
+			defer sp.(opentracing.Span).Finish()
+		}
 		err := p.Send(wmsg.Context, wmsg.Msg)
 		if err != nil {
 			log.Error("Message send error, dropping peer", "peer", p.ID(), "err", err)
@@ -130,7 +137,6 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer {
 // Deliver sends a storeRequestMsg protocol message to the peer
 // Depending on the `syncing` parameter we send different message types
 func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8, syncing bool) error {
-	var sp opentracing.Span
 	var msg interface{}
 
 	spanName := "send.chunk.delivery"
@@ -151,18 +157,22 @@ func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8,
 		}
 		spanName += ".retrieval"
 	}
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		spanName)
-	defer sp.Finish()
 
-	return p.SendPriority(ctx, msg, priority)
+	return p.SendPriority(ctx, msg, priority, spanName)
 }
 
 // SendPriority sends message to the peer using the outgoing priority queue
-func (p *Peer) SendPriority(ctx context.Context, msg interface{}, priority uint8) error {
+func (p *Peer) SendPriority(ctx context.Context, msg interface{}, priority uint8, traceId string) error {
 	defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("peer.sendpriority_t.%d", priority), nil).UpdateSince(time.Now())
 	metrics.GetOrRegisterCounter(fmt.Sprintf("peer.sendpriority.%d", priority), nil).Inc(1)
+	if traceId != "" {
+		var sp opentracing.Span
+		ctx, sp = spancontext.StartSpan(
+			ctx,
+			traceId,
+		)
+		p.spans.Store(ctx, sp)
+	}
 	wmsg := WrappedPriorityMsg{
 		Context: ctx,
 		Msg:     msg,
@@ -205,7 +215,7 @@ func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error {
 		Stream:        s.stream,
 	}
 	log.Trace("Swarm syncer offer batch", "peer", p.ID(), "stream", s.stream, "len", len(hashes), "from", from, "to", to)
-	return p.SendPriority(ctx, msg, s.priority)
+	return p.SendPriority(ctx, msg, s.priority, "send.offered.hashes")
 }
 
 func (p *Peer) getServer(s Stream) (*server, error) {

swarm/network/stream/race_test.go

@@ -0,0 +1,23 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build race
+
+package stream
+
+// Reduce the scope of some tests when running with race detector,
+// as it raises the memory consumption significantly.
+const raceTest = true

swarm/network/stream/snapshot_retrieval_test.go

@@ -151,7 +151,7 @@ func runFileRetrievalTest(nodeCount int) error {
 		return err
 	}
 
-	ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute)
+	ctx, cancelSimRun := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancelSimRun()
 
 	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {

swarm/network/stream/snapshot_sync_test.go

@@ -17,11 +17,12 @@ package stream
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"runtime"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -92,6 +93,15 @@ func TestSyncingViaGlobalSync(t *testing.T) {
 		if *longrunning {
 			chnkCnt = []int{1, 8, 32, 256, 1024}
 			nodeCnt = []int{16, 32, 64, 128, 256}
+		} else if raceTest {
+			// TestSyncingViaGlobalSync allocates a lot of memory
+			// with race detector. By reducing the number of chunks
+			// and nodes, memory consumption is lower and data races
+			// are still checked, while correctness of syncing is
+			// tested with more chunks and nodes in regular (!race)
+			// tests.
+			chnkCnt = []int{4}
+			nodeCnt = []int{16}
 		} else {
 			//default test
 			chnkCnt = []int{4, 32}
@@ -113,7 +123,23 @@ var simServiceMap = map[string]simulation.ServiceFunc{
 			return nil, nil, err
 		}
 
-		r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+		var dir string
+		var store *state.DBStore
+		if raceTest {
+			// Use on-disk DBStore to reduce memory consumption in race tests.
+			dir, err = ioutil.TempDir("", "swarm-stream-")
+			if err != nil {
+				return nil, nil, err
+			}
+			store, err = state.NewDBStore(dir)
+			if err != nil {
+				return nil, nil, err
+			}
+		} else {
+			store = state.NewInmemoryStore()
+		}
+
+		r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{
 			Retrieval:       RetrievalDisabled,
 			Syncing:         SyncingAutoSubscribe,
 			SyncUpdateDelay: 3 * time.Second,
@@ -156,36 +182,24 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
 		t.Fatal(err)
 	}
 
-	disconnections := sim.PeerEvents(
-		context.Background(),
-		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Drop(),
-	)
-
-	var disconnected atomic.Value
-	go func() {
-		for d := range disconnections {
-			if d.Error != nil {
-				log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-				disconnected.Store(true)
-			}
-		}
-	}()
-
 	result := runSim(conf, ctx, sim, chunkCount)
 
 	if result.Error != nil {
 		t.Fatal(result.Error)
 	}
-	if yes, ok := disconnected.Load().(bool); ok && yes {
-		t.Fatal("disconnect events received")
-	}
 	log.Info("Simulation ended")
 }
 
 func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulation, chunkCount int) simulation.Result {
 
-	return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+	return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
+		disconnected := watchDisconnections(ctx, sim)
+		defer func() {
+			if err != nil && disconnected.bool() {
+				err = errors.New("disconnect events received")
+			}
+		}()
+
 		nodeIDs := sim.UpNodeIDs()
 		for _, n := range nodeIDs {
 			//get the kademlia overlay address from this ID

swarm/network/stream/stream.go

@@ -359,7 +359,7 @@ func (r *Registry) Subscribe(peerId enode.ID, s Stream, h *Range, priority uint8
 	}
 	log.Debug("Subscribe ", "peer", peerId, "stream", s, "history", h)
 
-	return peer.SendPriority(context.TODO(), msg, priority)
+	return peer.SendPriority(context.TODO(), msg, priority, "")
 }
 
 func (r *Registry) Unsubscribe(peerId enode.ID, s Stream) error {
@@ -729,7 +729,8 @@ func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error
 		if err != nil {
 			return err
 		}
-		if err := p.SendPriority(context.TODO(), tp, c.priority); err != nil {
+
+		if err := p.SendPriority(context.TODO(), tp, c.priority, ""); err != nil {
 			return err
 		}
 		if c.to > 0 && tp.Takeover.End >= c.to {
@@ -938,16 +939,22 @@ It returns a map of node IDs with an array of string representations of Stream o
 func (api *API) GetPeerSubscriptions() map[string][]string {
 	//create the empty map
 	pstreams := make(map[string][]string)
+
 	//iterate all streamer peers
+	api.streamer.peersMu.RLock()
+	defer api.streamer.peersMu.RUnlock()
+
 	for id, p := range api.streamer.peers {
 		var streams []string
 		//every peer has a map of stream servers
 		//every stream server represents a subscription
+		p.serverMu.RLock()
 		for s := range p.servers {
 			//append the string representation of the stream
 			//to the list for this peer
 			streams = append(streams, s.String())
 		}
+		p.serverMu.RUnlock()
 		//set the array of stream servers to the map
 		pstreams[id.String()] = streams
 	}

swarm/network/stream/streamer_test.go

@@ -41,10 +41,10 @@ import (
 
 func TestStreamerSubscribe(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 	err = streamer.Subscribe(tester.Nodes[0].ID(), stream, NewRange(0, 0), Top)
@@ -55,10 +55,10 @@ func TestStreamerSubscribe(t *testing.T) {
 
 func TestStreamerRequestSubscription(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", false)
 	err = streamer.RequestSubscription(tester.Nodes[0].ID(), stream, &Range{}, Top)
@@ -146,10 +146,10 @@ func (self *testServer) Close() {
 
 func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) {
 		return newTestClient(t), nil
@@ -239,10 +239,10 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", false)
 
@@ -306,10 +306,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -372,10 +372,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 0), nil
@@ -416,10 +416,10 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
 
 func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -479,10 +479,10 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) {
 
 func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -544,10 +544,10 @@ func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) {
 
 func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	stream := NewStream("foo", "", true)
 
@@ -643,10 +643,10 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
 
 func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(nil)
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 10), nil
@@ -780,10 +780,10 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) {
 		Syncing:        SyncingDisabled,
 		MaxPeerServers: maxPeerServers,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 0), nil
@@ -854,10 +854,10 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{
 		MaxPeerServers: maxPeerServers,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
 		return newTestServer(t, 0), nil
@@ -940,10 +940,10 @@ func TestHasPriceImplementation(t *testing.T) {
 		Retrieval: RetrievalDisabled,
 		Syncing:   SyncingDisabled,
 	})
-	defer teardown()
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer teardown()
 
 	if r.prices == nil {
 		t.Fatal("No prices implementation available for the stream protocol")
@@ -1177,6 +1177,7 @@ starts the simulation, waits for SyncUpdateDelay in order to kick off
 stream registration, then tests that there are subscriptions.
 */
 func TestGetSubscriptionsRPC(t *testing.T) {
+
 	// arbitrarily set to 4
 	nodeCount := 4
 	// run with more nodes if `longrunning` flag is set
@@ -1188,19 +1189,16 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 	// holds the msg code for SubscribeMsg
 	var subscribeMsgCode uint64
 	var ok bool
-	var expectedMsgCount = 0
+	var expectedMsgCount counter
 
 	// this channel signalizes that the expected amount of subscriptiosn is done
 	allSubscriptionsDone := make(chan struct{})
-	lock := sync.RWMutex{}
 	// after the test, we need to reset the subscriptionFunc to the default
 	defer func() { subscriptionFunc = doRequestSubscription }()
 
 	// we use this subscriptionFunc for this test: just increases count and calls the actual subscription
 	subscriptionFunc = func(r *Registry, p *network.Peer, bin uint8, subs map[enode.ID]map[Stream]struct{}) bool {
-		lock.Lock()
+		expectedMsgCount.inc()
-		expectedMsgCount++
-		lock.Unlock()
 		doRequestSubscription(r, p, bin, subs)
 		return true
 	}
@@ -1290,24 +1288,24 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 		select {
 		case <-allSubscriptionsDone:
 		case <-ctx.Done():
-			t.Fatal("Context timed out")
+			return errors.New("Context timed out")
 		}
 
-		log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount)
+		log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount.count())
 		//now iterate again, this time we call each node via RPC to get its subscriptions
 		realCount := 0
 		for _, node := range nodes {
 			//create rpc client
 			client, err := node.Client()
 			if err != nil {
-				t.Fatalf("create node 1 rpc client fail: %v", err)
+				return fmt.Errorf("create node 1 rpc client fail: %v", err)
 			}
 
 			//ask it for subscriptions
 			pstreams := make(map[string][]string)
 			err = client.Call(&pstreams, "stream_getPeerSubscriptions")
 			if err != nil {
-				t.Fatal(err)
+				return fmt.Errorf("client call stream_getPeerSubscriptions: %v", err)
 			}
 			//length of the subscriptions can not be smaller than number of peers
 			log.Debug("node subscriptions", "node", node.String())
@@ -1324,8 +1322,9 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 			}
 		}
 		// every node is mutually subscribed to each other, so the actual count is half of it
-		if realCount/2 != expectedMsgCount {
+		emc := expectedMsgCount.count()
-			return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, expectedMsgCount)
+		if realCount/2 != emc {
+			return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, emc)
 		}
 		return nil
 	})
@@ -1333,3 +1332,26 @@ func TestGetSubscriptionsRPC(t *testing.T) {
 		t.Fatal(result.Error)
 	}
 }
+
+// counter is used to concurrently increment
+// and read an integer value.
+type counter struct {
+	v  int
+	mu sync.RWMutex
+}
+
+// Increment the counter.
+func (c *counter) inc() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.v++
+}
+
+// Read the counter value.
+func (c *counter) count() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.v
+}

swarm/network/stream/syncer_test.go

@@ -22,8 +22,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
+	"os"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -44,9 +44,15 @@ const dataChunkCount = 200
 
 func TestSyncerSimulation(t *testing.T) {
 	testSyncBetweenNodes(t, 2, dataChunkCount, true, 1)
-	testSyncBetweenNodes(t, 4, dataChunkCount, true, 1)
+	// This test uses much more memory when running with
-	testSyncBetweenNodes(t, 8, dataChunkCount, true, 1)
+	// race detector. Allow it to finish successfully by
-	testSyncBetweenNodes(t, 16, dataChunkCount, true, 1)
+	// reducing its scope, and still check for data races
+	// with the smallest number of nodes.
+	if !raceTest {
+		testSyncBetweenNodes(t, 4, dataChunkCount, true, 1)
+		testSyncBetweenNodes(t, 8, dataChunkCount, true, 1)
+		testSyncBetweenNodes(t, 16, dataChunkCount, true, 1)
+	}
 }
 
 func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
@@ -80,7 +86,23 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 				return nil, nil, err
 			}
 
-			r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+			var dir string
+			var store *state.DBStore
+			if raceTest {
+				// Use on-disk DBStore to reduce memory consumption in race tests.
+				dir, err = ioutil.TempDir("", "swarm-stream-")
+				if err != nil {
+					return nil, nil, err
+				}
+				store, err = state.NewDBStore(dir)
+				if err != nil {
+					return nil, nil, err
+				}
+			} else {
+				store = state.NewInmemoryStore()
+			}
+
+			r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{
 				Retrieval: RetrievalDisabled,
 				Syncing:   SyncingAutoSubscribe,
 				SkipCheck: skipCheck,
@@ -89,6 +111,9 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 			cleanup = func() {
 				r.Close()
 				clean()
+				if dir != "" {
+					os.RemoveAll(dir)
+				}
 			}
 
 			return r, cleanup, nil
@@ -114,26 +139,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 			nodeIndex[id] = i
 		}
 
-		disconnections := sim.PeerEvents(
+		disconnected := watchDisconnections(ctx, sim)
-			context.Background(),
-			sim.NodeIDs(),
-			simulation.NewPeerEventsFilter().Drop(),
-		)
-
-		var disconnected atomic.Value
-		go func() {
-			for d := range disconnections {
-				if d.Error != nil {
-					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-					disconnected.Store(true)
-				}
-			}
-		}()
 		defer func() {
-			if err != nil {
+			if err != nil && disconnected.bool() {
-				if yes, ok := disconnected.Load().(bool); ok && yes {
+				err = errors.New("disconnect events received")
-					err = errors.New("disconnect events received")
-				}
 			}
 		}()
 
@@ -142,7 +151,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 			id := nodeIDs[j]
 			client, err := sim.Net.GetNode(id).Client()
 			if err != nil {
-				t.Fatal(err)
+				return fmt.Errorf("node %s client: %v", id, err)
 			}
 			sid := nodeIDs[j+1]
 			client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream("SYNC", FormatSyncBinKey(1), false), NewRange(0, 0), Top)
@@ -158,7 +167,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
 				size := chunkCount * chunkSize
 				_, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false)
 				if err != nil {
-					t.Fatal(err.Error())
+					return fmt.Errorf("fileStore.Store: %v", err)
 				}
 				wait(ctx)
 			}
@@ -273,7 +282,7 @@ func TestSameVersionID(t *testing.T) {
 
 		//the peers should connect, thus getting the peer should not return nil
 		if registry.getPeer(nodes[1]) == nil {
-			t.Fatal("Expected the peer to not be nil, but it is")
+			return errors.New("Expected the peer to not be nil, but it is")
 		}
 		return nil
 	})
@@ -338,7 +347,7 @@ func TestDifferentVersionID(t *testing.T) {
 
 		//getting the other peer should fail due to the different version numbers
 		if registry.getPeer(nodes[1]) != nil {
-			t.Fatal("Expected the peer to be nil, but it is not")
+			return errors.New("Expected the peer to be nil, but it is not")
 		}
 		return nil
 	})

swarm/network/stream/visualized_snapshot_sync_sim_test.go

@@ -66,31 +66,6 @@ func setupSim(serviceMap map[string]simulation.ServiceFunc) (int, int, *simulati
 	return nodeCount, chunkCount, sim
 }
 
-//watch for disconnections and wait for healthy
-func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) {
-	ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute)
-
-	if _, err := sim.WaitTillHealthy(ctx); err != nil {
-		panic(err)
-	}
-
-	disconnections := sim.PeerEvents(
-		context.Background(),
-		sim.NodeIDs(),
-		simulation.NewPeerEventsFilter().Drop(),
-	)
-
-	go func() {
-		for d := range disconnections {
-			log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
-			panic("unexpected disconnect")
-			cancelSimRun()
-		}
-	}()
-
-	return ctx, cancelSimRun
-}
-
 //This test requests bogus hashes into the network
 func TestNonExistingHashesWithServer(t *testing.T) {
 
@@ -102,19 +77,25 @@ func TestNonExistingHashesWithServer(t *testing.T) {
 		panic(err)
 	}
 
-	ctx, cancelSimRun := watchSim(sim)
-	defer cancelSimRun()
-
 	//in order to get some meaningful visualization, it is beneficial
 	//to define a minimum duration of this test
 	testDuration := 20 * time.Second
 
-	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
+		disconnected := watchDisconnections(ctx, sim)
+		defer func() {
+			if err != nil {
+				if yes, ok := disconnected.Load().(bool); ok && yes {
+					err = errors.New("disconnect events received")
+				}
+			}
+		}()
+
 		//check on the node's FileStore (netstore)
 		id := sim.Net.GetRandomUpNode().ID()
 		item, ok := sim.NodeItem(id, bucketKeyFileStore)
 		if !ok {
-			t.Fatalf("No filestore")
+			return errors.New("No filestore")
 		}
 		fileStore := item.(*storage.FileStore)
 		//create a bogus hash
@@ -213,9 +194,6 @@ func TestSnapshotSyncWithServer(t *testing.T) {
 		panic(err)
 	}
 
-	ctx, cancelSimRun := watchSim(sim)
-	defer cancelSimRun()
-
 	//run the sim
 	result := runSim(conf, ctx, sim, chunkCount)
 

swarm/pss/client/client_test.go

@@ -38,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/state"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 )
 
 type protoCtrl struct {

swarm/pss/forwarding_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/protocols"
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pot"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 )
 
 type testCase struct {

swarm/pss/notify/notify_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/state"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 )
 
 var (

swarm/pss/protocol.go

@@ -228,6 +228,7 @@ func ToP2pMsg(msg []byte) (p2p.Msg, error) {
 // to link the peer to.
 // The key must exist in the pss store prior to adding the peer.
 func (p *Protocol) AddPeer(peer *p2p.Peer, topic Topic, asymmetric bool, key string) (p2p.MsgReadWriter, error) {
+	var ok bool
 	rw := &PssReadWriter{
 		Pss:   p.Pss,
 		rw:    make(chan p2p.Msg),
@@ -242,19 +243,21 @@ func (p *Protocol) AddPeer(peer *p2p.Peer, topic Topic, asymmetric bool, key str
 	}
 	if asymmetric {
 		p.Pss.pubKeyPoolMu.Lock()
-		if _, ok := p.Pss.pubKeyPool[key]; !ok {
+		_, ok = p.Pss.pubKeyPool[key]
+		p.Pss.pubKeyPoolMu.Unlock()
+		if !ok {
 			return nil, fmt.Errorf("asym key does not exist: %s", key)
 		}
-		p.Pss.pubKeyPoolMu.Unlock()
 		p.RWPoolMu.Lock()
 		p.pubKeyRWPool[key] = rw
 		p.RWPoolMu.Unlock()
 	} else {
 		p.Pss.symKeyPoolMu.Lock()
-		if _, ok := p.Pss.symKeyPool[key]; !ok {
+		_, ok = p.Pss.symKeyPool[key]
+		p.Pss.symKeyPoolMu.Unlock()
+		if !ok {
 			return nil, fmt.Errorf("symkey does not exist: %s", key)
 		}
-		p.Pss.symKeyPoolMu.Unlock()
 		p.RWPoolMu.Lock()
 		p.symKeyRWPool[key] = rw
 		p.RWPoolMu.Unlock()

swarm/pss/pss.go

@@ -38,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pot"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 	"golang.org/x/crypto/sha3"
 )
 
@@ -686,7 +686,7 @@ func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage,
 		if err != nil {
 			continue
 		}
-		if !recvmsg.Validate() {
+		if !recvmsg.ValidateAndParse() {
 			return nil, "", nil, fmt.Errorf("symmetrically encrypted message has invalid signature or is corrupt")
 		}
 		p.symKeyPoolMu.Lock()
@@ -713,7 +713,7 @@ func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage,
 		return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err)
 	}
 	// check signature (if signed), strip padding
-	if !recvmsg.Validate() {
+	if !recvmsg.ValidateAndParse() {
 		return nil, "", nil, fmt.Errorf("invalid message")
 	}
 	pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src))

swarm/pss/pss_test.go

@@ -50,7 +50,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pot"
 	"github.com/ethereum/go-ethereum/swarm/state"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 )
 
 var (

swarm/pss/types.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 )
 
 const (

swarm/storage/chunker.go

@@ -465,7 +465,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
 		length *= r.chunkSize
 	}
 	wg.Add(1)
-	go r.join(b, off, off+length, depth, treeSize/r.branches, r.chunkData, &wg, errC, quitC)
+	go r.join(cctx, b, off, off+length, depth, treeSize/r.branches, r.chunkData, &wg, errC, quitC)
 	go func() {
 		wg.Wait()
 		close(errC)
@@ -485,7 +485,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
 	return len(b), nil
 }
 
-func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunkData ChunkData, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
+func (r *LazyChunkReader) join(ctx context.Context, b []byte, off int64, eoff int64, depth int, treeSize int64, chunkData ChunkData, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
 	defer parentWg.Done()
 	// find appropriate block level
 	for chunkData.Size() < uint64(treeSize) && depth > r.depth {
@@ -533,7 +533,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS
 		go func(j int64) {
 			childAddress := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize]
 			startTime := time.Now()
-			chunkData, err := r.getter.Get(r.ctx, Reference(childAddress))
+			chunkData, err := r.getter.Get(ctx, Reference(childAddress))
 			if err != nil {
 				metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime)
 				log.Debug("lazychunkreader.join", "key", fmt.Sprintf("%x", childAddress), "err", err)
@@ -554,7 +554,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS
 			if soff < off {
 				soff = off
 			}
-			r.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/r.branches, chunkData, wg, errC, quitC)
+			r.join(ctx, b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/r.branches, chunkData, wg, errC, quitC)
 		}(i)
 	} //for
 }
@@ -581,6 +581,11 @@ var errWhence = errors.New("Seek: invalid whence")
 var errOffset = errors.New("Seek: invalid offset")
 
 func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
+	cctx, sp := spancontext.StartSpan(
+		r.ctx,
+		"lcr.seek")
+	defer sp.Finish()
+
 	log.Debug("lazychunkreader.seek", "key", r.addr, "offset", offset)
 	switch whence {
 	default:
@@ -590,8 +595,9 @@ func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
 	case 1:
 		offset += r.off
 	case 2:
+
 		if r.chunkData == nil { //seek from the end requires rootchunk for size. call Size first
-			_, err := r.Size(context.TODO(), nil)
+			_, err := r.Size(cctx, nil)
 			if err != nil {
 				return 0, fmt.Errorf("can't get size: %v", err)
 			}

swarm/storage/common_test.go

@@ -83,7 +83,7 @@ func newLDBStore(t *testing.T) (*LDBStore, func()) {
 	return db, cleanup
 }
 
-func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) {
+func mputRandomChunks(store ChunkStore, n int) ([]Chunk, error) {
 	return mput(store, n, GenerateRandomChunk)
 }
 
@@ -91,7 +91,7 @@ func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error
 	// put to localstore and wait for stored channel
 	// does not check delivery error state
 	errc := make(chan error)
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 	defer cancel()
 	for i := int64(0); i < int64(n); i++ {
 		chunk := f(ch.DefaultSize)
@@ -159,8 +159,8 @@ func (r *brokenLimitedReader) Read(buf []byte) (int, error) {
 	return r.lr.Read(buf)
 }
 
-func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) {
+func testStoreRandom(m ChunkStore, n int, t *testing.T) {
-	chunks, err := mputRandomChunks(m, n, chunksize)
+	chunks, err := mputRandomChunks(m, n)
 	if err != nil {
 		t.Fatalf("expected no error, got %v", err)
 	}
@@ -170,8 +170,8 @@ func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) {
 	}
 }
 
-func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) {
+func testStoreCorrect(m ChunkStore, n int, t *testing.T) {
-	chunks, err := mputRandomChunks(m, n, chunksize)
+	chunks, err := mputRandomChunks(m, n)
 	if err != nil {
 		t.Fatalf("expected no error, got %v", err)
 	}
@@ -195,7 +195,7 @@ func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) {
 	}
 }
 
-func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) {
+func benchmarkStorePut(store ChunkStore, n int, b *testing.B) {
 	chunks := make([]Chunk, n)
 	i := 0
 	f := func(dataSize int64) Chunk {
@@ -222,8 +222,8 @@ func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) {
 	}
 }
 
-func benchmarkStoreGet(store ChunkStore, n int, chunksize int64, b *testing.B) {
+func benchmarkStoreGet(store ChunkStore, n int, b *testing.B) {
-	chunks, err := mputRandomChunks(store, n, chunksize)
+	chunks, err := mputRandomChunks(store, n)
 	if err != nil {
 		b.Fatalf("expected no error, got %v", err)
 	}
@@ -267,6 +267,15 @@ func (m *MapChunkStore) Get(_ context.Context, ref Address) (Chunk, error) {
 	return chunk, nil
 }
 
+// Need to implement Has from SyncChunkStore
+func (m *MapChunkStore) Has(ctx context.Context, ref Address) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	_, has := m.chunks[ref.Hex()]
+	return has
+}
+
 func (m *MapChunkStore) Close() {
 }
 

swarm/storage/feed/testutil.go

@@ -40,9 +40,9 @@ func (t *TestHandler) Close() {
 
 type mockNetFetcher struct{}
 
-func (m *mockNetFetcher) Request(ctx context.Context, hopCount uint8) {
+func (m *mockNetFetcher) Request(hopCount uint8) {
 }
-func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) {
+func (m *mockNetFetcher) Offer(source *enode.ID) {
 }
 
 func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetFetcher {

swarm/storage/filestore.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"io"
 	"sort"
+	"sync"
 )
 
 /*
@@ -101,38 +102,45 @@ func (f *FileStore) HashSize() int {
 // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file
 func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) {
 	// create a special kind of putter, which only will store the references
-	putter := &HashExplorer{
+	putter := &hashExplorer{
 		hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt),
-		References:  make([]Reference, 0),
 	}
 	// do the actual splitting anyway, no way around it
-	_, _, err = PyramidSplit(ctx, data, putter, putter)
+	_, wait, err := PyramidSplit(ctx, data, putter, putter)
+	if err != nil {
+		return nil, err
+	}
+	// wait for splitting to be complete and all chunks processed
+	err = wait(ctx)
 	if err != nil {
 		return nil, err
 	}
 	// collect all references
 	addrs = NewAddressCollection(0)
-	for _, ref := range putter.References {
+	for _, ref := range putter.references {
 		addrs = append(addrs, Address(ref))
 	}
 	sort.Sort(addrs)
 	return addrs, nil
 }
 
-// HashExplorer is a special kind of putter which will only store chunk references
+// hashExplorer is a special kind of putter which will only store chunk references
-type HashExplorer struct {
+type hashExplorer struct {
 	*hasherStore
-	References []Reference
+	references []Reference
+	lock       sync.Mutex
 }
 
 // HashExplorer's Put will add just the chunk hashes to its `References`
-func (he *HashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
+func (he *hashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
 	// Need to do the actual Put, which returns the references
 	ref, err := he.hasherStore.Put(ctx, chunkData)
 	if err != nil {
 		return nil, err
 	}
 	// internally store the reference
-	he.References = append(he.References, ref)
+	he.lock.Lock()
+	he.references = append(he.references, ref)
+	he.lock.Unlock()
 	return ref, nil
 }

swarm/storage/ldbstore.go

@@ -969,6 +969,18 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error)
 	return s.get(addr)
 }
 
+// Has queries the underlying DB if a chunk with the given address is stored
+// Returns true if the chunk is found, false if not
+func (s *LDBStore) Has(_ context.Context, addr Address) bool {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	ikey := getIndexKey(addr)
+	_, err := s.db.Get(ikey)
+
+	return err == nil
+}
+
 // TODO: To conform with other private methods of this object indices should not be updated
 func (s *LDBStore) get(addr Address) (chunk *chunk, err error) {
 	if s.closed {
@@ -1037,7 +1049,6 @@ func (s *LDBStore) Close() {
 	s.lock.Unlock()
 	// force writing out current batch
 	s.writeCurrentBatch()
-	close(s.batchesC)
 	s.db.Close()
 }
 

swarm/storage/ldbstore_test.go

@@ -78,22 +78,22 @@ func testPoFunc(k Address) (ret uint8) {
 	return uint8(Proximity(basekey, k[:]))
 }
 
-func testDbStoreRandom(n int, chunksize int64, mock bool, t *testing.T) {
+func testDbStoreRandom(n int, mock bool, t *testing.T) {
 	db, cleanup, err := newTestDbStore(mock, true)
 	defer cleanup()
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
-	testStoreRandom(db, n, chunksize, t)
+	testStoreRandom(db, n, t)
 }
 
-func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) {
+func testDbStoreCorrect(n int, mock bool, t *testing.T) {
 	db, cleanup, err := newTestDbStore(mock, false)
 	defer cleanup()
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
-	testStoreCorrect(db, n, chunksize, t)
+	testStoreCorrect(db, n, t)
 }
 
 func TestMarkAccessed(t *testing.T) {
@@ -137,35 +137,35 @@ func TestMarkAccessed(t *testing.T) {
 }
 
 func TestDbStoreRandom_1(t *testing.T) {
-	testDbStoreRandom(1, 0, false, t)
+	testDbStoreRandom(1, false, t)
 }
 
 func TestDbStoreCorrect_1(t *testing.T) {
-	testDbStoreCorrect(1, 4096, false, t)
+	testDbStoreCorrect(1, false, t)
 }
 
 func TestDbStoreRandom_1k(t *testing.T) {
-	testDbStoreRandom(1000, 0, false, t)
+	testDbStoreRandom(1000, false, t)
 }
 
 func TestDbStoreCorrect_1k(t *testing.T) {
-	testDbStoreCorrect(1000, 4096, false, t)
+	testDbStoreCorrect(1000, false, t)
 }
 
 func TestMockDbStoreRandom_1(t *testing.T) {
-	testDbStoreRandom(1, 0, true, t)
+	testDbStoreRandom(1, true, t)
 }
 
 func TestMockDbStoreCorrect_1(t *testing.T) {
-	testDbStoreCorrect(1, 4096, true, t)
+	testDbStoreCorrect(1, true, t)
 }
 
 func TestMockDbStoreRandom_1k(t *testing.T) {
-	testDbStoreRandom(1000, 0, true, t)
+	testDbStoreRandom(1000, true, t)
 }
 
 func TestMockDbStoreCorrect_1k(t *testing.T) {
-	testDbStoreCorrect(1000, 4096, true, t)
+	testDbStoreCorrect(1000, true, t)
 }
 
 func testDbStoreNotFound(t *testing.T, mock bool) {
@@ -242,54 +242,38 @@ func TestMockIterator(t *testing.T) {
 	testIterator(t, true)
 }
 
-func benchmarkDbStorePut(n int, processors int, chunksize int64, mock bool, b *testing.B) {
+func benchmarkDbStorePut(n int, mock bool, b *testing.B) {
 	db, cleanup, err := newTestDbStore(mock, true)
 	defer cleanup()
 	if err != nil {
 		b.Fatalf("init dbStore failed: %v", err)
 	}
-	benchmarkStorePut(db, n, chunksize, b)
+	benchmarkStorePut(db, n, b)
 }
 
-func benchmarkDbStoreGet(n int, processors int, chunksize int64, mock bool, b *testing.B) {
+func benchmarkDbStoreGet(n int, mock bool, b *testing.B) {
 	db, cleanup, err := newTestDbStore(mock, true)
 	defer cleanup()
 	if err != nil {
 		b.Fatalf("init dbStore failed: %v", err)
 	}
-	benchmarkStoreGet(db, n, chunksize, b)
+	benchmarkStoreGet(db, n, b)
 }
 
-func BenchmarkDbStorePut_1_500(b *testing.B) {
+func BenchmarkDbStorePut_500(b *testing.B) {
-	benchmarkDbStorePut(500, 1, 4096, false, b)
+	benchmarkDbStorePut(500, false, b)
 }
 
-func BenchmarkDbStorePut_8_500(b *testing.B) {
+func BenchmarkDbStoreGet_500(b *testing.B) {
-	benchmarkDbStorePut(500, 8, 4096, false, b)
+	benchmarkDbStoreGet(500, false, b)
 }
 
-func BenchmarkDbStoreGet_1_500(b *testing.B) {
+func BenchmarkMockDbStorePut_500(b *testing.B) {
-	benchmarkDbStoreGet(500, 1, 4096, false, b)
+	benchmarkDbStorePut(500, true, b)
 }
 
-func BenchmarkDbStoreGet_8_500(b *testing.B) {
+func BenchmarkMockDbStoreGet_500(b *testing.B) {
-	benchmarkDbStoreGet(500, 8, 4096, false, b)
+	benchmarkDbStoreGet(500, true, b)
-}
-
-func BenchmarkMockDbStorePut_1_500(b *testing.B) {
-	benchmarkDbStorePut(500, 1, 4096, true, b)
-}
-
-func BenchmarkMockDbStorePut_8_500(b *testing.B) {
-	benchmarkDbStorePut(500, 8, 4096, true, b)
-}
-
-func BenchmarkMockDbStoreGet_1_500(b *testing.B) {
-	benchmarkDbStoreGet(500, 1, 4096, true, b)
-}
-
-func BenchmarkMockDbStoreGet_8_500(b *testing.B) {
-	benchmarkDbStoreGet(500, 8, 4096, true, b)
 }
 
 // TestLDBStoreWithoutCollectGarbage tests that we can put a number of random chunks in the LevelDB store, and
@@ -302,7 +286,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) {
 	ldb.setCapacity(uint64(capacity))
 	defer cleanup()
 
-	chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize))
+	chunks, err := mputRandomChunks(ldb, n)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -382,7 +366,7 @@ func testLDBStoreCollectGarbage(t *testing.T) {
 			putCount = roundTarget
 		}
 		remaining -= putCount
-		chunks, err := mputRandomChunks(ldb, putCount, int64(ch.DefaultSize))
+		chunks, err := mputRandomChunks(ldb, putCount)
 		if err != nil {
 			t.Fatal(err.Error())
 		}
@@ -429,7 +413,7 @@ func TestLDBStoreAddRemove(t *testing.T) {
 	defer cleanup()
 
 	n := 100
-	chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize))
+	chunks, err := mputRandomChunks(ldb, n)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
@@ -576,7 +560,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) {
 	ldb.setCapacity(uint64(capacity))
 	defer cleanup()
 
-	chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize))
+	chunks, err := mputRandomChunks(ldb, n)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -589,7 +573,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) {
 			t.Fatalf("fail add chunk #%d - %s: %v", i, chunks[i].Address(), err)
 		}
 	}
-	_, err = mputRandomChunks(ldb, 2, int64(ch.DefaultSize))
+	_, err = mputRandomChunks(ldb, 2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -621,7 +605,7 @@ func TestCleanIndex(t *testing.T) {
 	ldb.setCapacity(uint64(capacity))
 	defer cleanup()
 
-	chunks, err := mputRandomChunks(ldb, n, 4096)
+	chunks, err := mputRandomChunks(ldb, n)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -752,7 +736,7 @@ func TestCleanIndex(t *testing.T) {
 	}
 
 	// check that the iterator quits properly
-	chunks, err = mputRandomChunks(ldb, 4100, 4096)
+	chunks, err = mputRandomChunks(ldb, 4100)
 	if err != nil {
 		t.Fatal(err)
 	}

swarm/storage/localstore.go

@@ -132,6 +132,13 @@ func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error {
 	return err
 }
 
+// Has queries the underlying DbStore if a chunk with the given address
+// is being stored there.
+// Returns true if it is stored, false if not
+func (ls *LocalStore) Has(ctx context.Context, addr Address) bool {
+	return ls.DbStore.Has(ctx, addr)
+}
+
 // Get(chunk *Chunk) looks up a chunk in the local stores
 // This method is blocking until the chunk is retrieved
 // so additional timeout may be needed to wrap this call if

swarm/storage/localstore/doc.go

@@ -0,0 +1,56 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+Package localstore provides disk storage layer for Swarm Chunk persistence.
+It uses swarm/shed abstractions on top of github.com/syndtr/goleveldb LevelDB
+implementation.
+
+The main type is DB which manages the storage by providing methods to
+access and add Chunks and to manage their status.
+
+Modes are abstractions that do specific changes to Chunks. There are three
+mode types:
+
+ - ModeGet, for Chunk access
+ - ModePut, for adding Chunks to the database
+ - ModeSet, for changing Chunk statuses
+
+Every mode type has a corresponding type (Getter, Putter and Setter)
+that provides adequate method to perform the opperation and that type
+should be injected into localstore consumers instead the whole DB.
+This provides more clear insight which operations consumer is performing
+on the database.
+
+Getters, Putters and Setters accept different get, put and set modes
+to perform different actions. For example, ModeGet has two different
+variables ModeGetRequest and ModeGetSync and two different Getters
+can be constructed with them that are used when the chunk is requested
+or when the chunk is synced as this two events are differently changing
+the database.
+
+Subscription methods are implemented for a specific purpose of
+continuous iterations over Chunks that should be provided to
+Push and Pull syncing.
+
+DB implements an internal garbage collector that removes only synced
+Chunks from the database based on their most recent access time.
+
+Internally, DB stores Chunk data and any required information, such as
+store and access timestamps in different shed indexes that can be
+iterated on by garbage collector or subscriptions.
+*/
+package localstore

swarm/storage/localstore/gc.go

@@ -0,0 +1,302 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+Counting number of items in garbage collection index
+
+The number of items in garbage collection index is not the same as the number of
+chunks in retrieval index (total number of stored chunks). Chunk can be garbage
+collected only when it is set to a synced state by ModSetSync, and only then can
+be counted into garbage collection size, which determines whether a number of
+chunk should be removed from the storage by the garbage collection. This opens a
+possibility that the storage size exceeds the limit if files are locally
+uploaded and the node is not connected to other nodes or there is a problem with
+syncing.
+
+Tracking of garbage collection size (gcSize) is focused on performance. Key
+points:
+
+ 1. counting the number of key/value pairs in LevelDB takes around 0.7s for 1e6
+    on a very fast ssd (unacceptable long time in reality)
+ 2. locking leveldb batch writes with a global mutex (serial batch writes) is
+    not acceptable, we should use locking per chunk address
+
+Because of point 1. we cannot count the number of items in garbage collection
+index in New constructor as it could last very long for realistic scenarios
+where limit is 5e6 and nodes are running on slower hdd disks or cloud providers
+with low IOPS.
+
+Point 2. is a performance optimization to allow parallel batch writes with
+getters, putters and setters. Every single batch that they create contain only
+information related to a single chunk, no relations with other chunks or shared
+statistical data (like gcSize). This approach avoids race conditions on writing
+batches in parallel, but creates a problem of synchronizing statistical data
+values like gcSize. With global mutex lock, any data could be written by any
+batch, but would not use utilize the full potential of leveldb parallel writes.
+
+To mitigate this two problems, the implementation of counting and persisting
+gcSize is split into two parts. One is the in-memory value (gcSize) that is fast
+to read and write with a dedicated mutex (gcSizeMu) if the batch which adds or
+removes items from garbage collection index is successful. The second part is
+the reliable persistence of this value to leveldb database, as storedGCSize
+field. This database field is saved by writeGCSizeWorker and writeGCSize
+functions when in-memory gcSize variable is changed, but no too often to avoid
+very frequent database writes. This database writes are triggered by
+writeGCSizeTrigger when a call is made to function incGCSize. Trigger ensures
+that no database writes are done only when gcSize is changed (contrary to a
+simpler periodic writes or checks). A backoff of 10s in writeGCSizeWorker
+ensures that no frequent batch writes are made. Saving the storedGCSize on
+database Close function ensures that in-memory gcSize is persisted when database
+is closed.
+
+This persistence must be resilient to failures like panics. For this purpose, a
+collection of hashes that are added to the garbage collection index, but still
+not persisted to storedGCSize, must be tracked to count them in when DB is
+constructed again with New function after the failure (swarm node restarts). On
+every batch write that adds a new item to garbage collection index, the same
+hash is added to gcUncountedHashesIndex. This ensures that there is a persisted
+information which hashes were added to the garbage collection index. But, when
+the storedGCSize is saved by writeGCSize function, this values are removed in
+the same batch in which storedGCSize is changed to ensure consistency. When the
+panic happen, or database Close method is not saved. The database storage
+contains all information to reliably and efficiently get the correct number of
+items in garbage collection index. This is performed in the New function when
+all hashes in gcUncountedHashesIndex are counted, added to the storedGCSize and
+saved to the disk before the database is constructed again. Index
+gcUncountedHashesIndex is acting as dirty bit for recovery that provides
+information what needs to be corrected. With a simple dirty bit, the whole
+garbage collection index should me counted on recovery instead only the items in
+gcUncountedHashesIndex. Because of the triggering mechanizm of writeGCSizeWorker
+and relatively short backoff time, the number of hashes in
+gcUncountedHashesIndex should be low and it should take a very short time to
+recover from the previous failure. If there was no failure and
+gcUncountedHashesIndex is empty, which is the usual case, New function will take
+the minimal time to return.
+*/
+
+package localstore
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+var (
+	// gcTargetRatio defines the target number of items
+	// in garbage collection index that will not be removed
+	// on garbage collection. The target number of items
+	// is calculated by gcTarget function. This value must be
+	// in range (0,1]. For example, with 0.9 value,
+	// garbage collection will leave 90% of defined capacity
+	// in database after its run. This prevents frequent
+	// garbage collection runs.
+	gcTargetRatio = 0.9
+	// gcBatchSize limits the number of chunks in a single
+	// leveldb batch on garbage collection.
+	gcBatchSize int64 = 1000
+)
+
+// collectGarbageWorker is a long running function that waits for
+// collectGarbageTrigger channel to signal a garbage collection
+// run. GC run iterates on gcIndex and removes older items
+// form retrieval and other indexes.
+func (db *DB) collectGarbageWorker() {
+	for {
+		select {
+		case <-db.collectGarbageTrigger:
+			// run a single collect garbage run and
+			// if done is false, gcBatchSize is reached and
+			// another collect garbage run is needed
+			collectedCount, done, err := db.collectGarbage()
+			if err != nil {
+				log.Error("localstore collect garbage", "err", err)
+			}
+			// check if another gc run is needed
+			if !done {
+				db.triggerGarbageCollection()
+			}
+
+			if testHookCollectGarbage != nil {
+				testHookCollectGarbage(collectedCount)
+			}
+		case <-db.close:
+			return
+		}
+	}
+}
+
+// collectGarbage removes chunks from retrieval and other
+// indexes if maximal number of chunks in database is reached.
+// This function returns the number of removed chunks. If done
+// is false, another call to this function is needed to collect
+// the rest of the garbage as the batch size limit is reached.
+// This function is called in collectGarbageWorker.
+func (db *DB) collectGarbage() (collectedCount int64, done bool, err error) {
+	batch := new(leveldb.Batch)
+	target := db.gcTarget()
+
+	done = true
+	err = db.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
+		// protect parallel updates
+		unlock, err := db.lockAddr(item.Address)
+		if err != nil {
+			return false, err
+		}
+		defer unlock()
+
+		gcSize := db.getGCSize()
+		if gcSize-collectedCount <= target {
+			return true, nil
+		}
+		// delete from retrieve, pull, gc
+		db.retrievalDataIndex.DeleteInBatch(batch, item)
+		db.retrievalAccessIndex.DeleteInBatch(batch, item)
+		db.pullIndex.DeleteInBatch(batch, item)
+		db.gcIndex.DeleteInBatch(batch, item)
+		collectedCount++
+		if collectedCount >= gcBatchSize {
+			// bach size limit reached,
+			// another gc run is needed
+			done = false
+			return true, nil
+		}
+		return false, nil
+	}, nil)
+	if err != nil {
+		return 0, false, err
+	}
+
+	err = db.shed.WriteBatch(batch)
+	if err != nil {
+		return 0, false, err
+	}
+	// batch is written, decrement gcSize
+	db.incGCSize(-collectedCount)
+	return collectedCount, done, nil
+}
+
+// gcTrigger retruns the absolute value for garbage collection
+// target value, calculated from db.capacity and gcTargetRatio.
+func (db *DB) gcTarget() (target int64) {
+	return int64(float64(db.capacity) * gcTargetRatio)
+}
+
+// incGCSize increments gcSize by the provided number.
+// If count is negative, it will decrement gcSize.
+func (db *DB) incGCSize(count int64) {
+	if count == 0 {
+		return
+	}
+
+	db.gcSizeMu.Lock()
+	new := db.gcSize + count
+	db.gcSize = new
+	db.gcSizeMu.Unlock()
+
+	select {
+	case db.writeGCSizeTrigger <- struct{}{}:
+	default:
+	}
+	if new >= db.capacity {
+		db.triggerGarbageCollection()
+	}
+}
+
+// getGCSize returns gcSize value by locking it
+// with gcSizeMu mutex.
+func (db *DB) getGCSize() (count int64) {
+	db.gcSizeMu.RLock()
+	count = db.gcSize
+	db.gcSizeMu.RUnlock()
+	return count
+}
+
+// triggerGarbageCollection signals collectGarbageWorker
+// to call collectGarbage.
+func (db *DB) triggerGarbageCollection() {
+	select {
+	case db.collectGarbageTrigger <- struct{}{}:
+	case <-db.close:
+	default:
+	}
+}
+
+// writeGCSizeWorker writes gcSize on trigger event
+// and waits writeGCSizeDelay after each write.
+// It implements a linear backoff with delay of
+// writeGCSizeDelay duration to avoid very frequent
+// database operations.
+func (db *DB) writeGCSizeWorker() {
+	for {
+		select {
+		case <-db.writeGCSizeTrigger:
+			err := db.writeGCSize(db.getGCSize())
+			if err != nil {
+				log.Error("localstore write gc size", "err", err)
+			}
+			// Wait some time before writing gc size in the next
+			// iteration. This prevents frequent I/O operations.
+			select {
+			case <-time.After(10 * time.Second):
+			case <-db.close:
+				return
+			}
+		case <-db.close:
+			return
+		}
+	}
+}
+
+// writeGCSize stores the number of items in gcIndex.
+// It removes all hashes from gcUncountedHashesIndex
+// not to include them on the next DB initialization
+// (New function) when gcSize is counted.
+func (db *DB) writeGCSize(gcSize int64) (err error) {
+	const maxBatchSize = 1000
+
+	batch := new(leveldb.Batch)
+	db.storedGCSize.PutInBatch(batch, uint64(gcSize))
+	batchSize := 1
+
+	// use only one iterator as it acquires its snapshot
+	// not to remove hashes from index that are added
+	// after stored gc size is written
+	err = db.gcUncountedHashesIndex.Iterate(func(item shed.Item) (stop bool, err error) {
+		db.gcUncountedHashesIndex.DeleteInBatch(batch, item)
+		batchSize++
+		if batchSize >= maxBatchSize {
+			err = db.shed.WriteBatch(batch)
+			if err != nil {
+				return false, err
+			}
+			batch.Reset()
+			batchSize = 0
+		}
+		return false, nil
+	}, nil)
+	if err != nil {
+		return err
+	}
+	return db.shed.WriteBatch(batch)
+}
+
+// testHookCollectGarbage is a hook that can provide
+// information when a garbage collection run is done
+// and how many items it removed.
+var testHookCollectGarbage func(collectedCount int64)

swarm/storage/localstore/gc_test.go

@@ -0,0 +1,358 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// TestDB_collectGarbageWorker tests garbage collection runs
+// by uploading and syncing a number of chunks.
+func TestDB_collectGarbageWorker(t *testing.T) {
+	testDB_collectGarbageWorker(t)
+}
+
+// TestDB_collectGarbageWorker_multipleBatches tests garbage
+// collection runs by uploading and syncing a number of
+// chunks by having multiple smaller batches.
+func TestDB_collectGarbageWorker_multipleBatches(t *testing.T) {
+	// lower the maximal number of chunks in a single
+	// gc batch to ensure multiple batches.
+	defer func(s int64) { gcBatchSize = s }(gcBatchSize)
+	gcBatchSize = 2
+
+	testDB_collectGarbageWorker(t)
+}
+
+// testDB_collectGarbageWorker is a helper test function to test
+// garbage collection runs by uploading and syncing a number of chunks.
+func testDB_collectGarbageWorker(t *testing.T) {
+	chunkCount := 150
+
+	testHookCollectGarbageChan := make(chan int64)
+	defer setTestHookCollectGarbage(func(collectedCount int64) {
+		testHookCollectGarbageChan <- collectedCount
+	})()
+
+	db, cleanupFunc := newTestDB(t, &Options{
+		Capacity: 100,
+	})
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+	syncer := db.NewSetter(ModeSetSync)
+
+	addrs := make([]storage.Address, 0)
+
+	// upload random chunks
+	for i := 0; i < chunkCount; i++ {
+		chunk := generateRandomChunk()
+
+		err := uploader.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = syncer.Set(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addrs = append(addrs, chunk.Address())
+	}
+
+	gcTarget := db.gcTarget()
+
+	for {
+		select {
+		case <-testHookCollectGarbageChan:
+		case <-time.After(10 * time.Second):
+			t.Error("collect garbage timeout")
+		}
+		gcSize := db.getGCSize()
+		if gcSize == gcTarget {
+			break
+		}
+	}
+
+	t.Run("pull index count", newItemsCountTest(db.pullIndex, int(gcTarget)))
+
+	t.Run("gc index count", newItemsCountTest(db.gcIndex, int(gcTarget)))
+
+	t.Run("gc size", newIndexGCSizeTest(db))
+
+	// the first synced chunk should be removed
+	t.Run("get the first synced chunk", func(t *testing.T) {
+		_, err := db.NewGetter(ModeGetRequest).Get(addrs[0])
+		if err != storage.ErrChunkNotFound {
+			t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound)
+		}
+	})
+
+	// last synced chunk should not be removed
+	t.Run("get most recent synced chunk", func(t *testing.T) {
+		_, err := db.NewGetter(ModeGetRequest).Get(addrs[len(addrs)-1])
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// cleanup: drain the last testHookCollectGarbageChan
+	// element before calling deferred functions not to block
+	// collectGarbageWorker loop, preventing the race in
+	// setting testHookCollectGarbage function
+	select {
+	case <-testHookCollectGarbageChan:
+	default:
+	}
+}
+
+// TestDB_collectGarbageWorker_withRequests is a helper test function
+// to test garbage collection runs by uploading, syncing and
+// requesting a number of chunks.
+func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, &Options{
+		Capacity: 100,
+	})
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+	syncer := db.NewSetter(ModeSetSync)
+
+	testHookCollectGarbageChan := make(chan int64)
+	defer setTestHookCollectGarbage(func(collectedCount int64) {
+		testHookCollectGarbageChan <- collectedCount
+	})()
+
+	addrs := make([]storage.Address, 0)
+
+	// upload random chunks just up to the capacity
+	for i := 0; i < int(db.capacity)-1; i++ {
+		chunk := generateRandomChunk()
+
+		err := uploader.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = syncer.Set(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addrs = append(addrs, chunk.Address())
+	}
+
+	// request the latest synced chunk
+	// to prioritize it in the gc index
+	// not to be collected
+	_, err := db.NewGetter(ModeGetRequest).Get(addrs[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// upload and sync another chunk to trigger
+	// garbage collection
+	chunk := generateRandomChunk()
+	err = uploader.Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = syncer.Set(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+	addrs = append(addrs, chunk.Address())
+
+	// wait for garbage collection
+
+	gcTarget := db.gcTarget()
+
+	var totalCollectedCount int64
+	for {
+		select {
+		case c := <-testHookCollectGarbageChan:
+			totalCollectedCount += c
+		case <-time.After(10 * time.Second):
+			t.Error("collect garbage timeout")
+		}
+		gcSize := db.getGCSize()
+		if gcSize == gcTarget {
+			break
+		}
+	}
+
+	wantTotalCollectedCount := int64(len(addrs)) - gcTarget
+	if totalCollectedCount != wantTotalCollectedCount {
+		t.Errorf("total collected chunks %v, want %v", totalCollectedCount, wantTotalCollectedCount)
+	}
+
+	t.Run("pull index count", newItemsCountTest(db.pullIndex, int(gcTarget)))
+
+	t.Run("gc index count", newItemsCountTest(db.gcIndex, int(gcTarget)))
+
+	t.Run("gc size", newIndexGCSizeTest(db))
+
+	// requested chunk should not be removed
+	t.Run("get requested chunk", func(t *testing.T) {
+		_, err := db.NewGetter(ModeGetRequest).Get(addrs[0])
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// the second synced chunk should be removed
+	t.Run("get gc-ed chunk", func(t *testing.T) {
+		_, err := db.NewGetter(ModeGetRequest).Get(addrs[1])
+		if err != storage.ErrChunkNotFound {
+			t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound)
+		}
+	})
+
+	// last synced chunk should not be removed
+	t.Run("get most recent synced chunk", func(t *testing.T) {
+		_, err := db.NewGetter(ModeGetRequest).Get(addrs[len(addrs)-1])
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+// TestDB_gcSize checks if gcSize has a correct value after
+// database is initialized with existing data.
+func TestDB_gcSize(t *testing.T) {
+	dir, err := ioutil.TempDir("", "localstore-stored-gc-size")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	baseKey := make([]byte, 32)
+	if _, err := rand.Read(baseKey); err != nil {
+		t.Fatal(err)
+	}
+	db, err := New(dir, baseKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	uploader := db.NewPutter(ModePutUpload)
+	syncer := db.NewSetter(ModeSetSync)
+
+	count := 100
+
+	for i := 0; i < count; i++ {
+		chunk := generateRandomChunk()
+
+		err := uploader.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = syncer.Set(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// DB.Close writes gc size to disk, so
+	// Instead calling Close, simulate database shutdown
+	// without it.
+	close(db.close)
+	db.updateGCWG.Wait()
+	err = db.shed.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db, err = New(dir, baseKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("gc index size", newIndexGCSizeTest(db))
+
+	t.Run("gc uncounted hashes index count", newItemsCountTest(db.gcUncountedHashesIndex, 0))
+}
+
+// setTestHookCollectGarbage sets testHookCollectGarbage and
+// returns a function that will reset it to the
+// value before the change.
+func setTestHookCollectGarbage(h func(collectedCount int64)) (reset func()) {
+	current := testHookCollectGarbage
+	reset = func() { testHookCollectGarbage = current }
+	testHookCollectGarbage = h
+	return reset
+}
+
+// TestSetTestHookCollectGarbage tests if setTestHookCollectGarbage changes
+// testHookCollectGarbage function correctly and if its reset function
+// resets the original function.
+func TestSetTestHookCollectGarbage(t *testing.T) {
+	// Set the current function after the test finishes.
+	defer func(h func(collectedCount int64)) { testHookCollectGarbage = h }(testHookCollectGarbage)
+
+	// expected value for the unchanged function
+	original := 1
+	// expected value for the changed function
+	changed := 2
+
+	// this variable will be set with two different functions
+	var got int
+
+	// define the original (unchanged) functions
+	testHookCollectGarbage = func(_ int64) {
+		got = original
+	}
+
+	// set got variable
+	testHookCollectGarbage(0)
+
+	// test if got variable is set correctly
+	if got != original {
+		t.Errorf("got hook value %v, want %v", got, original)
+	}
+
+	// set the new function
+	reset := setTestHookCollectGarbage(func(_ int64) {
+		got = changed
+	})
+
+	// set got variable
+	testHookCollectGarbage(0)
+
+	// test if got variable is set correctly to changed value
+	if got != changed {
+		t.Errorf("got hook value %v, want %v", got, changed)
+	}
+
+	// set the function to the original one
+	reset()
+
+	// set got variable
+	testHookCollectGarbage(0)
+
+	// test if got variable is set correctly to original value
+	if got != original {
+		t.Errorf("got hook value %v, want %v", got, original)
+	}
+}

swarm/storage/localstore/index_test.go

@@ -0,0 +1,227 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"math/rand"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// TestDB_pullIndex validates the ordering of keys in pull index.
+// Pull index key contains PO prefix which is calculated from
+// DB base key and chunk address. This is not an Item field
+// which are checked in Mode tests.
+// This test uploads chunks, sorts them in expected order and
+// validates that pull index iterator will iterate it the same
+// order.
+func TestDB_pullIndex(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	chunkCount := 50
+
+	chunks := make([]testIndexChunk, chunkCount)
+
+	// upload random chunks
+	for i := 0; i < chunkCount; i++ {
+		chunk := generateRandomChunk()
+
+		err := uploader.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		chunks[i] = testIndexChunk{
+			Chunk: chunk,
+			// this timestamp is not the same as in
+			// the index, but given that uploads
+			// are sequential and that only ordering
+			// of events matter, this information is
+			// sufficient
+			storeTimestamp: now(),
+		}
+	}
+
+	testItemsOrder(t, db.pullIndex, chunks, func(i, j int) (less bool) {
+		poi := storage.Proximity(db.baseKey, chunks[i].Address())
+		poj := storage.Proximity(db.baseKey, chunks[j].Address())
+		if poi < poj {
+			return true
+		}
+		if poi > poj {
+			return false
+		}
+		if chunks[i].storeTimestamp < chunks[j].storeTimestamp {
+			return true
+		}
+		if chunks[i].storeTimestamp > chunks[j].storeTimestamp {
+			return false
+		}
+		return bytes.Compare(chunks[i].Address(), chunks[j].Address()) == -1
+	})
+}
+
+// TestDB_gcIndex validates garbage collection index by uploading
+// a chunk with and performing operations using synced, access and
+// request modes.
+func TestDB_gcIndex(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	chunkCount := 50
+
+	chunks := make([]testIndexChunk, chunkCount)
+
+	// upload random chunks
+	for i := 0; i < chunkCount; i++ {
+		chunk := generateRandomChunk()
+
+		err := uploader.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		chunks[i] = testIndexChunk{
+			Chunk: chunk,
+		}
+	}
+
+	// check if all chunks are stored
+	newItemsCountTest(db.pullIndex, chunkCount)(t)
+
+	// check that chunks are not collectable for garbage
+	newItemsCountTest(db.gcIndex, 0)(t)
+
+	// set update gc test hook to signal when
+	// update gc goroutine is done by sending to
+	// testHookUpdateGCChan channel, which is
+	// used to wait for indexes change verifications
+	testHookUpdateGCChan := make(chan struct{})
+	defer setTestHookUpdateGC(func() {
+		testHookUpdateGCChan <- struct{}{}
+	})()
+
+	t.Run("request unsynced", func(t *testing.T) {
+		chunk := chunks[1]
+
+		_, err := db.NewGetter(ModeGetRequest).Get(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+		// wait for update gc goroutine to be done
+		<-testHookUpdateGCChan
+
+		// the chunk is not synced
+		// should not be in the garbace collection index
+		newItemsCountTest(db.gcIndex, 0)(t)
+
+		newIndexGCSizeTest(db)(t)
+	})
+
+	t.Run("sync one chunk", func(t *testing.T) {
+		chunk := chunks[0]
+
+		err := db.NewSetter(ModeSetSync).Set(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// the chunk is synced and should be in gc index
+		newItemsCountTest(db.gcIndex, 1)(t)
+
+		newIndexGCSizeTest(db)(t)
+	})
+
+	t.Run("sync all chunks", func(t *testing.T) {
+		setter := db.NewSetter(ModeSetSync)
+
+		for i := range chunks {
+			err := setter.Set(chunks[i].Address())
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		testItemsOrder(t, db.gcIndex, chunks, nil)
+
+		newIndexGCSizeTest(db)(t)
+	})
+
+	t.Run("request one chunk", func(t *testing.T) {
+		i := 6
+
+		_, err := db.NewGetter(ModeGetRequest).Get(chunks[i].Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+		// wait for update gc goroutine to be done
+		<-testHookUpdateGCChan
+
+		// move the chunk to the end of the expected gc
+		c := chunks[i]
+		chunks = append(chunks[:i], chunks[i+1:]...)
+		chunks = append(chunks, c)
+
+		testItemsOrder(t, db.gcIndex, chunks, nil)
+
+		newIndexGCSizeTest(db)(t)
+	})
+
+	t.Run("random chunk request", func(t *testing.T) {
+		requester := db.NewGetter(ModeGetRequest)
+
+		rand.Shuffle(len(chunks), func(i, j int) {
+			chunks[i], chunks[j] = chunks[j], chunks[i]
+		})
+
+		for _, chunk := range chunks {
+			_, err := requester.Get(chunk.Address())
+			if err != nil {
+				t.Fatal(err)
+			}
+			// wait for update gc goroutine to be done
+			<-testHookUpdateGCChan
+		}
+
+		testItemsOrder(t, db.gcIndex, chunks, nil)
+
+		newIndexGCSizeTest(db)(t)
+	})
+
+	t.Run("remove one chunk", func(t *testing.T) {
+		i := 3
+
+		err := db.NewSetter(modeSetRemove).Set(chunks[i].Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// remove the chunk from the expected chunks in gc index
+		chunks = append(chunks[:i], chunks[i+1:]...)
+
+		testItemsOrder(t, db.gcIndex, chunks, nil)
+
+		newIndexGCSizeTest(db)(t)
+	})
+}

swarm/storage/localstore/localstore.go

@@ -0,0 +1,431 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+var (
+	// ErrInvalidMode is retuned when an unknown Mode
+	// is provided to the function.
+	ErrInvalidMode = errors.New("invalid mode")
+	// ErrAddressLockTimeout is returned when the same chunk
+	// is updated in parallel and one of the updates
+	// takes longer then the configured timeout duration.
+	ErrAddressLockTimeout = errors.New("address lock timeout")
+)
+
+var (
+	// Default value for Capacity DB option.
+	defaultCapacity int64 = 5000000
+	// Limit the number of goroutines created by Getters
+	// that call updateGC function. Value 0 sets no limit.
+	maxParallelUpdateGC = 1000
+)
+
+// DB is the local store implementation and holds
+// database related objects.
+type DB struct {
+	shed *shed.DB
+
+	// schema name of loaded data
+	schemaName shed.StringField
+	// field that stores number of intems in gc index
+	storedGCSize shed.Uint64Field
+
+	// retrieval indexes
+	retrievalDataIndex   shed.Index
+	retrievalAccessIndex shed.Index
+	// push syncing index
+	pushIndex shed.Index
+	// push syncing subscriptions triggers
+	pushTriggers   []chan struct{}
+	pushTriggersMu sync.RWMutex
+
+	// pull syncing index
+	pullIndex shed.Index
+	// pull syncing subscriptions triggers per bin
+	pullTriggers   map[uint8][]chan struct{}
+	pullTriggersMu sync.RWMutex
+
+	// garbage collection index
+	gcIndex shed.Index
+	// index that stores hashes that are not
+	// counted in and saved to storedGCSize
+	gcUncountedHashesIndex shed.Index
+
+	// number of elements in garbage collection index
+	// it must be always read by getGCSize and
+	// set with incGCSize which are locking gcSizeMu
+	gcSize   int64
+	gcSizeMu sync.RWMutex
+	// garbage collection is triggered when gcSize exceeds
+	// the capacity value
+	capacity int64
+
+	// triggers garbage collection event loop
+	collectGarbageTrigger chan struct{}
+	// triggers write gc size event loop
+	writeGCSizeTrigger chan struct{}
+
+	// a buffered channel acting as a semaphore
+	// to limit the maximal number of goroutines
+	// created by Getters to call updateGC function
+	updateGCSem chan struct{}
+	// a wait group to ensure all updateGC goroutines
+	// are done before closing the database
+	updateGCWG sync.WaitGroup
+
+	baseKey []byte
+
+	addressLocks sync.Map
+
+	// this channel is closed when close function is called
+	// to terminate other goroutines
+	close chan struct{}
+}
+
+// Options struct holds optional parameters for configuring DB.
+type Options struct {
+	// MockStore is a mock node store that is used to store
+	// chunk data in a central store. It can be used to reduce
+	// total storage space requirements in testing large number
+	// of swarm nodes with chunk data deduplication provided by
+	// the mock global store.
+	MockStore *mock.NodeStore
+	// Capacity is a limit that triggers garbage collection when
+	// number of items in gcIndex equals or exceeds it.
+	Capacity int64
+	// MetricsPrefix defines a prefix for metrics names.
+	MetricsPrefix string
+}
+
+// New returns a new DB.  All fields and indexes are initialized
+// and possible conflicts with schema from existing database is checked.
+// One goroutine for writing batches is created.
+func New(path string, baseKey []byte, o *Options) (db *DB, err error) {
+	if o == nil {
+		o = new(Options)
+	}
+	db = &DB{
+		capacity: o.Capacity,
+		baseKey:  baseKey,
+		// channels collectGarbageTrigger and writeGCSizeTrigger
+		// need to be buffered with the size of 1
+		// to signal another event if it
+		// is triggered during already running function
+		collectGarbageTrigger: make(chan struct{}, 1),
+		writeGCSizeTrigger:    make(chan struct{}, 1),
+		close:                 make(chan struct{}),
+	}
+	if db.capacity <= 0 {
+		db.capacity = defaultCapacity
+	}
+	if maxParallelUpdateGC > 0 {
+		db.updateGCSem = make(chan struct{}, maxParallelUpdateGC)
+	}
+
+	db.shed, err = shed.NewDB(path, o.MetricsPrefix)
+	if err != nil {
+		return nil, err
+	}
+	// Identify current storage schema by arbitrary name.
+	db.schemaName, err = db.shed.NewStringField("schema-name")
+	if err != nil {
+		return nil, err
+	}
+	// Persist gc size.
+	db.storedGCSize, err = db.shed.NewUint64Field("gc-size")
+	if err != nil {
+		return nil, err
+	}
+	// Functions for retrieval data index.
+	var (
+		encodeValueFunc func(fields shed.Item) (value []byte, err error)
+		decodeValueFunc func(keyItem shed.Item, value []byte) (e shed.Item, err error)
+	)
+	if o.MockStore != nil {
+		encodeValueFunc = func(fields shed.Item) (value []byte, err error) {
+			b := make([]byte, 8)
+			binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
+			err = o.MockStore.Put(fields.Address, fields.Data)
+			if err != nil {
+				return nil, err
+			}
+			return b, nil
+		}
+		decodeValueFunc = func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
+			e.Data, err = o.MockStore.Get(keyItem.Address)
+			return e, err
+		}
+	} else {
+		encodeValueFunc = func(fields shed.Item) (value []byte, err error) {
+			b := make([]byte, 8)
+			binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
+			value = append(b, fields.Data...)
+			return value, nil
+		}
+		decodeValueFunc = func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
+			e.Data = value[8:]
+			return e, nil
+		}
+	}
+	// Index storing actual chunk address, data and store timestamp.
+	db.retrievalDataIndex, err = db.shed.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
+		EncodeKey: func(fields shed.Item) (key []byte, err error) {
+			return fields.Address, nil
+		},
+		DecodeKey: func(key []byte) (e shed.Item, err error) {
+			e.Address = key
+			return e, nil
+		},
+		EncodeValue: encodeValueFunc,
+		DecodeValue: decodeValueFunc,
+	})
+	if err != nil {
+		return nil, err
+	}
+	// Index storing access timestamp for a particular address.
+	// It is needed in order to update gc index keys for iteration order.
+	db.retrievalAccessIndex, err = db.shed.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{
+		EncodeKey: func(fields shed.Item) (key []byte, err error) {
+			return fields.Address, nil
+		},
+		DecodeKey: func(key []byte) (e shed.Item, err error) {
+			e.Address = key
+			return e, nil
+		},
+		EncodeValue: func(fields shed.Item) (value []byte, err error) {
+			b := make([]byte, 8)
+			binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
+			return b, nil
+		},
+		DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
+			return e, nil
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// pull index allows history and live syncing per po bin
+	db.pullIndex, err = db.shed.NewIndex("PO|StoredTimestamp|Hash->nil", shed.IndexFuncs{
+		EncodeKey: func(fields shed.Item) (key []byte, err error) {
+			key = make([]byte, 41)
+			key[0] = db.po(fields.Address)
+			binary.BigEndian.PutUint64(key[1:9], uint64(fields.StoreTimestamp))
+			copy(key[9:], fields.Address[:])
+			return key, nil
+		},
+		DecodeKey: func(key []byte) (e shed.Item, err error) {
+			e.Address = key[9:]
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[1:9]))
+			return e, nil
+		},
+		EncodeValue: func(fields shed.Item) (value []byte, err error) {
+			return nil, nil
+		},
+		DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			return e, nil
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// create a pull syncing triggers used by SubscribePull function
+	db.pullTriggers = make(map[uint8][]chan struct{})
+	// push index contains as yet unsynced chunks
+	db.pushIndex, err = db.shed.NewIndex("StoredTimestamp|Hash->nil", shed.IndexFuncs{
+		EncodeKey: func(fields shed.Item) (key []byte, err error) {
+			key = make([]byte, 40)
+			binary.BigEndian.PutUint64(key[:8], uint64(fields.StoreTimestamp))
+			copy(key[8:], fields.Address[:])
+			return key, nil
+		},
+		DecodeKey: func(key []byte) (e shed.Item, err error) {
+			e.Address = key[8:]
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
+			return e, nil
+		},
+		EncodeValue: func(fields shed.Item) (value []byte, err error) {
+			return nil, nil
+		},
+		DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			return e, nil
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// create a push syncing triggers used by SubscribePush function
+	db.pushTriggers = make([]chan struct{}, 0)
+	// gc index for removable chunk ordered by ascending last access time
+	db.gcIndex, err = db.shed.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", shed.IndexFuncs{
+		EncodeKey: func(fields shed.Item) (key []byte, err error) {
+			b := make([]byte, 16, 16+len(fields.Address))
+			binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
+			binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
+			key = append(b, fields.Address...)
+			return key, nil
+		},
+		DecodeKey: func(key []byte) (e shed.Item, err error) {
+			e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
+			e.Address = key[16:]
+			return e, nil
+		},
+		EncodeValue: func(fields shed.Item) (value []byte, err error) {
+			return nil, nil
+		},
+		DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			return e, nil
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// gc uncounted hashes index keeps hashes that are in gc index
+	// but not counted in and saved to storedGCSize
+	db.gcUncountedHashesIndex, err = db.shed.NewIndex("Hash->nil", shed.IndexFuncs{
+		EncodeKey: func(fields shed.Item) (key []byte, err error) {
+			return fields.Address, nil
+		},
+		DecodeKey: func(key []byte) (e shed.Item, err error) {
+			e.Address = key
+			return e, nil
+		},
+		EncodeValue: func(fields shed.Item) (value []byte, err error) {
+			return nil, nil
+		},
+		DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
+			return e, nil
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// count number of elements in garbage collection index
+	gcSize, err := db.storedGCSize.Get()
+	if err != nil {
+		return nil, err
+	}
+	// get number of uncounted hashes
+	gcUncountedSize, err := db.gcUncountedHashesIndex.Count()
+	if err != nil {
+		return nil, err
+	}
+	gcSize += uint64(gcUncountedSize)
+	// remove uncounted hashes from the index and
+	// save the total gcSize after uncounted hashes are removed
+	err = db.writeGCSize(int64(gcSize))
+	if err != nil {
+		return nil, err
+	}
+	db.incGCSize(int64(gcSize))
+
+	// start worker to write gc size
+	go db.writeGCSizeWorker()
+	// start garbage collection worker
+	go db.collectGarbageWorker()
+	return db, nil
+}
+
+// Close closes the underlying database.
+func (db *DB) Close() (err error) {
+	close(db.close)
+	db.updateGCWG.Wait()
+	if err := db.writeGCSize(db.getGCSize()); err != nil {
+		log.Error("localstore: write gc size", "err", err)
+	}
+	return db.shed.Close()
+}
+
+// po computes the proximity order between the address
+// and database base key.
+func (db *DB) po(addr storage.Address) (bin uint8) {
+	return uint8(storage.Proximity(db.baseKey, addr))
+}
+
+var (
+	// Maximal time for lockAddr to wait until it
+	// returns error.
+	addressLockTimeout = 3 * time.Second
+	// duration between two lock checks in lockAddr.
+	addressLockCheckDelay = 30 * time.Microsecond
+)
+
+// lockAddr sets the lock on a particular address
+// using addressLocks sync.Map and returns unlock function.
+// If the address is locked this function will check it
+// in a for loop for addressLockTimeout time, after which
+// it will return ErrAddressLockTimeout error.
+func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) {
+	start := time.Now()
+	lockKey := hex.EncodeToString(addr)
+	for {
+		_, loaded := db.addressLocks.LoadOrStore(lockKey, struct{}{})
+		if !loaded {
+			break
+		}
+		time.Sleep(addressLockCheckDelay)
+		if time.Since(start) > addressLockTimeout {
+			return nil, ErrAddressLockTimeout
+		}
+	}
+	return func() { db.addressLocks.Delete(lockKey) }, nil
+}
+
+// chunkToItem creates new Item with data provided by the Chunk.
+func chunkToItem(ch storage.Chunk) shed.Item {
+	return shed.Item{
+		Address: ch.Address(),
+		Data:    ch.Data(),
+	}
+}
+
+// addressToItem creates new Item with a provided address.
+func addressToItem(addr storage.Address) shed.Item {
+	return shed.Item{
+		Address: addr,
+	}
+}
+
+// now is a helper function that returns a current unix timestamp
+// in UTC timezone.
+// It is set in the init function for usage in production, and
+// optionally overridden in tests for data validation.
+var now func() int64
+
+func init() {
+	// set the now function
+	now = func() (t int64) {
+		return time.Now().UTC().UnixNano()
+	}
+}

swarm/storage/localstore/localstore_test.go

@@ -0,0 +1,520 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"sort"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	ch "github.com/ethereum/go-ethereum/swarm/chunk"
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// TestDB validates if the chunk can be uploaded and
+// correctly retrieved.
+func TestDB(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := db.NewGetter(ModeGetRequest).Get(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(got.Address(), chunk.Address()) {
+		t.Errorf("got address %x, want %x", got.Address(), chunk.Address())
+	}
+	if !bytes.Equal(got.Data(), chunk.Data()) {
+		t.Errorf("got data %x, want %x", got.Data(), chunk.Data())
+	}
+}
+
+// TestDB_updateGCSem tests maxParallelUpdateGC limit.
+// This test temporary sets the limit to a low number,
+// makes updateGC function execution time longer by
+// setting a custom testHookUpdateGC function with a sleep
+// and a count current and maximal number of goroutines.
+func TestDB_updateGCSem(t *testing.T) {
+	updateGCSleep := time.Second
+	var count int
+	var max int
+	var mu sync.Mutex
+	defer setTestHookUpdateGC(func() {
+		mu.Lock()
+		// add to the count of current goroutines
+		count++
+		if count > max {
+			// set maximal detected numbers of goroutines
+			max = count
+		}
+		mu.Unlock()
+
+		// wait for some time to ensure multiple parallel goroutines
+		time.Sleep(updateGCSleep)
+
+		mu.Lock()
+		count--
+		mu.Unlock()
+	})()
+
+	defer func(m int) { maxParallelUpdateGC = m }(maxParallelUpdateGC)
+	maxParallelUpdateGC = 3
+
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	getter := db.NewGetter(ModeGetRequest)
+
+	// get more chunks then maxParallelUpdateGC
+	// in time shorter then updateGCSleep
+	for i := 0; i < 5; i++ {
+		_, err = getter.Get(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if max != maxParallelUpdateGC {
+		t.Errorf("got max %v, want %v", max, maxParallelUpdateGC)
+	}
+}
+
+// BenchmarkNew measures the time that New function
+// needs to initialize and count the number of key/value
+// pairs in GC index.
+// This benchmark generates a number of chunks, uploads them,
+// sets them to synced state for them to enter the GC index,
+// and measures the execution time of New function by creating
+// new databases with the same data directory.
+//
+// This benchmark takes significant amount of time.
+//
+// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014) show
+// that New function executes around 1s for database with 1M chunks.
+//
+// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkNew -v -timeout 20m
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore
+// BenchmarkNew/1000-8         	     200	  11672414 ns/op	 9570960 B/op	   10008 allocs/op
+// BenchmarkNew/10000-8        	     100	  14890609 ns/op	10490118 B/op	    7759 allocs/op
+// BenchmarkNew/100000-8       	      20	  58334080 ns/op	17763157 B/op	   22978 allocs/op
+// BenchmarkNew/1000000-8      	       2	 748595153 ns/op	45297404 B/op	  253242 allocs/op
+// PASS
+func BenchmarkNew(b *testing.B) {
+	if testing.Short() {
+		b.Skip("skipping benchmark in short mode")
+	}
+	for _, count := range []int{
+		1000,
+		10000,
+		100000,
+		1000000,
+	} {
+		b.Run(strconv.Itoa(count), func(b *testing.B) {
+			dir, err := ioutil.TempDir("", "localstore-new-benchmark")
+			if err != nil {
+				b.Fatal(err)
+			}
+			defer os.RemoveAll(dir)
+			baseKey := make([]byte, 32)
+			if _, err := rand.Read(baseKey); err != nil {
+				b.Fatal(err)
+			}
+			db, err := New(dir, baseKey, nil)
+			if err != nil {
+				b.Fatal(err)
+			}
+			uploader := db.NewPutter(ModePutUpload)
+			syncer := db.NewSetter(ModeSetSync)
+			for i := 0; i < count; i++ {
+				chunk := generateFakeRandomChunk()
+				err := uploader.Put(chunk)
+				if err != nil {
+					b.Fatal(err)
+				}
+				err = syncer.Set(chunk.Address())
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+			err = db.Close()
+			if err != nil {
+				b.Fatal(err)
+			}
+			b.ResetTimer()
+
+			for n := 0; n < b.N; n++ {
+				b.StartTimer()
+				db, err := New(dir, baseKey, nil)
+				b.StopTimer()
+
+				if err != nil {
+					b.Fatal(err)
+				}
+				err = db.Close()
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// newTestDB is a helper function that constructs a
+// temporary database and returns a cleanup function that must
+// be called to remove the data.
+func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) {
+	t.Helper()
+
+	dir, err := ioutil.TempDir("", "localstore-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cleanupFunc = func() { os.RemoveAll(dir) }
+	baseKey := make([]byte, 32)
+	if _, err := rand.Read(baseKey); err != nil {
+		t.Fatal(err)
+	}
+	db, err = New(dir, baseKey, o)
+	if err != nil {
+		cleanupFunc()
+		t.Fatal(err)
+	}
+	cleanupFunc = func() {
+		err := db.Close()
+		if err != nil {
+			t.Error(err)
+		}
+		os.RemoveAll(dir)
+	}
+	return db, cleanupFunc
+}
+
+// generateRandomChunk generates a valid Chunk with
+// data size of default chunk size.
+func generateRandomChunk() storage.Chunk {
+	return storage.GenerateRandomChunk(ch.DefaultSize)
+}
+
+func init() {
+	// needed for generateFakeRandomChunk
+	rand.Seed(time.Now().UnixNano())
+}
+
+// generateFakeRandomChunk generates a Chunk that is not
+// valid, but it contains a random key and a random value.
+// This function is faster then storage.GenerateRandomChunk
+// which generates a valid chunk.
+// Some tests in this package do not need valid chunks, just
+// random data, and their execution time can be decreased
+// using this function.
+func generateFakeRandomChunk() storage.Chunk {
+	data := make([]byte, ch.DefaultSize)
+	rand.Read(data)
+	key := make([]byte, 32)
+	rand.Read(key)
+	return storage.NewChunk(key, data)
+}
+
+// TestGenerateFakeRandomChunk validates that
+// generateFakeRandomChunk returns random data by comparing
+// two generated chunks.
+func TestGenerateFakeRandomChunk(t *testing.T) {
+	c1 := generateFakeRandomChunk()
+	c2 := generateFakeRandomChunk()
+	addrLen := len(c1.Address())
+	if addrLen != 32 {
+		t.Errorf("first chunk address length %v, want %v", addrLen, 32)
+	}
+	dataLen := len(c1.Data())
+	if dataLen != ch.DefaultSize {
+		t.Errorf("first chunk data length %v, want %v", dataLen, ch.DefaultSize)
+	}
+	addrLen = len(c2.Address())
+	if addrLen != 32 {
+		t.Errorf("second chunk address length %v, want %v", addrLen, 32)
+	}
+	dataLen = len(c2.Data())
+	if dataLen != ch.DefaultSize {
+		t.Errorf("second chunk data length %v, want %v", dataLen, ch.DefaultSize)
+	}
+	if bytes.Equal(c1.Address(), c2.Address()) {
+		t.Error("fake chunks addresses do not differ")
+	}
+	if bytes.Equal(c1.Data(), c2.Data()) {
+		t.Error("fake chunks data bytes do not differ")
+	}
+}
+
+// newRetrieveIndexesTest returns a test function that validates if the right
+// chunk values are in the retrieval indexes.
+func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
+	return func(t *testing.T) {
+		item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
+		if err != nil {
+			t.Fatal(err)
+		}
+		validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0)
+
+		// access index should not be set
+		wantErr := leveldb.ErrNotFound
+		item, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address()))
+		if err != wantErr {
+			t.Errorf("got error %v, want %v", err, wantErr)
+		}
+	}
+}
+
+// newRetrieveIndexesTestWithAccess returns a test function that validates if the right
+// chunk values are in the retrieval indexes when access time must be stored.
+func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
+	return func(t *testing.T) {
+		item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
+		if err != nil {
+			t.Fatal(err)
+		}
+		validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0)
+
+		if accessTimestamp > 0 {
+			item, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address()))
+			if err != nil {
+				t.Fatal(err)
+			}
+			validateItem(t, item, chunk.Address(), nil, 0, accessTimestamp)
+		}
+	}
+}
+
+// newPullIndexTest returns a test function that validates if the right
+// chunk values are in the pull index.
+func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
+	return func(t *testing.T) {
+		item, err := db.pullIndex.Get(shed.Item{
+			Address:        chunk.Address(),
+			StoreTimestamp: storeTimestamp,
+		})
+		if err != wantError {
+			t.Errorf("got error %v, want %v", err, wantError)
+		}
+		if err == nil {
+			validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0)
+		}
+	}
+}
+
+// newPushIndexTest returns a test function that validates if the right
+// chunk values are in the push index.
+func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
+	return func(t *testing.T) {
+		item, err := db.pushIndex.Get(shed.Item{
+			Address:        chunk.Address(),
+			StoreTimestamp: storeTimestamp,
+		})
+		if err != wantError {
+			t.Errorf("got error %v, want %v", err, wantError)
+		}
+		if err == nil {
+			validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0)
+		}
+	}
+}
+
+// newGCIndexTest returns a test function that validates if the right
+// chunk values are in the push index.
+func newGCIndexTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
+	return func(t *testing.T) {
+		item, err := db.gcIndex.Get(shed.Item{
+			Address:         chunk.Address(),
+			StoreTimestamp:  storeTimestamp,
+			AccessTimestamp: accessTimestamp,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		validateItem(t, item, chunk.Address(), nil, storeTimestamp, accessTimestamp)
+	}
+}
+
+// newItemsCountTest returns a test function that validates if
+// an index contains expected number of key/value pairs.
+func newItemsCountTest(i shed.Index, want int) func(t *testing.T) {
+	return func(t *testing.T) {
+		var c int
+		err := i.Iterate(func(item shed.Item) (stop bool, err error) {
+			c++
+			return
+		}, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if c != want {
+			t.Errorf("got %v items in index, want %v", c, want)
+		}
+	}
+}
+
+// newIndexGCSizeTest retruns a test function that validates if DB.gcSize
+// value is the same as the number of items in DB.gcIndex.
+func newIndexGCSizeTest(db *DB) func(t *testing.T) {
+	return func(t *testing.T) {
+		var want int64
+		err := db.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
+			want++
+			return
+		}, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got := db.getGCSize()
+		if got != want {
+			t.Errorf("got gc size %v, want %v", got, want)
+		}
+	}
+}
+
+// testIndexChunk embeds storageChunk with additional data that is stored
+// in database. It is used for index values validations.
+type testIndexChunk struct {
+	storage.Chunk
+	storeTimestamp int64
+}
+
+// testItemsOrder tests the order of chunks in the index. If sortFunc is not nil,
+// chunks will be sorted with it before validation.
+func testItemsOrder(t *testing.T, i shed.Index, chunks []testIndexChunk, sortFunc func(i, j int) (less bool)) {
+	newItemsCountTest(i, len(chunks))(t)
+
+	if sortFunc != nil {
+		sort.Slice(chunks, sortFunc)
+	}
+
+	var cursor int
+	err := i.Iterate(func(item shed.Item) (stop bool, err error) {
+		want := chunks[cursor].Address()
+		got := item.Address
+		if !bytes.Equal(got, want) {
+			return true, fmt.Errorf("got address %x at position %v, want %x", got, cursor, want)
+		}
+		cursor++
+		return false, nil
+	}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// validateItem is a helper function that checks Item values.
+func validateItem(t *testing.T, item shed.Item, address, data []byte, storeTimestamp, accessTimestamp int64) {
+	t.Helper()
+
+	if !bytes.Equal(item.Address, address) {
+		t.Errorf("got item address %x, want %x", item.Address, address)
+	}
+	if !bytes.Equal(item.Data, data) {
+		t.Errorf("got item data %x, want %x", item.Data, data)
+	}
+	if item.StoreTimestamp != storeTimestamp {
+		t.Errorf("got item store timestamp %v, want %v", item.StoreTimestamp, storeTimestamp)
+	}
+	if item.AccessTimestamp != accessTimestamp {
+		t.Errorf("got item access timestamp %v, want %v", item.AccessTimestamp, accessTimestamp)
+	}
+}
+
+// setNow replaces now function and
+// returns a function that will reset it to the
+// value before the change.
+func setNow(f func() int64) (reset func()) {
+	current := now
+	reset = func() { now = current }
+	now = f
+	return reset
+}
+
+// TestSetNow tests if setNow function changes now function
+// correctly and if its reset function resets the original function.
+func TestSetNow(t *testing.T) {
+	// set the current function after the test finishes
+	defer func(f func() int64) { now = f }(now)
+
+	// expected value for the unchanged function
+	var original int64 = 1
+	// expected value for the changed function
+	var changed int64 = 2
+
+	// define the original (unchanged) functions
+	now = func() int64 {
+		return original
+	}
+
+	// get the time
+	got := now()
+
+	// test if got variable is set correctly
+	if got != original {
+		t.Errorf("got now value %v, want %v", got, original)
+	}
+
+	// set the new function
+	reset := setNow(func() int64 {
+		return changed
+	})
+
+	// get the time
+	got = now()
+
+	// test if got variable is set correctly to changed value
+	if got != changed {
+		t.Errorf("got hook value %v, want %v", got, changed)
+	}
+
+	// set the function to the original one
+	reset()
+
+	// get the time
+	got = now()
+
+	// test if got variable is set correctly to original value
+	if got != original {
+		t.Errorf("got hook value %v, want %v", got, original)
+	}
+}

swarm/storage/localstore/mode_get.go

@@ -0,0 +1,154 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// ModeGet enumerates different Getter modes.
+type ModeGet int
+
+// Getter modes.
+const (
+	// ModeGetRequest: when accessed for retrieval
+	ModeGetRequest ModeGet = iota
+	// ModeGetSync: when accessed for syncing or proof of custody request
+	ModeGetSync
+)
+
+// Getter provides Get method to retrieve Chunks
+// from database.
+type Getter struct {
+	db   *DB
+	mode ModeGet
+}
+
+// NewGetter returns a new Getter on database
+// with a specific Mode.
+func (db *DB) NewGetter(mode ModeGet) *Getter {
+	return &Getter{
+		mode: mode,
+		db:   db,
+	}
+}
+
+// Get returns a chunk from the database. If the chunk is
+// not found storage.ErrChunkNotFound will be returned.
+// All required indexes will be updated required by the
+// Getter Mode.
+func (g *Getter) Get(addr storage.Address) (chunk storage.Chunk, err error) {
+	out, err := g.db.get(g.mode, addr)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, storage.ErrChunkNotFound
+		}
+		return nil, err
+	}
+	return storage.NewChunk(out.Address, out.Data), nil
+}
+
+// get returns Item from the retrieval index
+// and updates other indexes.
+func (db *DB) get(mode ModeGet, addr storage.Address) (out shed.Item, err error) {
+	item := addressToItem(addr)
+
+	out, err = db.retrievalDataIndex.Get(item)
+	if err != nil {
+		return out, err
+	}
+	switch mode {
+	// update the access timestamp and gc index
+	case ModeGetRequest:
+		if db.updateGCSem != nil {
+			// wait before creating new goroutines
+			// if updateGCSem buffer id full
+			db.updateGCSem <- struct{}{}
+		}
+		db.updateGCWG.Add(1)
+		go func() {
+			defer db.updateGCWG.Done()
+			if db.updateGCSem != nil {
+				// free a spot in updateGCSem buffer
+				// for a new goroutine
+				defer func() { <-db.updateGCSem }()
+			}
+			err := db.updateGC(out)
+			if err != nil {
+				log.Error("localstore update gc", "err", err)
+			}
+			// if gc update hook is defined, call it
+			if testHookUpdateGC != nil {
+				testHookUpdateGC()
+			}
+		}()
+
+	// no updates to indexes
+	case ModeGetSync:
+	default:
+		return out, ErrInvalidMode
+	}
+	return out, nil
+}
+
+// updateGC updates garbage collection index for
+// a single item. Provided item is expected to have
+// only Address and Data fields with non zero values,
+// which is ensured by the get function.
+func (db *DB) updateGC(item shed.Item) (err error) {
+	unlock, err := db.lockAddr(item.Address)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	batch := new(leveldb.Batch)
+
+	// update accessTimeStamp in retrieve, gc
+
+	i, err := db.retrievalAccessIndex.Get(item)
+	switch err {
+	case nil:
+		item.AccessTimestamp = i.AccessTimestamp
+	case leveldb.ErrNotFound:
+		// no chunk accesses
+	default:
+		return err
+	}
+	if item.AccessTimestamp == 0 {
+		// chunk is not yet synced
+		// do not add it to the gc index
+		return nil
+	}
+	// delete current entry from the gc index
+	db.gcIndex.DeleteInBatch(batch, item)
+	// update access timestamp
+	item.AccessTimestamp = now()
+	// update retrieve access index
+	db.retrievalAccessIndex.PutInBatch(batch, item)
+	// add new entry to gc index
+	db.gcIndex.PutInBatch(batch, item)
+
+	return db.shed.WriteBatch(batch)
+}
+
+// testHookUpdateGC is a hook that can provide
+// information when a garbage collection index is updated.
+var testHookUpdateGC func()

swarm/storage/localstore/mode_get_test.go

@@ -0,0 +1,237 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"testing"
+	"time"
+)
+
+// TestModeGetRequest validates ModeGetRequest index values on the provided DB.
+func TestModeGetRequest(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploadTimestamp := time.Now().UTC().UnixNano()
+	defer setNow(func() (t int64) {
+		return uploadTimestamp
+	})()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requester := db.NewGetter(ModeGetRequest)
+
+	// set update gc test hook to signal when
+	// update gc goroutine is done by sending to
+	// testHookUpdateGCChan channel, which is
+	// used to wait for garbage colletion index
+	// changes
+	testHookUpdateGCChan := make(chan struct{})
+	defer setTestHookUpdateGC(func() {
+		testHookUpdateGCChan <- struct{}{}
+	})()
+
+	t.Run("get unsynced", func(t *testing.T) {
+		got, err := requester.Get(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+		// wait for update gc goroutine to be done
+		<-testHookUpdateGCChan
+
+		if !bytes.Equal(got.Address(), chunk.Address()) {
+			t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address())
+		}
+
+		if !bytes.Equal(got.Data(), chunk.Data()) {
+			t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data())
+		}
+
+		t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, 0))
+
+		t.Run("gc index count", newItemsCountTest(db.gcIndex, 0))
+
+		t.Run("gc size", newIndexGCSizeTest(db))
+	})
+
+	// set chunk to synced state
+	err = db.NewSetter(ModeSetSync).Set(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("first get", func(t *testing.T) {
+		got, err := requester.Get(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+		// wait for update gc goroutine to be done
+		<-testHookUpdateGCChan
+
+		if !bytes.Equal(got.Address(), chunk.Address()) {
+			t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address())
+		}
+
+		if !bytes.Equal(got.Data(), chunk.Data()) {
+			t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data())
+		}
+
+		t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, uploadTimestamp))
+
+		t.Run("gc index", newGCIndexTest(db, chunk, uploadTimestamp, uploadTimestamp))
+
+		t.Run("gc index count", newItemsCountTest(db.gcIndex, 1))
+
+		t.Run("gc size", newIndexGCSizeTest(db))
+	})
+
+	t.Run("second get", func(t *testing.T) {
+		accessTimestamp := time.Now().UTC().UnixNano()
+		defer setNow(func() (t int64) {
+			return accessTimestamp
+		})()
+
+		got, err := requester.Get(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+		// wait for update gc goroutine to be done
+		<-testHookUpdateGCChan
+
+		if !bytes.Equal(got.Address(), chunk.Address()) {
+			t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address())
+		}
+
+		if !bytes.Equal(got.Data(), chunk.Data()) {
+			t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data())
+		}
+
+		t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, accessTimestamp))
+
+		t.Run("gc index", newGCIndexTest(db, chunk, uploadTimestamp, accessTimestamp))
+
+		t.Run("gc index count", newItemsCountTest(db.gcIndex, 1))
+
+		t.Run("gc size", newIndexGCSizeTest(db))
+	})
+}
+
+// TestModeGetSync validates ModeGetSync index values on the provided DB.
+func TestModeGetSync(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploadTimestamp := time.Now().UTC().UnixNano()
+	defer setNow(func() (t int64) {
+		return uploadTimestamp
+	})()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := db.NewGetter(ModeGetSync).Get(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(got.Address(), chunk.Address()) {
+		t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address())
+	}
+
+	if !bytes.Equal(got.Data(), chunk.Data()) {
+		t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data())
+	}
+
+	t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, 0))
+
+	t.Run("gc index count", newItemsCountTest(db.gcIndex, 0))
+
+	t.Run("gc size", newIndexGCSizeTest(db))
+}
+
+// setTestHookUpdateGC sets testHookUpdateGC and
+// returns a function that will reset it to the
+// value before the change.
+func setTestHookUpdateGC(h func()) (reset func()) {
+	current := testHookUpdateGC
+	reset = func() { testHookUpdateGC = current }
+	testHookUpdateGC = h
+	return reset
+}
+
+// TestSetTestHookUpdateGC tests if setTestHookUpdateGC changes
+// testHookUpdateGC function correctly and if its reset function
+// resets the original function.
+func TestSetTestHookUpdateGC(t *testing.T) {
+	// Set the current function after the test finishes.
+	defer func(h func()) { testHookUpdateGC = h }(testHookUpdateGC)
+
+	// expected value for the unchanged function
+	original := 1
+	// expected value for the changed function
+	changed := 2
+
+	// this variable will be set with two different functions
+	var got int
+
+	// define the original (unchanged) functions
+	testHookUpdateGC = func() {
+		got = original
+	}
+
+	// set got variable
+	testHookUpdateGC()
+
+	// test if got variable is set correctly
+	if got != original {
+		t.Errorf("got hook value %v, want %v", got, original)
+	}
+
+	// set the new function
+	reset := setTestHookUpdateGC(func() {
+		got = changed
+	})
+
+	// set got variable
+	testHookUpdateGC()
+
+	// test if got variable is set correctly to changed value
+	if got != changed {
+		t.Errorf("got hook value %v, want %v", got, changed)
+	}
+
+	// set the function to the original one
+	reset()
+
+	// set got variable
+	testHookUpdateGC()
+
+	// test if got variable is set correctly to original value
+	if got != original {
+		t.Errorf("got hook value %v, want %v", got, original)
+	}
+}

swarm/storage/localstore/mode_put.go

@@ -0,0 +1,160 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// ModePut enumerates different Putter modes.
+type ModePut int
+
+// Putter modes.
+const (
+	// ModePutRequest: when a chunk is received as a result of retrieve request and delivery
+	ModePutRequest ModePut = iota
+	// ModePutSync: when a chunk is received via syncing
+	ModePutSync
+	// ModePutUpload: when a chunk is created by local upload
+	ModePutUpload
+)
+
+// Putter provides Put method to store Chunks
+// to database.
+type Putter struct {
+	db   *DB
+	mode ModePut
+}
+
+// NewPutter returns a new Putter on database
+// with a specific Mode.
+func (db *DB) NewPutter(mode ModePut) *Putter {
+	return &Putter{
+		mode: mode,
+		db:   db,
+	}
+}
+
+// Put stores the Chunk to database and depending
+// on the Putter mode, it updates required indexes.
+func (p *Putter) Put(ch storage.Chunk) (err error) {
+	return p.db.put(p.mode, chunkToItem(ch))
+}
+
+// put stores Item to database and updates other
+// indexes. It acquires lockAddr to protect two calls
+// of this function for the same address in parallel.
+// Item fields Address and Data must not be
+// with their nil values.
+func (db *DB) put(mode ModePut, item shed.Item) (err error) {
+	// protect parallel updates
+	unlock, err := db.lockAddr(item.Address)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	batch := new(leveldb.Batch)
+
+	// variables that provide information for operations
+	// to be done after write batch function successfully executes
+	var gcSizeChange int64   // number to add or subtract from gcSize
+	var triggerPullFeed bool // signal pull feed subscriptions to iterate
+	var triggerPushFeed bool // signal push feed subscriptions to iterate
+
+	switch mode {
+	case ModePutRequest:
+		// put to indexes: retrieve, gc; it does not enter the syncpool
+
+		// check if the chunk already is in the database
+		// as gc index is updated
+		i, err := db.retrievalAccessIndex.Get(item)
+		switch err {
+		case nil:
+			item.AccessTimestamp = i.AccessTimestamp
+		case leveldb.ErrNotFound:
+			// no chunk accesses
+		default:
+			return err
+		}
+		i, err = db.retrievalDataIndex.Get(item)
+		switch err {
+		case nil:
+			item.StoreTimestamp = i.StoreTimestamp
+		case leveldb.ErrNotFound:
+			// no chunk accesses
+		default:
+			return err
+		}
+		if item.AccessTimestamp != 0 {
+			// delete current entry from the gc index
+			db.gcIndex.DeleteInBatch(batch, item)
+			gcSizeChange--
+		}
+		if item.StoreTimestamp == 0 {
+			item.StoreTimestamp = now()
+		}
+		// update access timestamp
+		item.AccessTimestamp = now()
+		// update retrieve access index
+		db.retrievalAccessIndex.PutInBatch(batch, item)
+		// add new entry to gc index
+		db.gcIndex.PutInBatch(batch, item)
+		db.gcUncountedHashesIndex.PutInBatch(batch, item)
+		gcSizeChange++
+
+		db.retrievalDataIndex.PutInBatch(batch, item)
+
+	case ModePutUpload:
+		// put to indexes: retrieve, push, pull
+
+		item.StoreTimestamp = now()
+		db.retrievalDataIndex.PutInBatch(batch, item)
+		db.pullIndex.PutInBatch(batch, item)
+		triggerPullFeed = true
+		db.pushIndex.PutInBatch(batch, item)
+		triggerPushFeed = true
+
+	case ModePutSync:
+		// put to indexes: retrieve, pull
+
+		item.StoreTimestamp = now()
+		db.retrievalDataIndex.PutInBatch(batch, item)
+		db.pullIndex.PutInBatch(batch, item)
+		triggerPullFeed = true
+
+	default:
+		return ErrInvalidMode
+	}
+
+	err = db.shed.WriteBatch(batch)
+	if err != nil {
+		return err
+	}
+	if gcSizeChange != 0 {
+		db.incGCSize(gcSizeChange)
+	}
+	if triggerPullFeed {
+		db.triggerPullSubscriptions(db.po(item.Address))
+	}
+	if triggerPushFeed {
+		db.triggerPushSubscriptions()
+	}
+	return nil
+}

swarm/storage/localstore/mode_put_test.go

@@ -0,0 +1,300 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// TestModePutRequest validates ModePutRequest index values on the provided DB.
+func TestModePutRequest(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	putter := db.NewPutter(ModePutRequest)
+
+	chunk := generateRandomChunk()
+
+	// keep the record when the chunk is stored
+	var storeTimestamp int64
+
+	t.Run("first put", func(t *testing.T) {
+		wantTimestamp := time.Now().UTC().UnixNano()
+		defer setNow(func() (t int64) {
+			return wantTimestamp
+		})()
+
+		storeTimestamp = wantTimestamp
+
+		err := putter.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, wantTimestamp, wantTimestamp))
+
+		t.Run("gc index count", newItemsCountTest(db.gcIndex, 1))
+
+		t.Run("gc size", newIndexGCSizeTest(db))
+	})
+
+	t.Run("second put", func(t *testing.T) {
+		wantTimestamp := time.Now().UTC().UnixNano()
+		defer setNow(func() (t int64) {
+			return wantTimestamp
+		})()
+
+		err := putter.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, storeTimestamp, wantTimestamp))
+
+		t.Run("gc index count", newItemsCountTest(db.gcIndex, 1))
+
+		t.Run("gc size", newIndexGCSizeTest(db))
+	})
+}
+
+// TestModePutSync validates ModePutSync index values on the provided DB.
+func TestModePutSync(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	wantTimestamp := time.Now().UTC().UnixNano()
+	defer setNow(func() (t int64) {
+		return wantTimestamp
+	})()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutSync).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("retrieve indexes", newRetrieveIndexesTest(db, chunk, wantTimestamp, 0))
+
+	t.Run("pull index", newPullIndexTest(db, chunk, wantTimestamp, nil))
+}
+
+// TestModePutUpload validates ModePutUpload index values on the provided DB.
+func TestModePutUpload(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	wantTimestamp := time.Now().UTC().UnixNano()
+	defer setNow(func() (t int64) {
+		return wantTimestamp
+	})()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("retrieve indexes", newRetrieveIndexesTest(db, chunk, wantTimestamp, 0))
+
+	t.Run("pull index", newPullIndexTest(db, chunk, wantTimestamp, nil))
+
+	t.Run("push index", newPushIndexTest(db, chunk, wantTimestamp, nil))
+}
+
+// TestModePutUpload_parallel uploads chunks in parallel
+// and validates if all chunks can be retrieved with correct data.
+func TestModePutUpload_parallel(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	chunkCount := 1000
+	workerCount := 100
+
+	chunkChan := make(chan storage.Chunk)
+	errChan := make(chan error)
+	doneChan := make(chan struct{})
+	defer close(doneChan)
+
+	// start uploader workers
+	for i := 0; i < workerCount; i++ {
+		go func(i int) {
+			uploader := db.NewPutter(ModePutUpload)
+			for {
+				select {
+				case chunk, ok := <-chunkChan:
+					if !ok {
+						return
+					}
+					err := uploader.Put(chunk)
+					select {
+					case errChan <- err:
+					case <-doneChan:
+					}
+				case <-doneChan:
+					return
+				}
+			}
+		}(i)
+	}
+
+	chunks := make([]storage.Chunk, 0)
+	var chunksMu sync.Mutex
+
+	// send chunks to workers
+	go func() {
+		for i := 0; i < chunkCount; i++ {
+			chunk := generateRandomChunk()
+			select {
+			case chunkChan <- chunk:
+			case <-doneChan:
+				return
+			}
+			chunksMu.Lock()
+			chunks = append(chunks, chunk)
+			chunksMu.Unlock()
+		}
+
+		close(chunkChan)
+	}()
+
+	// validate every error from workers
+	for i := 0; i < chunkCount; i++ {
+		err := <-errChan
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// get every chunk and validate its data
+	getter := db.NewGetter(ModeGetRequest)
+
+	chunksMu.Lock()
+	defer chunksMu.Unlock()
+	for _, chunk := range chunks {
+		got, err := getter.Get(chunk.Address())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(got.Data(), chunk.Data()) {
+			t.Fatalf("got chunk %s data %x, want %x", chunk.Address().Hex(), got.Data(), chunk.Data())
+		}
+	}
+}
+
+// BenchmarkPutUpload runs a series of benchmarks that upload
+// a specific number of chunks in parallel.
+//
+// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014)
+//
+// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkPutUpload -v
+//
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore
+// BenchmarkPutUpload/count_100_parallel_1-8         	     300	   5107704 ns/op	 2081461 B/op	    2374 allocs/op
+// BenchmarkPutUpload/count_100_parallel_2-8         	     300	   5411742 ns/op	 2081608 B/op	    2364 allocs/op
+// BenchmarkPutUpload/count_100_parallel_4-8         	     500	   3704964 ns/op	 2081696 B/op	    2324 allocs/op
+// BenchmarkPutUpload/count_100_parallel_8-8         	     500	   2932663 ns/op	 2082594 B/op	    2295 allocs/op
+// BenchmarkPutUpload/count_100_parallel_16-8        	     500	   3117157 ns/op	 2085438 B/op	    2282 allocs/op
+// BenchmarkPutUpload/count_100_parallel_32-8        	     500	   3449122 ns/op	 2089721 B/op	    2286 allocs/op
+// BenchmarkPutUpload/count_1000_parallel_1-8        	      20	  79784470 ns/op	25211240 B/op	   23225 allocs/op
+// BenchmarkPutUpload/count_1000_parallel_2-8        	      20	  75422164 ns/op	25210730 B/op	   23187 allocs/op
+// BenchmarkPutUpload/count_1000_parallel_4-8        	      20	  70698378 ns/op	25206522 B/op	   22692 allocs/op
+// BenchmarkPutUpload/count_1000_parallel_8-8        	      20	  71285528 ns/op	25213436 B/op	   22345 allocs/op
+// BenchmarkPutUpload/count_1000_parallel_16-8       	      20	  71301826 ns/op	25205040 B/op	   22090 allocs/op
+// BenchmarkPutUpload/count_1000_parallel_32-8       	      30	  57713506 ns/op	25219781 B/op	   21848 allocs/op
+// BenchmarkPutUpload/count_10000_parallel_1-8       	       2	 656719345 ns/op	216792908 B/op	  248940 allocs/op
+// BenchmarkPutUpload/count_10000_parallel_2-8       	       2	 646301962 ns/op	216730800 B/op	  248270 allocs/op
+// BenchmarkPutUpload/count_10000_parallel_4-8       	       2	 532784228 ns/op	216667080 B/op	  241910 allocs/op
+// BenchmarkPutUpload/count_10000_parallel_8-8       	       3	 494290188 ns/op	216297749 B/op	  236247 allocs/op
+// BenchmarkPutUpload/count_10000_parallel_16-8      	       3	 483485315 ns/op	216060384 B/op	  231090 allocs/op
+// BenchmarkPutUpload/count_10000_parallel_32-8      	       3	 434461294 ns/op	215371280 B/op	  224800 allocs/op
+// BenchmarkPutUpload/count_100000_parallel_1-8      	       1	22767894338 ns/op	2331372088 B/op	 4049876 allocs/op
+// BenchmarkPutUpload/count_100000_parallel_2-8      	       1	25347872677 ns/op	2344140160 B/op	 4106763 allocs/op
+// BenchmarkPutUpload/count_100000_parallel_4-8      	       1	23580460174 ns/op	2338582576 B/op	 4027452 allocs/op
+// BenchmarkPutUpload/count_100000_parallel_8-8      	       1	22197559193 ns/op	2321803496 B/op	 3877553 allocs/op
+// BenchmarkPutUpload/count_100000_parallel_16-8     	       1	22527046476 ns/op	2327854800 B/op	 3885455 allocs/op
+// BenchmarkPutUpload/count_100000_parallel_32-8     	       1	21332243613 ns/op	2299654568 B/op	 3697181 allocs/op
+// PASS
+func BenchmarkPutUpload(b *testing.B) {
+	for _, count := range []int{
+		100,
+		1000,
+		10000,
+		100000,
+	} {
+		for _, maxParallelUploads := range []int{
+			1,
+			2,
+			4,
+			8,
+			16,
+			32,
+		} {
+			name := fmt.Sprintf("count %v parallel %v", count, maxParallelUploads)
+			b.Run(name, func(b *testing.B) {
+				for n := 0; n < b.N; n++ {
+					benchmarkPutUpload(b, nil, count, maxParallelUploads)
+				}
+			})
+		}
+	}
+}
+
+// benchmarkPutUpload runs a benchmark by uploading a specific number
+// of chunks with specified max parallel uploads.
+func benchmarkPutUpload(b *testing.B, o *Options, count, maxParallelUploads int) {
+	b.StopTimer()
+	db, cleanupFunc := newTestDB(b, o)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+	chunks := make([]storage.Chunk, count)
+	for i := 0; i < count; i++ {
+		chunks[i] = generateFakeRandomChunk()
+	}
+	errs := make(chan error)
+	b.StartTimer()
+
+	go func() {
+		sem := make(chan struct{}, maxParallelUploads)
+		for i := 0; i < count; i++ {
+			sem <- struct{}{}
+
+			go func(i int) {
+				defer func() { <-sem }()
+
+				errs <- uploader.Put(chunks[i])
+			}(i)
+		}
+	}()
+
+	for i := 0; i < count; i++ {
+		err := <-errs
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

swarm/storage/localstore/mode_set.go

@@ -0,0 +1,205 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// ModeSet enumerates different Setter modes.
+type ModeSet int
+
+// Setter modes.
+const (
+	// ModeSetAccess: when an update request is received for a chunk or chunk is retrieved for delivery
+	ModeSetAccess ModeSet = iota
+	// ModeSetSync: when push sync receipt is received
+	ModeSetSync
+	// modeSetRemove: when GC-d
+	// unexported as no external packages should remove chunks from database
+	modeSetRemove
+)
+
+// Setter sets the state of a particular
+// Chunk in database by changing indexes.
+type Setter struct {
+	db   *DB
+	mode ModeSet
+}
+
+// NewSetter returns a new Setter on database
+// with a specific Mode.
+func (db *DB) NewSetter(mode ModeSet) *Setter {
+	return &Setter{
+		mode: mode,
+		db:   db,
+	}
+}
+
+// Set updates database indexes for a specific
+// chunk represented by the address.
+func (s *Setter) Set(addr storage.Address) (err error) {
+	return s.db.set(s.mode, addr)
+}
+
+// set updates database indexes for a specific
+// chunk represented by the address.
+// It acquires lockAddr to protect two calls
+// of this function for the same address in parallel.
+func (db *DB) set(mode ModeSet, addr storage.Address) (err error) {
+	// protect parallel updates
+	unlock, err := db.lockAddr(addr)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	batch := new(leveldb.Batch)
+
+	// variables that provide information for operations
+	// to be done after write batch function successfully executes
+	var gcSizeChange int64   // number to add or subtract from gcSize
+	var triggerPullFeed bool // signal pull feed subscriptions to iterate
+
+	item := addressToItem(addr)
+
+	switch mode {
+	case ModeSetAccess:
+		// add to pull, insert to gc
+
+		// need to get access timestamp here as it is not
+		// provided by the access function, and it is not
+		// a property of a chunk provided to Accessor.Put.
+
+		i, err := db.retrievalDataIndex.Get(item)
+		switch err {
+		case nil:
+			item.StoreTimestamp = i.StoreTimestamp
+		case leveldb.ErrNotFound:
+			db.pushIndex.DeleteInBatch(batch, item)
+			item.StoreTimestamp = now()
+		default:
+			return err
+		}
+
+		i, err = db.retrievalAccessIndex.Get(item)
+		switch err {
+		case nil:
+			item.AccessTimestamp = i.AccessTimestamp
+			db.gcIndex.DeleteInBatch(batch, item)
+			gcSizeChange--
+		case leveldb.ErrNotFound:
+			// the chunk is not accessed before
+		default:
+			return err
+		}
+		item.AccessTimestamp = now()
+		db.retrievalAccessIndex.PutInBatch(batch, item)
+		db.pullIndex.PutInBatch(batch, item)
+		triggerPullFeed = true
+		db.gcIndex.PutInBatch(batch, item)
+		db.gcUncountedHashesIndex.PutInBatch(batch, item)
+		gcSizeChange++
+
+	case ModeSetSync:
+		// delete from push, insert to gc
+
+		// need to get access timestamp here as it is not
+		// provided by the access function, and it is not
+		// a property of a chunk provided to Accessor.Put.
+		i, err := db.retrievalDataIndex.Get(item)
+		if err != nil {
+			if err == leveldb.ErrNotFound {
+				// chunk is not found,
+				// no need to update gc index
+				// just delete from the push index
+				// if it is there
+				db.pushIndex.DeleteInBatch(batch, item)
+				return nil
+			}
+			return err
+		}
+		item.StoreTimestamp = i.StoreTimestamp
+
+		i, err = db.retrievalAccessIndex.Get(item)
+		switch err {
+		case nil:
+			item.AccessTimestamp = i.AccessTimestamp
+			db.gcIndex.DeleteInBatch(batch, item)
+			gcSizeChange--
+		case leveldb.ErrNotFound:
+			// the chunk is not accessed before
+		default:
+			return err
+		}
+		item.AccessTimestamp = now()
+		db.retrievalAccessIndex.PutInBatch(batch, item)
+		db.pushIndex.DeleteInBatch(batch, item)
+		db.gcIndex.PutInBatch(batch, item)
+		db.gcUncountedHashesIndex.PutInBatch(batch, item)
+		gcSizeChange++
+
+	case modeSetRemove:
+		// delete from retrieve, pull, gc
+
+		// need to get access timestamp here as it is not
+		// provided by the access function, and it is not
+		// a property of a chunk provided to Accessor.Put.
+
+		i, err := db.retrievalAccessIndex.Get(item)
+		switch err {
+		case nil:
+			item.AccessTimestamp = i.AccessTimestamp
+		case leveldb.ErrNotFound:
+		default:
+			return err
+		}
+		i, err = db.retrievalDataIndex.Get(item)
+		if err != nil {
+			return err
+		}
+		item.StoreTimestamp = i.StoreTimestamp
+
+		db.retrievalDataIndex.DeleteInBatch(batch, item)
+		db.retrievalAccessIndex.DeleteInBatch(batch, item)
+		db.pullIndex.DeleteInBatch(batch, item)
+		db.gcIndex.DeleteInBatch(batch, item)
+		db.gcUncountedHashesIndex.DeleteInBatch(batch, item)
+		// a check is needed for decrementing gcSize
+		// as delete is not reporting if the key/value pair
+		// is deleted or not
+		if _, err := db.gcIndex.Get(item); err == nil {
+			gcSizeChange = -1
+		}
+
+	default:
+		return ErrInvalidMode
+	}
+
+	err = db.shed.WriteBatch(batch)
+	if err != nil {
+		return err
+	}
+	if gcSizeChange != 0 {
+		db.incGCSize(gcSizeChange)
+	}
+	if triggerPullFeed {
+		db.triggerPullSubscriptions(db.po(item.Address))
+	}
+	return nil
+}

swarm/storage/localstore/mode_set_test.go

@@ -0,0 +1,128 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// TestModeSetAccess validates ModeSetAccess index values on the provided DB.
+func TestModeSetAccess(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	chunk := generateRandomChunk()
+
+	wantTimestamp := time.Now().UTC().UnixNano()
+	defer setNow(func() (t int64) {
+		return wantTimestamp
+	})()
+
+	err := db.NewSetter(ModeSetAccess).Set(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("pull index", newPullIndexTest(db, chunk, wantTimestamp, nil))
+
+	t.Run("pull index count", newItemsCountTest(db.pullIndex, 1))
+
+	t.Run("gc index", newGCIndexTest(db, chunk, wantTimestamp, wantTimestamp))
+
+	t.Run("gc index count", newItemsCountTest(db.gcIndex, 1))
+
+	t.Run("gc size", newIndexGCSizeTest(db))
+}
+
+// TestModeSetSync validates ModeSetSync index values on the provided DB.
+func TestModeSetSync(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	chunk := generateRandomChunk()
+
+	wantTimestamp := time.Now().UTC().UnixNano()
+	defer setNow(func() (t int64) {
+		return wantTimestamp
+	})()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.NewSetter(ModeSetSync).Set(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, wantTimestamp, wantTimestamp))
+
+	t.Run("push index", newPushIndexTest(db, chunk, wantTimestamp, leveldb.ErrNotFound))
+
+	t.Run("gc index", newGCIndexTest(db, chunk, wantTimestamp, wantTimestamp))
+
+	t.Run("gc index count", newItemsCountTest(db.gcIndex, 1))
+
+	t.Run("gc size", newIndexGCSizeTest(db))
+}
+
+// TestModeSetRemove validates ModeSetRemove index values on the provided DB.
+func TestModeSetRemove(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	chunk := generateRandomChunk()
+
+	err := db.NewPutter(ModePutUpload).Put(chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.NewSetter(modeSetRemove).Set(chunk.Address())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("retrieve indexes", func(t *testing.T) {
+		wantErr := leveldb.ErrNotFound
+		_, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
+		if err != wantErr {
+			t.Errorf("got error %v, want %v", err, wantErr)
+		}
+		t.Run("retrieve data index count", newItemsCountTest(db.retrievalDataIndex, 0))
+
+		// access index should not be set
+		_, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address()))
+		if err != wantErr {
+			t.Errorf("got error %v, want %v", err, wantErr)
+		}
+		t.Run("retrieve access index count", newItemsCountTest(db.retrievalAccessIndex, 0))
+	})
+
+	t.Run("pull index", newPullIndexTest(db, chunk, 0, leveldb.ErrNotFound))
+
+	t.Run("pull index count", newItemsCountTest(db.pullIndex, 0))
+
+	t.Run("gc index count", newItemsCountTest(db.gcIndex, 0))
+
+	t.Run("gc size", newIndexGCSizeTest(db))
+
+}

swarm/storage/localstore/retrieval_index_test.go

@@ -0,0 +1,150 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// BenchmarkRetrievalIndexes uploads a number of chunks in order to measure
+// total time of updating their retrieval indexes by setting them
+// to synced state and requesting them.
+//
+// This benchmark takes significant amount of time.
+//
+// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014) show
+// that two separated indexes perform better.
+//
+// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkRetrievalIndexes -v
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore
+// BenchmarkRetrievalIndexes/1000-8         	      20       75556686 ns/op      19033493 B/op       84500 allocs/op
+// BenchmarkRetrievalIndexes/10000-8        	       1     1079084922 ns/op     382792064 B/op     1429644 allocs/op
+// BenchmarkRetrievalIndexes/100000-8       	       1    16891305737 ns/op    2629165304 B/op    12465019 allocs/op
+// PASS
+func BenchmarkRetrievalIndexes(b *testing.B) {
+	for _, count := range []int{
+		1000,
+		10000,
+		100000,
+	} {
+		b.Run(strconv.Itoa(count)+"-split", func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				benchmarkRetrievalIndexes(b, nil, count)
+			}
+		})
+	}
+}
+
+// benchmarkRetrievalIndexes is used in BenchmarkRetrievalIndexes
+// to do benchmarks with a specific number of chunks and different
+// database options.
+func benchmarkRetrievalIndexes(b *testing.B, o *Options, count int) {
+	b.StopTimer()
+	db, cleanupFunc := newTestDB(b, o)
+	defer cleanupFunc()
+	uploader := db.NewPutter(ModePutUpload)
+	syncer := db.NewSetter(ModeSetSync)
+	requester := db.NewGetter(ModeGetRequest)
+	addrs := make([]storage.Address, count)
+	for i := 0; i < count; i++ {
+		chunk := generateFakeRandomChunk()
+		err := uploader.Put(chunk)
+		if err != nil {
+			b.Fatal(err)
+		}
+		addrs[i] = chunk.Address()
+	}
+	// set update gc test hook to signal when
+	// update gc goroutine is done by sending to
+	// testHookUpdateGCChan channel, which is
+	// used to wait for gc index updates to be
+	// included in the benchmark time
+	testHookUpdateGCChan := make(chan struct{})
+	defer setTestHookUpdateGC(func() {
+		testHookUpdateGCChan <- struct{}{}
+	})()
+	b.StartTimer()
+
+	for i := 0; i < count; i++ {
+		err := syncer.Set(addrs[i])
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		_, err = requester.Get(addrs[i])
+		if err != nil {
+			b.Fatal(err)
+		}
+		// wait for update gc goroutine to be done
+		<-testHookUpdateGCChan
+	}
+}
+
+// BenchmarkUpload compares uploading speed for different
+// retrieval indexes and various number of chunks.
+//
+// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014).
+//
+// go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkUpload -v
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore
+// BenchmarkUpload/1000-8         	      20       59437463 ns/op     25205193 B/op    23208 allocs/op
+// BenchmarkUpload/10000-8        	       2      580646362 ns/op    216532932 B/op	  248090 allocs/op
+// BenchmarkUpload/100000-8       	       1    22373390892 ns/op   2323055312 B/op	 3995903 allocs/op
+// PASS
+func BenchmarkUpload(b *testing.B) {
+	for _, count := range []int{
+		1000,
+		10000,
+		100000,
+	} {
+		b.Run(strconv.Itoa(count), func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				benchmarkUpload(b, nil, count)
+			}
+		})
+	}
+}
+
+// benchmarkUpload is used in BenchmarkUpload
+// to do benchmarks with a specific number of chunks and different
+// database options.
+func benchmarkUpload(b *testing.B, o *Options, count int) {
+	b.StopTimer()
+	db, cleanupFunc := newTestDB(b, o)
+	defer cleanupFunc()
+	uploader := db.NewPutter(ModePutUpload)
+	chunks := make([]storage.Chunk, count)
+	for i := 0; i < count; i++ {
+		chunk := generateFakeRandomChunk()
+		chunks[i] = chunk
+	}
+	b.StartTimer()
+
+	for i := 0; i < count; i++ {
+		err := uploader.Put(chunks[i])
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

swarm/storage/localstore/subscription_pull.go

@@ -0,0 +1,193 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// SubscribePull returns a channel that provides chunk addresses and stored times from pull syncing index.
+// Pull syncing index can be only subscribed to a particular proximity order bin. If since
+// is not nil, the iteration will start from the first item stored after that timestamp. If until is not nil,
+// only chunks stored up to this timestamp will be send to the channel, and the returned channel will be
+// closed. The since-until interval is open on the left and closed on the right (since,until]. Returned stop
+// function will terminate current and further iterations without errors, and also close the returned channel.
+// Make sure that you check the second returned parameter from the channel to stop iteration when its value
+// is false.
+func (db *DB) SubscribePull(ctx context.Context, bin uint8, since, until *ChunkDescriptor) (c <-chan ChunkDescriptor, stop func()) {
+	chunkDescriptors := make(chan ChunkDescriptor)
+	trigger := make(chan struct{}, 1)
+
+	db.pullTriggersMu.Lock()
+	if _, ok := db.pullTriggers[bin]; !ok {
+		db.pullTriggers[bin] = make([]chan struct{}, 0)
+	}
+	db.pullTriggers[bin] = append(db.pullTriggers[bin], trigger)
+	db.pullTriggersMu.Unlock()
+
+	// send signal for the initial iteration
+	trigger <- struct{}{}
+
+	stopChan := make(chan struct{})
+	var stopChanOnce sync.Once
+
+	// used to provide information from the iterator to
+	// stop subscription when until chunk descriptor is reached
+	var errStopSubscription = errors.New("stop subscription")
+
+	go func() {
+		// close the returned ChunkDescriptor channel at the end to
+		// signal that the subscription is done
+		defer close(chunkDescriptors)
+		// sinceItem is the Item from which the next iteration
+		// should start. The first iteration starts from the first Item.
+		var sinceItem *shed.Item
+		if since != nil {
+			sinceItem = &shed.Item{
+				Address:        since.Address,
+				StoreTimestamp: since.StoreTimestamp,
+			}
+		}
+		for {
+			select {
+			case <-trigger:
+				// iterate until:
+				// - last index Item is reached
+				// - subscription stop is called
+				// - context is done
+				err := db.pullIndex.Iterate(func(item shed.Item) (stop bool, err error) {
+					select {
+					case chunkDescriptors <- ChunkDescriptor{
+						Address:        item.Address,
+						StoreTimestamp: item.StoreTimestamp,
+					}:
+						// until chunk descriptor is sent
+						// break the iteration
+						if until != nil &&
+							(item.StoreTimestamp >= until.StoreTimestamp ||
+								bytes.Equal(item.Address, until.Address)) {
+							return true, errStopSubscription
+						}
+						// set next iteration start item
+						// when its chunk is successfully sent to channel
+						sinceItem = &item
+						return false, nil
+					case <-stopChan:
+						// gracefully stop the iteration
+						// on stop
+						return true, nil
+					case <-db.close:
+						// gracefully stop the iteration
+						// on database close
+						return true, nil
+					case <-ctx.Done():
+						return true, ctx.Err()
+					}
+				}, &shed.IterateOptions{
+					StartFrom: sinceItem,
+					// sinceItem was sent as the last Address in the previous
+					// iterator call, skip it in this one
+					SkipStartFromItem: true,
+					Prefix:            []byte{bin},
+				})
+				if err != nil {
+					if err == errStopSubscription {
+						// stop subscription without any errors
+						// if until is reached
+						return
+					}
+					log.Error("localstore pull subscription iteration", "bin", bin, "since", since, "until", until, "err", err)
+					return
+				}
+			case <-stopChan:
+				// terminate the subscription
+				// on stop
+				return
+			case <-db.close:
+				// terminate the subscription
+				// on database close
+				return
+			case <-ctx.Done():
+				err := ctx.Err()
+				if err != nil {
+					log.Error("localstore pull subscription", "bin", bin, "since", since, "until", until, "err", err)
+				}
+				return
+			}
+		}
+	}()
+
+	stop = func() {
+		stopChanOnce.Do(func() {
+			close(stopChan)
+		})
+
+		db.pullTriggersMu.Lock()
+		defer db.pullTriggersMu.Unlock()
+
+		for i, t := range db.pullTriggers[bin] {
+			if t == trigger {
+				db.pullTriggers[bin] = append(db.pullTriggers[bin][:i], db.pullTriggers[bin][i+1:]...)
+				break
+			}
+		}
+	}
+
+	return chunkDescriptors, stop
+}
+
+// ChunkDescriptor holds information required for Pull syncing. This struct
+// is provided by subscribing to pull index.
+type ChunkDescriptor struct {
+	Address        storage.Address
+	StoreTimestamp int64
+}
+
+func (c *ChunkDescriptor) String() string {
+	if c == nil {
+		return "none"
+	}
+	return fmt.Sprintf("%s stored at %v", c.Address.Hex(), c.StoreTimestamp)
+}
+
+// triggerPullSubscriptions is used internally for starting iterations
+// on Pull subscriptions for a particular bin. When new item with address
+// that is in particular bin for DB's baseKey is added to pull index
+// this function should be called.
+func (db *DB) triggerPullSubscriptions(bin uint8) {
+	db.pullTriggersMu.RLock()
+	triggers, ok := db.pullTriggers[bin]
+	db.pullTriggersMu.RUnlock()
+	if !ok {
+		return
+	}
+
+	for _, t := range triggers {
+		select {
+		case t <- struct{}{}:
+		default:
+		}
+	}
+}

swarm/storage/localstore/subscription_pull_test.go

@@ -0,0 +1,478 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// TestDB_SubscribePull uploads some chunks before and after
+// pull syncing subscription is created and validates if
+// all addresses are received in the right order
+// for expected proximity order bins.
+func TestDB_SubscribePull(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	addrs := make(map[uint8][]storage.Address)
+	var addrsMu sync.Mutex
+	var wantedChunksCount int
+
+	// prepopulate database with some chunks
+	// before the subscription
+	uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 10)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
+		ch, stop := db.SubscribePull(ctx, bin, nil, nil)
+		defer stop()
+
+		// receive and validate addresses from the subscription
+		go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan)
+	}
+
+	// upload some chunks just after subscribe
+	uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 5)
+
+	time.Sleep(200 * time.Millisecond)
+
+	// upload some chunks after some short time
+	// to ensure that subscription will include them
+	// in a dynamic environment
+	uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 3)
+
+	checkErrChan(ctx, t, errChan, wantedChunksCount)
+}
+
+// TestDB_SubscribePull_multiple uploads chunks before and after
+// multiple pull syncing subscriptions are created and
+// validates if all addresses are received in the right order
+// for expected proximity order bins.
+func TestDB_SubscribePull_multiple(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	addrs := make(map[uint8][]storage.Address)
+	var addrsMu sync.Mutex
+	var wantedChunksCount int
+
+	// prepopulate database with some chunks
+	// before the subscription
+	uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 10)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	subsCount := 10
+
+	// start a number of subscriptions
+	// that all of them will write every address error to errChan
+	for j := 0; j < subsCount; j++ {
+		for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
+			ch, stop := db.SubscribePull(ctx, bin, nil, nil)
+			defer stop()
+
+			// receive and validate addresses from the subscription
+			go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan)
+		}
+	}
+
+	// upload some chunks just after subscribe
+	uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 5)
+
+	time.Sleep(200 * time.Millisecond)
+
+	// upload some chunks after some short time
+	// to ensure that subscription will include them
+	// in a dynamic environment
+	uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 3)
+
+	checkErrChan(ctx, t, errChan, wantedChunksCount*subsCount)
+}
+
+// TestDB_SubscribePull_since uploads chunks before and after
+// pull syncing subscriptions are created with a since argument
+// and validates if all expected addresses are received in the
+// right order for expected proximity order bins.
+func TestDB_SubscribePull_since(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	addrs := make(map[uint8][]storage.Address)
+	var addrsMu sync.Mutex
+	var wantedChunksCount int
+
+	lastTimestamp := time.Now().UTC().UnixNano()
+	var lastTimestampMu sync.RWMutex
+	defer setNow(func() (t int64) {
+		lastTimestampMu.Lock()
+		defer lastTimestampMu.Unlock()
+		lastTimestamp++
+		return lastTimestamp
+	})()
+
+	uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) {
+		last = make(map[uint8]ChunkDescriptor)
+		for i := 0; i < count; i++ {
+			chunk := generateRandomChunk()
+
+			err := uploader.Put(chunk)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			bin := db.po(chunk.Address())
+
+			addrsMu.Lock()
+			if _, ok := addrs[bin]; !ok {
+				addrs[bin] = make([]storage.Address, 0)
+			}
+			if wanted {
+				addrs[bin] = append(addrs[bin], chunk.Address())
+				wantedChunksCount++
+			}
+			addrsMu.Unlock()
+
+			lastTimestampMu.RLock()
+			storeTimestamp := lastTimestamp
+			lastTimestampMu.RUnlock()
+
+			last[bin] = ChunkDescriptor{
+				Address:        chunk.Address(),
+				StoreTimestamp: storeTimestamp,
+			}
+		}
+		return last
+	}
+
+	// prepopulate database with some chunks
+	// before the subscription
+	last := uploadRandomChunks(30, false)
+
+	uploadRandomChunks(25, true)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
+		var since *ChunkDescriptor
+		if c, ok := last[bin]; ok {
+			since = &c
+		}
+		ch, stop := db.SubscribePull(ctx, bin, since, nil)
+		defer stop()
+
+		// receive and validate addresses from the subscription
+		go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan)
+
+	}
+
+	// upload some chunks just after subscribe
+	uploadRandomChunks(15, true)
+
+	checkErrChan(ctx, t, errChan, wantedChunksCount)
+}
+
+// TestDB_SubscribePull_until uploads chunks before and after
+// pull syncing subscriptions are created with an until argument
+// and validates if all expected addresses are received in the
+// right order for expected proximity order bins.
+func TestDB_SubscribePull_until(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	addrs := make(map[uint8][]storage.Address)
+	var addrsMu sync.Mutex
+	var wantedChunksCount int
+
+	lastTimestamp := time.Now().UTC().UnixNano()
+	var lastTimestampMu sync.RWMutex
+	defer setNow(func() (t int64) {
+		lastTimestampMu.Lock()
+		defer lastTimestampMu.Unlock()
+		lastTimestamp++
+		return lastTimestamp
+	})()
+
+	uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) {
+		last = make(map[uint8]ChunkDescriptor)
+		for i := 0; i < count; i++ {
+			chunk := generateRandomChunk()
+
+			err := uploader.Put(chunk)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			bin := db.po(chunk.Address())
+
+			addrsMu.Lock()
+			if _, ok := addrs[bin]; !ok {
+				addrs[bin] = make([]storage.Address, 0)
+			}
+			if wanted {
+				addrs[bin] = append(addrs[bin], chunk.Address())
+				wantedChunksCount++
+			}
+			addrsMu.Unlock()
+
+			lastTimestampMu.RLock()
+			storeTimestamp := lastTimestamp
+			lastTimestampMu.RUnlock()
+
+			last[bin] = ChunkDescriptor{
+				Address:        chunk.Address(),
+				StoreTimestamp: storeTimestamp,
+			}
+		}
+		return last
+	}
+
+	// prepopulate database with some chunks
+	// before the subscription
+	last := uploadRandomChunks(30, true)
+
+	uploadRandomChunks(25, false)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
+		until, ok := last[bin]
+		if !ok {
+			continue
+		}
+		ch, stop := db.SubscribePull(ctx, bin, nil, &until)
+		defer stop()
+
+		// receive and validate addresses from the subscription
+		go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan)
+	}
+
+	// upload some chunks just after subscribe
+	uploadRandomChunks(15, false)
+
+	checkErrChan(ctx, t, errChan, wantedChunksCount)
+}
+
+// TestDB_SubscribePull_sinceAndUntil uploads chunks before and
+// after pull syncing subscriptions are created with since
+// and until arguments, and validates if all expected addresses
+// are received in the right order for expected proximity order bins.
+func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	addrs := make(map[uint8][]storage.Address)
+	var addrsMu sync.Mutex
+	var wantedChunksCount int
+
+	lastTimestamp := time.Now().UTC().UnixNano()
+	var lastTimestampMu sync.RWMutex
+	defer setNow(func() (t int64) {
+		lastTimestampMu.Lock()
+		defer lastTimestampMu.Unlock()
+		lastTimestamp++
+		return lastTimestamp
+	})()
+
+	uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) {
+		last = make(map[uint8]ChunkDescriptor)
+		for i := 0; i < count; i++ {
+			chunk := generateRandomChunk()
+
+			err := uploader.Put(chunk)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			bin := db.po(chunk.Address())
+
+			addrsMu.Lock()
+			if _, ok := addrs[bin]; !ok {
+				addrs[bin] = make([]storage.Address, 0)
+			}
+			if wanted {
+				addrs[bin] = append(addrs[bin], chunk.Address())
+				wantedChunksCount++
+			}
+			addrsMu.Unlock()
+
+			lastTimestampMu.RLock()
+			storeTimestamp := lastTimestamp
+			lastTimestampMu.RUnlock()
+
+			last[bin] = ChunkDescriptor{
+				Address:        chunk.Address(),
+				StoreTimestamp: storeTimestamp,
+			}
+		}
+		return last
+	}
+
+	// all chunks from upload1 are not expected
+	// as upload1 chunk is used as since for subscriptions
+	upload1 := uploadRandomChunks(100, false)
+
+	// all chunks from upload2 are expected
+	// as upload2 chunk is used as until for subscriptions
+	upload2 := uploadRandomChunks(100, true)
+
+	// upload some chunks before subscribe but after
+	// wanted chunks
+	uploadRandomChunks(8, false)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
+		var since *ChunkDescriptor
+		if c, ok := upload1[bin]; ok {
+			since = &c
+		}
+		until, ok := upload2[bin]
+		if !ok {
+			// no chunks un this bin uploaded in the upload2
+			// skip this bin from testing
+			continue
+		}
+		ch, stop := db.SubscribePull(ctx, bin, since, &until)
+		defer stop()
+
+		// receive and validate addresses from the subscription
+		go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan)
+	}
+
+	// upload some chunks just after subscribe
+	uploadRandomChunks(15, false)
+
+	checkErrChan(ctx, t, errChan, wantedChunksCount)
+}
+
+// uploadRandomChunksBin uploads random chunks to database and adds them to
+// the map of addresses ber bin.
+func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) {
+	for i := 0; i < count; i++ {
+		chunk := generateRandomChunk()
+
+		err := uploader.Put(chunk)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addrsMu.Lock()
+		bin := db.po(chunk.Address())
+		if _, ok := addrs[bin]; !ok {
+			addrs[bin] = make([]storage.Address, 0)
+		}
+		addrs[bin] = append(addrs[bin], chunk.Address())
+		addrsMu.Unlock()
+
+		*wantedChunksCount++
+	}
+}
+
+// readPullSubscriptionBin is a helper function that reads all ChunkDescriptors from a channel and
+// sends error to errChan, even if it is nil, to count the number of ChunkDescriptors
+// returned by the channel.
+func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, errChan chan error) {
+	var i int // address index
+	for {
+		select {
+		case got, ok := <-ch:
+			if !ok {
+				return
+			}
+			addrsMu.Lock()
+			if i+1 > len(addrs[bin]) {
+				errChan <- fmt.Errorf("got more chunk addresses %v, then expected %v, for bin %v", i+1, len(addrs[bin]), bin)
+			}
+			want := addrs[bin][i]
+			addrsMu.Unlock()
+			var err error
+			if !bytes.Equal(got.Address, want) {
+				err = fmt.Errorf("got chunk address %v in bin %v %s, want %s", i, bin, got.Address.Hex(), want)
+			}
+			i++
+			// send one and only one error per received address
+			errChan <- err
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// checkErrChan expects the number of wantedChunksCount errors from errChan
+// and calls t.Error for the ones that are not nil.
+func checkErrChan(ctx context.Context, t *testing.T, errChan chan error, wantedChunksCount int) {
+	t.Helper()
+
+	for i := 0; i < wantedChunksCount; i++ {
+		select {
+		case err := <-errChan:
+			if err != nil {
+				t.Error(err)
+			}
+		case <-ctx.Done():
+			t.Fatal(ctx.Err())
+		}
+	}
+}

swarm/storage/localstore/subscription_push.go

@@ -0,0 +1,145 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"context"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/shed"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// SubscribePush returns a channel that provides storage chunks with ordering from push syncing index.
+// Returned stop function will terminate current and further iterations, and also it will close
+// the returned channel without any errors. Make sure that you check the second returned parameter
+// from the channel to stop iteration when its value is false.
+func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop func()) {
+	chunks := make(chan storage.Chunk)
+	trigger := make(chan struct{}, 1)
+
+	db.pushTriggersMu.Lock()
+	db.pushTriggers = append(db.pushTriggers, trigger)
+	db.pushTriggersMu.Unlock()
+
+	// send signal for the initial iteration
+	trigger <- struct{}{}
+
+	stopChan := make(chan struct{})
+	var stopChanOnce sync.Once
+
+	go func() {
+		// close the returned chunkInfo channel at the end to
+		// signal that the subscription is done
+		defer close(chunks)
+		// sinceItem is the Item from which the next iteration
+		// should start. The first iteration starts from the first Item.
+		var sinceItem *shed.Item
+		for {
+			select {
+			case <-trigger:
+				// iterate until:
+				// - last index Item is reached
+				// - subscription stop is called
+				// - context is done
+				err := db.pushIndex.Iterate(func(item shed.Item) (stop bool, err error) {
+					// get chunk data
+					dataItem, err := db.retrievalDataIndex.Get(item)
+					if err != nil {
+						return true, err
+					}
+
+					select {
+					case chunks <- storage.NewChunk(dataItem.Address, dataItem.Data):
+						// set next iteration start item
+						// when its chunk is successfully sent to channel
+						sinceItem = &item
+						return false, nil
+					case <-stopChan:
+						// gracefully stop the iteration
+						// on stop
+						return true, nil
+					case <-db.close:
+						// gracefully stop the iteration
+						// on database close
+						return true, nil
+					case <-ctx.Done():
+						return true, ctx.Err()
+					}
+				}, &shed.IterateOptions{
+					StartFrom: sinceItem,
+					// sinceItem was sent as the last Address in the previous
+					// iterator call, skip it in this one
+					SkipStartFromItem: true,
+				})
+				if err != nil {
+					log.Error("localstore push subscription iteration", "err", err)
+					return
+				}
+			case <-stopChan:
+				// terminate the subscription
+				// on stop
+				return
+			case <-db.close:
+				// terminate the subscription
+				// on database close
+				return
+			case <-ctx.Done():
+				err := ctx.Err()
+				if err != nil {
+					log.Error("localstore push subscription", "err", err)
+				}
+				return
+			}
+		}
+	}()
+
+	stop = func() {
+		stopChanOnce.Do(func() {
+			close(stopChan)
+		})
+
+		db.pushTriggersMu.Lock()
+		defer db.pushTriggersMu.Unlock()
+
+		for i, t := range db.pushTriggers {
+			if t == trigger {
+				db.pushTriggers = append(db.pushTriggers[:i], db.pushTriggers[i+1:]...)
+				break
+			}
+		}
+	}
+
+	return chunks, stop
+}
+
+// triggerPushSubscriptions is used internally for starting iterations
+// on Push subscriptions. Whenever new item is added to the push index,
+// this function should be called.
+func (db *DB) triggerPushSubscriptions() {
+	db.pushTriggersMu.RLock()
+	triggers := db.pushTriggers
+	db.pushTriggersMu.RUnlock()
+
+	for _, t := range triggers {
+		select {
+		case t <- struct{}{}:
+		default:
+		}
+	}
+}

swarm/storage/localstore/subscription_push_test.go

@@ -0,0 +1,200 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package localstore
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// TestDB_SubscribePush uploads some chunks before and after
+// push syncing subscription is created and validates if
+// all addresses are received in the right order.
+func TestDB_SubscribePush(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	chunks := make([]storage.Chunk, 0)
+	var chunksMu sync.Mutex
+
+	uploadRandomChunks := func(count int) {
+		for i := 0; i < count; i++ {
+			chunk := generateRandomChunk()
+
+			err := uploader.Put(chunk)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			chunksMu.Lock()
+			chunks = append(chunks, chunk)
+			chunksMu.Unlock()
+		}
+	}
+
+	// prepopulate database with some chunks
+	// before the subscription
+	uploadRandomChunks(10)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	ch, stop := db.SubscribePush(ctx)
+	defer stop()
+
+	// receive and validate addresses from the subscription
+	go func() {
+		var i int // address index
+		for {
+			select {
+			case got, ok := <-ch:
+				if !ok {
+					return
+				}
+				chunksMu.Lock()
+				want := chunks[i]
+				chunksMu.Unlock()
+				var err error
+				if !bytes.Equal(got.Data(), want.Data()) {
+					err = fmt.Errorf("got chunk %v data %x, want %x", i, got.Data(), want.Data())
+				}
+				if !bytes.Equal(got.Address(), want.Address()) {
+					err = fmt.Errorf("got chunk %v address %s, want %s", i, got.Address().Hex(), want.Address().Hex())
+				}
+				i++
+				// send one and only one error per received address
+				errChan <- err
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	// upload some chunks just after subscribe
+	uploadRandomChunks(5)
+
+	time.Sleep(200 * time.Millisecond)
+
+	// upload some chunks after some short time
+	// to ensure that subscription will include them
+	// in a dynamic environment
+	uploadRandomChunks(3)
+
+	checkErrChan(ctx, t, errChan, len(chunks))
+}
+
+// TestDB_SubscribePush_multiple uploads chunks before and after
+// multiple push syncing subscriptions are created and
+// validates if all addresses are received in the right order.
+func TestDB_SubscribePush_multiple(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, nil)
+	defer cleanupFunc()
+
+	uploader := db.NewPutter(ModePutUpload)
+
+	addrs := make([]storage.Address, 0)
+	var addrsMu sync.Mutex
+
+	uploadRandomChunks := func(count int) {
+		for i := 0; i < count; i++ {
+			chunk := generateRandomChunk()
+
+			err := uploader.Put(chunk)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			addrsMu.Lock()
+			addrs = append(addrs, chunk.Address())
+			addrsMu.Unlock()
+		}
+	}
+
+	// prepopulate database with some chunks
+	// before the subscription
+	uploadRandomChunks(10)
+
+	// set a timeout on subscription
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// collect all errors from validating addresses, even nil ones
+	// to validate the number of addresses received by the subscription
+	errChan := make(chan error)
+
+	subsCount := 10
+
+	// start a number of subscriptions
+	// that all of them will write every addresses error to errChan
+	for j := 0; j < subsCount; j++ {
+		ch, stop := db.SubscribePush(ctx)
+		defer stop()
+
+		// receive and validate addresses from the subscription
+		go func(j int) {
+			var i int // address index
+			for {
+				select {
+				case got, ok := <-ch:
+					if !ok {
+						return
+					}
+					addrsMu.Lock()
+					want := addrs[i]
+					addrsMu.Unlock()
+					var err error
+					if !bytes.Equal(got.Address(), want) {
+						err = fmt.Errorf("got chunk %v address on subscription %v %s, want %s", i, j, got, want)
+					}
+					i++
+					// send one and only one error per received address
+					errChan <- err
+				case <-ctx.Done():
+					return
+				}
+			}
+		}(j)
+	}
+
+	// upload some chunks just after subscribe
+	uploadRandomChunks(5)
+
+	time.Sleep(200 * time.Millisecond)
+
+	// upload some chunks after some short time
+	// to ensure that subscription will include them
+	// in a dynamic environment
+	uploadRandomChunks(3)
+
+	// number of addresses received by all subscriptions
+	wantedChunksCount := len(addrs) * subsCount
+
+	checkErrChan(ctx, t, errChan, wantedChunksCount)
+}

swarm/storage/localstore_test.go

@@ -209,3 +209,36 @@ func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func())
 
 	return store, cleanup
 }
+
+func TestHas(t *testing.T) {
+	ldbCap := defaultGCRatio
+	store, cleanup := setupLocalStore(t, ldbCap)
+	defer cleanup()
+
+	nonStoredAddr := GenerateRandomChunk(128).Address()
+
+	has := store.Has(context.Background(), nonStoredAddr)
+	if has {
+		t.Fatal("Expected Has() to return false, but returned true!")
+	}
+
+	storeChunks := GenerateRandomChunks(128, 3)
+	for _, ch := range storeChunks {
+		err := store.Put(context.Background(), ch)
+		if err != nil {
+			t.Fatalf("Expected store to store chunk, but it failed: %v", err)
+		}
+
+		has := store.Has(context.Background(), ch.Address())
+		if !has {
+			t.Fatal("Expected Has() to return true, but returned false!")
+		}
+	}
+
+	//let's be paranoic and test again that the non-existent chunk returns false
+	has = store.Has(context.Background(), nonStoredAddr)
+	if has {
+		t.Fatal("Expected Has() to return false, but returned true!")
+	}
+
+}