diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index f7f6916c54..9a61d39325 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -9,9 +9,9 @@ les/ @zsfelfoldi
light/ @zsfelfoldi
mobile/ @karalabe
p2p/ @fjl @zsfelfoldi
-p2p/simulations @lmars
-p2p/protocols @zelig
-swarm/api/http @justelad
+p2p/simulations @lmars
+p2p/protocols @zelig
+swarm/api/http @justelad
swarm/bmt @zelig
swarm/dev @lmars
swarm/fuse @jmozah @holisticode
diff --git a/.travis.yml b/.travis.yml
index 69535b7ef3..33a4f89497 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,6 +29,14 @@ matrix:
- os: osx
go: 1.11.x
script:
+ - echo "Increase the maximum number of open file descriptors on macOS"
+ - NOFILE=20480
+ - sudo sysctl -w kern.maxfiles=$NOFILE
+ - sudo sysctl -w kern.maxfilesperproc=$NOFILE
+ - sudo launchctl limit maxfiles $NOFILE $NOFILE
+ - sudo launchctl limit maxfiles
+ - ulimit -S -n $NOFILE
+ - ulimit -n
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
@@ -148,7 +156,7 @@ matrix:
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
+ - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go
- export GOPATH=$HOME/go
diff --git a/README.md b/README.md
index f308fb1011..4b62bfde03 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
[Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
on the wiki.
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
You can install them using your favourite package manager.
Once the dependencies are installed, run
diff --git a/appveyor.yml b/appveyor.yml
index 11848ddb95..e5126b2529 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -23,8 +23,8 @@ environment:
install:
- git submodule update --init
- rmdir C:\go /s /q
- - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
- - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+ - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
+ - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version
- gcc --version
diff --git a/cmd/geth/main.go b/cmd/geth/main.go
index 0288b33806..69802a48a7 100644
--- a/cmd/geth/main.go
+++ b/cmd/geth/main.go
@@ -89,6 +89,7 @@ var (
utils.LightKDFFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
+ utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
utils.ListenPortFlag,
diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go
index 8b0491ce3c..82f17e0ee8 100644
--- a/cmd/geth/usage.go
+++ b/cmd/geth/usage.go
@@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{
Flags: []cli.Flag{
utils.CacheFlag,
utils.CacheDatabaseFlag,
+ utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
},
diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go
index 629781edd8..072541b659 100644
--- a/cmd/swarm/access.go
+++ b/cmd/swarm/access.go
@@ -114,6 +114,9 @@ func accessNewPass(ctx *cli.Context) {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+ if err != nil {
+ utils.Fatalf("had an error generating the manifest: %v", err)
+ }
if dryRun {
err = printManifests(m, nil)
if err != nil {
@@ -147,6 +150,9 @@ func accessNewPK(ctx *cli.Context) {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+ if err != nil {
+ utils.Fatalf("had an error generating the manifest: %v", err)
+ }
if dryRun {
err = printManifests(m, nil)
if err != nil {
diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go
index b4d2e1dbc5..9357c577e3 100644
--- a/cmd/swarm/access_test.go
+++ b/cmd/swarm/access_test.go
@@ -14,8 +14,6 @@
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see .
-// +build !windows
-
package main
import (
@@ -28,6 +26,7 @@ import (
gorand "math/rand"
"net/http"
"os"
+ "runtime"
"strings"
"testing"
"time"
@@ -37,7 +36,7 @@ import (
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
- swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+ swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
@@ -48,22 +47,41 @@ const (
var DefaultCurve = crypto.S256()
-// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
+func TestACT(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip()
+ }
+
+ initCluster(t)
+
+ cases := []struct {
+ name string
+ f func(t *testing.T)
+ }{
+ {"Password", testPassword},
+ {"PK", testPK},
+ {"ACTWithoutBogus", testACTWithoutBogus},
+ {"ACTWithBogus", testACTWithBogus},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, tc.f)
+ }
+}
+
+// testPassword tests for the correct creation of an ACT manifest protected by a password.
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
// is then fetched through 2nd node. since the tested code is not key-aware - we can just
// fetch from the 2nd node using HTTP BasicAuth
-func TestAccessPassword(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
- defer srv.Close()
-
+func testPassword(t *testing.T) {
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
- srv.URL, //it doesn't matter through which node we upload content
+ cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
@@ -137,16 +155,17 @@ func TestAccessPassword(t *testing.T) {
if a.Publisher != "" {
t.Fatal("should be empty")
}
- client := swarm.NewClient(srv.URL)
+
+ client := swarmapi.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
- httpClient := &http.Client{}
+ url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
- url := srv.URL + "/" + "bzz:/" + hash
+ httpClient := &http.Client{}
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
@@ -188,7 +207,7 @@ func TestAccessPassword(t *testing.T) {
//download file with 'swarm down' with wrong password
up = runSwarm(t,
"--bzzapi",
- srv.URL,
+ cluster.Nodes[0].URL,
"down",
"bzz:/"+hash,
tmp,
@@ -202,16 +221,12 @@ func TestAccessPassword(t *testing.T) {
up.ExpectExit()
}
-// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
+// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
// the test will fail if the proxy's given private key is not granted on the ACT.
-func TestAccessPK(t *testing.T) {
- // Setup Swarm and upload a test file to it
- cluster := newTestCluster(t, 2)
- defer cluster.Shutdown()
-
+func testPK(t *testing.T) {
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
@@ -317,7 +332,7 @@ func TestAccessPK(t *testing.T) {
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
- client := swarm.NewClient(cluster.Nodes[0].URL)
+ client := swarmapi.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
@@ -343,29 +358,24 @@ func TestAccessPK(t *testing.T) {
}
}
-// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
-func TestAccessACT(t *testing.T) {
- testAccessACT(t, 0)
+// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
+func testACTWithoutBogus(t *testing.T) {
+ testACT(t, 0)
}
-// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
-func TestAccessACTScale(t *testing.T) {
- testAccessACT(t, 1000)
+// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
+func testACTWithBogus(t *testing.T) {
+ testACT(t, 100)
}
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
// the publisher uploads through one of the nodes then disappears.
-func testAccessACT(t *testing.T, bogusEntries int) {
- // Setup Swarm and upload a test file to it
- const clusterSize = 3
- cluster := newTestCluster(t, clusterSize)
- defer cluster.Shutdown()
-
+func testACT(t *testing.T, bogusEntries int) {
var uploadThroughNode = cluster.Nodes[0]
- client := swarm.NewClient(uploadThroughNode.URL)
+ client := swarmapi.NewClient(uploadThroughNode.URL)
r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)
diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go
index 02198f878e..18be316e5f 100644
--- a/cmd/swarm/config_test.go
+++ b/cmd/swarm/config_test.go
@@ -26,14 +26,14 @@ import (
"testing"
"time"
+ "github.com/docker/docker/pkg/reexec"
+ "github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm"
"github.com/ethereum/go-ethereum/swarm/api"
-
- "github.com/docker/docker/pkg/reexec"
)
-func TestDumpConfig(t *testing.T) {
+func TestConfigDump(t *testing.T) {
swarm := runSwarm(t, "dumpconfig")
defaultConf := api.NewConfig()
out, err := tomlSettings.Marshal(&defaultConf)
@@ -91,8 +91,8 @@ func TestConfigCmdLineOverrides(t *testing.T) {
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name),
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
- "--datadir", dir,
- "--ipcpath", conf.IPCPath,
+ fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
@@ -189,9 +189,9 @@ func TestConfigFileOverrides(t *testing.T) {
flags := []string{
fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
- "--ens-api", "",
- "--ipcpath", conf.IPCPath,
- "--datadir", dir,
+ fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
+ fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
@@ -407,9 +407,9 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
- "--ens-api", "",
- "--datadir", dir,
- "--ipcpath", conf.IPCPath,
+ fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
+ fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
@@ -466,7 +466,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
node.Shutdown()
}
-func TestValidateConfig(t *testing.T) {
+func TestConfigValidate(t *testing.T) {
for _, c := range []struct {
cfg *api.Config
err string
diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go
index c533511af0..e8671eea79 100644
--- a/cmd/swarm/export_test.go
+++ b/cmd/swarm/export_test.go
@@ -19,9 +19,7 @@ package main
import (
"bytes"
"crypto/md5"
- "crypto/rand"
"io"
- "io/ioutil"
"net/http"
"os"
"runtime"
@@ -29,6 +27,7 @@ import (
"testing"
"github.com/ethereum/go-ethereum/swarm"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
// TestCLISwarmExportImport perform the following test:
@@ -44,12 +43,13 @@ func TestCLISwarmExportImport(t *testing.T) {
}
cluster := newTestCluster(t, 1)
- // generate random 10mb file
- f, cleanup := generateRandomFile(t, 10000000)
- defer cleanup()
+ // generate random 1mb file
+ content := testutil.RandomBytes(1, 1000000)
+ fileName := testutil.TempFileWithContent(t, string(content))
+ defer os.Remove(fileName)
// upload the file with 'swarm up' and expect a hash
- up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+ up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
hash := matches[0]
@@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
}
// compare downloaded file with the generated random file
- mustEqualFiles(t, f, res.Body)
+ mustEqualFiles(t, bytes.NewReader(content), res.Body)
}
func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
}
}
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
- // create a tmp file
- tmp, err := ioutil.TempFile("", "swarm-test")
- if err != nil {
- t.Fatal(err)
- }
-
- // callback for tmp file cleanup
- teardown = func() {
- tmp.Close()
- os.Remove(tmp.Name())
- }
-
- // write 10mb random data to file
- buf := make([]byte, 10000000)
- _, err = rand.Read(buf)
- if err != nil {
- t.Fatal(err)
- }
- ioutil.WriteFile(tmp.Name(), buf, 0755)
-
- return tmp, teardown
-}
diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go
index 46727c21d3..a0cedf0d36 100644
--- a/cmd/swarm/feeds_test.go
+++ b/cmd/swarm/feeds_test.go
@@ -20,49 +20,35 @@ import (
"bytes"
"encoding/json"
"fmt"
- "io"
"io/ioutil"
"os"
"testing"
- "github.com/ethereum/go-ethereum/swarm/api"
- "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
- "github.com/ethereum/go-ethereum/swarm/testutil"
-
- "github.com/ethereum/go-ethereum/crypto"
- "github.com/ethereum/go-ethereum/swarm/storage/feed"
-
"github.com/ethereum/go-ethereum/common/hexutil"
+ "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+ "github.com/ethereum/go-ethereum/swarm/storage/feed"
+ "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestCLIFeedUpdate(t *testing.T) {
-
- srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+ srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}, nil)
log.Info("starting a test swarm server")
defer srv.Close()
// create a private key file for signing
- pkfile, err := ioutil.TempFile("", "swarm-test")
- if err != nil {
- t.Fatal(err)
- }
- defer pkfile.Close()
- defer os.Remove(pkfile.Name())
-
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
privKey, _ := crypto.HexToECDSA(privkeyHex)
address := crypto.PubkeyToAddress(privKey.PublicKey)
- // save the private key to a file
- _, err = io.WriteString(pkfile, privkeyHex)
- if err != nil {
- t.Fatal(err)
- }
+ pkFileName := testutil.TempFileWithContent(t, privkeyHex)
+ defer os.Remove(pkFileName)
// compose a topic. We'll be doing quotes about Miguel de Cervantes
var topic feed.Topic
@@ -76,7 +62,7 @@ func TestCLIFeedUpdate(t *testing.T) {
flags := []string{
"--bzzapi", srv.URL,
- "--bzzaccount", pkfile.Name(),
+ "--bzzaccount", pkFileName,
"feed", "update",
"--topic", topic.Hex(),
"--name", name,
@@ -89,13 +75,10 @@ func TestCLIFeedUpdate(t *testing.T) {
// now try to get the update using the client
client := swarm.NewClient(srv.URL)
- if err != nil {
- t.Fatal(err)
- }
// build the same topic as before, this time
// we use NewTopic to create a topic automatically.
- topic, err = feed.NewTopic(name, subject)
+ topic, err := feed.NewTopic(name, subject)
if err != nil {
t.Fatal(err)
}
@@ -153,7 +136,7 @@ func TestCLIFeedUpdate(t *testing.T) {
// test publishing a manifest
flags = []string{
"--bzzapi", srv.URL,
- "--bzzaccount", pkfile.Name(),
+ "--bzzaccount", pkFileName,
"feed", "create",
"--topic", topic.Hex(),
}
diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go
index 7e891f3022..0dedca674b 100644
--- a/cmd/swarm/flags.go
+++ b/cmd/swarm/flags.go
@@ -88,7 +88,7 @@ var (
}
SwarmApiFlag = cli.StringFlag{
Name: "bzzapi",
- Usage: "Swarm HTTP endpoint",
+ Usage: "Specifies the Swarm HTTP endpoint to connect to",
Value: "http://127.0.0.1:8500",
}
SwarmRecursiveFlag = cli.BoolFlag{
diff --git a/cmd/swarm/fs.go b/cmd/swarm/fs.go
index b970b2e8c2..edeeddff84 100644
--- a/cmd/swarm/fs.go
+++ b/cmd/swarm/fs.go
@@ -24,7 +24,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
- "github.com/ethereum/go-ethereum/node"
+ "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/fuse"
"gopkg.in/urfave/cli.v1"
@@ -41,27 +41,24 @@ var fsCommand = cli.Command{
Action: mount,
CustomHelpTemplate: helpTemplate,
Name: "mount",
- Flags: []cli.Flag{utils.IPCPathFlag},
Usage: "mount a swarm hash to a mount point",
- ArgsUsage: "swarm fs mount --ipcpath ",
+ ArgsUsage: "swarm fs mount ",
Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
{
Action: unmount,
CustomHelpTemplate: helpTemplate,
Name: "unmount",
- Flags: []cli.Flag{utils.IPCPathFlag},
Usage: "unmount a swarmfs mount",
- ArgsUsage: "swarm fs unmount --ipcpath ",
+ ArgsUsage: "swarm fs unmount ",
Description: "Unmounts a swarmfs mount residing at . This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
{
Action: listMounts,
CustomHelpTemplate: helpTemplate,
Name: "list",
- Flags: []cli.Flag{utils.IPCPathFlag},
Usage: "list swarmfs mounts",
- ArgsUsage: "swarm fs list --ipcpath ",
+ ArgsUsage: "swarm fs list",
Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
},
@@ -70,7 +67,7 @@ var fsCommand = cli.Command{
func mount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 2 {
- utils.Fatalf("Usage: swarm fs mount --ipcpath ")
+ utils.Fatalf("Usage: swarm fs mount ")
}
client, err := dialRPC(cliContext)
@@ -97,7 +94,7 @@ func unmount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 1 {
- utils.Fatalf("Usage: swarm fs unmount --ipcpath ")
+ utils.Fatalf("Usage: swarm fs unmount ")
}
client, err := dialRPC(cliContext)
if err != nil {
@@ -145,20 +142,21 @@ func listMounts(cliContext *cli.Context) {
}
func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
- var endpoint string
+ endpoint := getIPCEndpoint(ctx)
+ log.Info("IPC endpoint", "path", endpoint)
+ return rpc.Dial(endpoint)
+}
- if ctx.IsSet(utils.IPCPathFlag.Name) {
- endpoint = ctx.String(utils.IPCPathFlag.Name)
- } else {
- utils.Fatalf("swarm ipc endpoint not specified")
- }
+func getIPCEndpoint(ctx *cli.Context) string {
+ cfg := defaultNodeConfig
+ utils.SetNodeConfig(ctx, &cfg)
- if endpoint == "" {
- endpoint = node.DefaultIPCEndpoint(clientIdentifier)
- } else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
+ endpoint := cfg.IPCEndpoint()
+
+ if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
// Backwards compatibility with geth < 1.5 which required
// these prefixes.
endpoint = endpoint[4:]
}
- return rpc.Dial(endpoint)
+ return endpoint
}
diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go
index 4f38b094bb..5f58d6c0d8 100644
--- a/cmd/swarm/fs_test.go
+++ b/cmd/swarm/fs_test.go
@@ -20,6 +20,7 @@ package main
import (
"bytes"
+ "fmt"
"io"
"io/ioutil"
"os"
@@ -28,20 +29,35 @@ import (
"testing"
"time"
+ "github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log"
- colorable "github.com/mattn/go-colorable"
)
-func init() {
- log.PrintOrigins(true)
- log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
-}
-
type testFile struct {
filePath string
content string
}
+// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list
+// can find and correctly connect to a running Swarm node on the default
+// IPCPath.
+func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
+ cluster := newTestCluster(t, 1)
+ defer cluster.Shutdown()
+
+ handlingNode := cluster.Nodes[0]
+ list := runSwarm(t, []string{
+ "--datadir", handlingNode.Dir,
+ "fs",
+ "list",
+ }...)
+
+ list.WaitExit()
+ if list.Err != nil {
+ t.Fatal(list.Err)
+ }
+}
+
// TestCLISwarmFs is a high-level test of swarmfs
//
// This test fails on travis for macOS as this executable exits with code 1
@@ -65,9 +81,9 @@ func TestCLISwarmFs(t *testing.T) {
log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
mount := runSwarm(t, []string{
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
- "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
mhash,
mountPoint,
}...)
@@ -80,6 +96,9 @@ func TestCLISwarmFs(t *testing.T) {
t.Fatal(err)
}
dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
+ if err != nil {
+ t.Fatal(err)
+ }
dummyContent := "somerandomtestcontentthatshouldbeasserted"
dirs := []string{
@@ -104,9 +123,9 @@ func TestCLISwarmFs(t *testing.T) {
log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmount := runSwarm(t, []string{
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"unmount",
- "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
mountPoint,
}...)
_, matches := unmount.ExpectRegexp(hashRegexp)
@@ -139,9 +158,9 @@ func TestCLISwarmFs(t *testing.T) {
//remount, check files
newMount := runSwarm(t, []string{
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
- "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
hash, // the latest hash
secondMountPoint,
}...)
@@ -175,9 +194,9 @@ func TestCLISwarmFs(t *testing.T) {
log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmountSec := runSwarm(t, []string{
+ fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"unmount",
- "--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
secondMountPoint,
}...)
diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go
index 7ea4e0c45b..01d982aa7a 100644
--- a/cmd/swarm/manifest_test.go
+++ b/cmd/swarm/manifest_test.go
@@ -26,7 +26,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
- "github.com/ethereum/go-ethereum/swarm/testutil"
+ swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
)
// TestManifestChange tests manifest add, update and remove
@@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) {
// Argument encrypt controls whether to use encryption or not.
func testManifestChange(t *testing.T, encrypt bool) {
t.Parallel()
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
@@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
t.Parallel()
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go
index 55199e955e..680d238d06 100644
--- a/cmd/swarm/run_test.go
+++ b/cmd/swarm/run_test.go
@@ -42,7 +42,6 @@ import (
"github.com/ethereum/go-ethereum/swarm"
"github.com/ethereum/go-ethereum/swarm/api"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
- "github.com/ethereum/go-ethereum/swarm/testutil"
)
var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@@ -58,7 +57,18 @@ func init() {
})
}
-func serverFunc(api *api.API) testutil.TestServer {
+const clusterSize = 3
+
+var clusteronce sync.Once
+var cluster *testCluster
+
+func initCluster(t *testing.T) {
+ clusteronce.Do(func() {
+ cluster = newTestCluster(t, clusterSize)
+ })
+}
+
+func serverFunc(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}
func TestMain(m *testing.M) {
diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go
index 97054dd479..4ff17fd5b7 100644
--- a/cmd/swarm/swarm-smoke/main.go
+++ b/cmd/swarm/swarm-smoke/main.go
@@ -45,8 +45,8 @@ func main() {
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "cluster-endpoint",
- Value: "testing",
- Usage: "cluster to point to (local, open or testing)",
+ Value: "prod",
+ Usage: "cluster to point to (prod or a given namespace)",
Destination: &cluster,
},
cli.IntFlag{
diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go
index 358141c7cc..7872421d3d 100644
--- a/cmd/swarm/swarm-smoke/upload_and_sync.go
+++ b/cmd/swarm/swarm-smoke/upload_and_sync.go
@@ -20,6 +20,7 @@ import (
"bytes"
"crypto/md5"
crand "crypto/rand"
+ "crypto/tls"
"errors"
"fmt"
"io"
@@ -32,6 +33,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
+ colorable "github.com/mattn/go-colorable"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
@@ -39,18 +41,13 @@ import (
func generateEndpoints(scheme string, cluster string, from int, to int) {
if cluster == "prod" {
- cluster = ""
- } else if cluster == "local" {
for port := from; port <= to; port++ {
- endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
+ endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
}
- return
} else {
- cluster = cluster + "."
- }
-
- for port := from; port <= to; port++ {
- endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
+ for port := from; port <= to; port++ {
+ endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster))
+ }
}
if includeLocalhost {
@@ -59,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
}
func cliUploadAndSync(c *cli.Context) error {
+ log.PrintOrigins(true)
+ log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
generateEndpoints(scheme, cluster, from, to)
@@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
time.Sleep(3 * time.Second)
log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
- res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
+ client := &http.Client{Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }}
+ res, err := client.Get(endpoint + "/bzz:/" + hash + "/")
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go
index 0ac2456a51..616486e37c 100644
--- a/cmd/swarm/upload_test.go
+++ b/cmd/swarm/upload_test.go
@@ -31,7 +31,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
- swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+ swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/mattn/go-colorable"
)
@@ -41,69 +41,66 @@ func init() {
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
-// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
+func TestSwarmUp(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip()
+ }
+
+ initCluster(t)
+
+ cases := []struct {
+ name string
+ f func(t *testing.T)
+ }{
+ {"NoEncryption", testNoEncryption},
+ {"Encrypted", testEncrypted},
+ {"RecursiveNoEncryption", testRecursiveNoEncryption},
+ {"RecursiveEncrypted", testRecursiveEncrypted},
+ {"DefaultPathAll", testDefaultPathAll},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, tc.f)
+ }
+}
+
+// testNoEncryption tests that running 'swarm up' makes the resulting file
// available from all nodes via the HTTP API
-func TestCLISwarmUp(t *testing.T) {
- if runtime.GOOS == "windows" {
- t.Skip()
- }
-
- testCLISwarmUp(false, t)
-}
-func TestCLISwarmUpRecursive(t *testing.T) {
- if runtime.GOOS == "windows" {
- t.Skip()
- }
- testCLISwarmUpRecursive(false, t)
+func testNoEncryption(t *testing.T) {
+ testDefault(false, t)
}
-// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
+// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
// available from all nodes via the HTTP API
-func TestCLISwarmUpEncrypted(t *testing.T) {
- if runtime.GOOS == "windows" {
- t.Skip()
- }
- testCLISwarmUp(true, t)
-}
-func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
- if runtime.GOOS == "windows" {
- t.Skip()
- }
- testCLISwarmUpRecursive(true, t)
+func testEncrypted(t *testing.T) {
+ testDefault(true, t)
}
-func testCLISwarmUp(toEncrypt bool, t *testing.T) {
- log.Info("starting 3 node cluster")
- cluster := newTestCluster(t, 3)
- defer cluster.Shutdown()
+func testRecursiveNoEncryption(t *testing.T) {
+ testRecursive(false, t)
+}
- // create a tmp file
- tmp, err := ioutil.TempFile("", "swarm-test")
- if err != nil {
- t.Fatal(err)
- }
- defer tmp.Close()
- defer os.Remove(tmp.Name())
+func testRecursiveEncrypted(t *testing.T) {
+ testRecursive(true, t)
+}
+
+func testDefault(toEncrypt bool, t *testing.T) {
+ tmpFileName := testutil.TempFileWithContent(t, data)
+ defer os.Remove(tmpFileName)
// write data to file
- data := "notsorandomdata"
- _, err = io.WriteString(tmp, data)
- if err != nil {
- t.Fatal(err)
- }
-
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
- tmp.Name()}
+ tmpFileName}
if toEncrypt {
hashRegexp = `[a-f\d]{128}`
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
"--encrypt",
- tmp.Name()}
+ tmpFileName}
}
// upload the file with 'swarm up' and expect a hash
log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@@ -192,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
}
}
-func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
- fmt.Println("starting 3 node cluster")
- cluster := newTestCluster(t, 3)
- defer cluster.Shutdown()
-
+func testRecursive(toEncrypt bool, t *testing.T) {
tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpUploadDir)
// create tmp files
- data := "notsorandomdata"
for _, path := range []string{"tmp1", "tmp2"} {
if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
t.Fatal(err)
@@ -243,8 +235,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
}
defer os.RemoveAll(tmpDownload)
bzzLocator := "bzz:/" + hash
- flagss := []string{}
- flagss = []string{
+ flagss := []string{
"--bzzapi", cluster.Nodes[0].URL,
"down",
"--recursive",
@@ -265,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
switch mode := fi.Mode(); {
case mode.IsRegular():
- if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
+ if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
} else {
ff := make([]byte, len(data))
@@ -286,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
}
}
-// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// testDefaultPathAll tests swarm recursive upload with relative and absolute
// default paths and with encryption.
-func TestCLISwarmUpDefaultPath(t *testing.T) {
- if runtime.GOOS == "windows" {
- t.Skip()
- }
- testCLISwarmUpDefaultPath(false, false, t)
- testCLISwarmUpDefaultPath(false, true, t)
- testCLISwarmUpDefaultPath(true, false, t)
- testCLISwarmUpDefaultPath(true, true, t)
+func testDefaultPathAll(t *testing.T) {
+ testDefaultPath(false, false, t)
+ testDefaultPath(false, true, t)
+ testDefaultPath(true, false, t)
+ testDefaultPath(true, true, t)
}
-func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
- defer srv.Close()
-
+func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
if err != nil {
t.Fatal(err)
@@ -324,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
args := []string{
"--bzzapi",
- srv.URL,
+ cluster.Nodes[0].URL,
"--recursive",
"--defaultpath",
defaultPath,
@@ -341,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
up.ExpectExit()
hash := matches[0]
- client := swarm.NewClient(srv.URL)
+ client := swarmapi.NewClient(cluster.Nodes[0].URL)
m, isEncrypted, err := client.DownloadManifest(hash)
if err != nil {
diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go
index 58d72f32ba..f23aa5775b 100644
--- a/cmd/utils/cmd.go
+++ b/cmd/utils/cmd.go
@@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
// Accumulate the preimages and flush when enough ws gathered
preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
if len(preimages) > 1024 {
- rawdb.WritePreimages(db, 0, preimages)
+ rawdb.WritePreimages(db, preimages)
preimages = make(map[common.Hash][]byte)
}
}
// Flush the last batch preimage data
if len(preimages) > 0 {
- rawdb.WritePreimages(db, 0, preimages)
+ rawdb.WritePreimages(db, preimages)
}
return nil
}
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 429c2bbb9b..d0597c2f18 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -295,7 +295,12 @@ var (
CacheDatabaseFlag = cli.IntFlag{
Name: "cache.database",
Usage: "Percentage of cache memory allowance to use for database io",
- Value: 75,
+ Value: 50,
+ }
+ CacheTrieFlag = cli.IntFlag{
+ Name: "cache.trie",
+ Usage: "Percentage of cache memory allowance to use for trie caching",
+ Value: 25,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
@@ -973,16 +978,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
setWS(ctx, cfg)
setNodeUserIdent(ctx, cfg)
- switch {
- case ctx.GlobalIsSet(DataDirFlag.Name):
- cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
- case ctx.GlobalBool(DeveloperFlag.Name):
- cfg.DataDir = "" // unless explicitly requested, use memory databases
- case ctx.GlobalBool(TestnetFlag.Name):
- cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
- case ctx.GlobalBool(RinkebyFlag.Name):
- cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
- }
+ setDataDir(ctx, cfg)
if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
@@ -995,6 +991,19 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
}
}
+func setDataDir(ctx *cli.Context, cfg *node.Config) {
+ switch {
+ case ctx.GlobalIsSet(DataDirFlag.Name):
+ cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
+ case ctx.GlobalBool(DeveloperFlag.Name):
+ cfg.DataDir = "" // unless explicitly requested, use memory databases
+ case ctx.GlobalBool(TestnetFlag.Name):
+ cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
+ case ctx.GlobalBool(RinkebyFlag.Name):
+ cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
+ }
+}
+
func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
if ctx.GlobalIsSet(GpoBlocksFlag.Name) {
cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name)
@@ -1157,8 +1166,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
}
cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
+ if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+ cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+ }
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
- cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+ cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
@@ -1393,12 +1405,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cache := &core.CacheConfig{
- Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
- TrieNodeLimit: eth.DefaultConfig.TrieCache,
- TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
+ Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
+ TrieCleanLimit: eth.DefaultConfig.TrieCleanCache,
+ TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
+ TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
+ }
+ if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+ cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
- cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+ cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)
diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go
index f6e8d2e42a..b7c8ec5638 100644
--- a/common/compiler/solidity.go
+++ b/common/compiler/solidity.go
@@ -31,14 +31,15 @@ import (
var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
-// Contract contains information about a compiled contract, alongside its code.
+// Contract contains information about a compiled contract, alongside its code and runtime code.
type Contract struct {
- Code string `json:"code"`
- Info ContractInfo `json:"info"`
+ Code string `json:"code"`
+ RuntimeCode string `json:"runtime-code"`
+ Info ContractInfo `json:"info"`
}
// ContractInfo contains information about a compiled contract, including access
-// to the ABI definition, user and developer docs, and metadata.
+// to the ABI definition, source mapping, user and developer docs, and metadata.
//
// Depending on the source, language version, compiler version, and compiler
// options will provide information about how the contract was compiled.
@@ -48,6 +49,8 @@ type ContractInfo struct {
LanguageVersion string `json:"languageVersion"`
CompilerVersion string `json:"compilerVersion"`
CompilerOptions string `json:"compilerOptions"`
+ SrcMap string `json:"srcMap"`
+ SrcMapRuntime string `json:"srcMapRuntime"`
AbiDefinition interface{} `json:"abiDefinition"`
UserDoc interface{} `json:"userDoc"`
DeveloperDoc interface{} `json:"developerDoc"`
@@ -63,14 +66,16 @@ type Solidity struct {
// --combined-output format
type solcOutput struct {
Contracts map[string]struct {
- Bin, Abi, Devdoc, Userdoc, Metadata string
+ BinRuntime string `json:"bin-runtime"`
+ SrcMapRuntime string `json:"srcmap-runtime"`
+ Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string
}
Version string
}
func (s *Solidity) makeArgs() []string {
p := []string{
- "--combined-json", "bin,abi,userdoc,devdoc",
+ "--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
"--optimize", // code optimizer switched on
}
if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
@@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro
// provided source, language and compiler version, and compiler options are all
// passed through into the Contract structs.
//
-// The solc output is expected to contain ABI, user docs, and dev docs.
+// The solc output is expected to contain ABI, source mapping, user docs, and dev docs.
//
// Returns an error if the JSON is malformed or missing data, or if the JSON
// embedded within the JSON is malformed.
@@ -184,13 +189,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
}
contracts[name] = &Contract{
- Code: "0x" + info.Bin,
+ Code: "0x" + info.Bin,
+ RuntimeCode: "0x" + info.BinRuntime,
Info: ContractInfo{
Source: source,
Language: "Solidity",
LanguageVersion: languageVersion,
CompilerVersion: compilerVersion,
CompilerOptions: compilerOptions,
+ SrcMap: info.SrcMap,
+ SrcMapRuntime: info.SrcMapRuntime,
AbiDefinition: abi,
UserDoc: userdoc,
DeveloperDoc: devdoc,
diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go
index eae09f91df..0cb72c35c7 100644
--- a/consensus/clique/clique.go
+++ b/consensus/clique/clique.go
@@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash {
return sigHash(header)
}
-// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
+// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
func (c *Clique) Close() error {
return nil
}
diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go
index a04ea235d9..4d8eed4161 100644
--- a/consensus/ethash/api.go
+++ b/consensus/ethash/api.go
@@ -37,27 +37,28 @@ type API struct {
// result[0] - 32 bytes hex encoded current block header pow-hash
// result[1] - 32 bytes hex encoded seed hash used for DAG
// result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
-func (api *API) GetWork() ([3]string, error) {
+// result[3] - hex encoded block number
+func (api *API) GetWork() ([4]string, error) {
if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest {
- return [3]string{}, errors.New("not supported")
+ return [4]string{}, errors.New("not supported")
}
var (
- workCh = make(chan [3]string, 1)
+ workCh = make(chan [4]string, 1)
errc = make(chan error, 1)
)
select {
case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}:
case <-api.ethash.exitCh:
- return [3]string{}, errEthashStopped
+ return [4]string{}, errEthashStopped
}
select {
case work := <-workCh:
return work, nil
case err := <-errc:
- return [3]string{}, err
+ return [4]string{}, err
}
}
diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go
index d124cb1e20..78892e1da8 100644
--- a/consensus/ethash/ethash.go
+++ b/consensus/ethash/ethash.go
@@ -432,7 +432,7 @@ type hashrate struct {
// sealWork wraps a seal work package for remote sealer.
type sealWork struct {
errc chan error
- res chan [3]string
+ res chan [4]string
}
// Ethash is a consensus engine based on proof-of-work implementing the ethash
diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go
index 8eded2ca81..90cb6470f7 100644
--- a/consensus/ethash/ethash_test.go
+++ b/consensus/ethash/ethash_test.go
@@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) {
ethash.Seal(nil, block, results, nil)
var (
- work [3]string
+ work [4]string
err error
)
if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {
diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go
index 06c98a7811..3a0919ca99 100644
--- a/consensus/ethash/sealer.go
+++ b/consensus/ethash/sealer.go
@@ -30,6 +30,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
@@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
results chan<- *types.Block
currentBlock *types.Block
- currentWork [3]string
+ currentWork [4]string
notifyTransport = &http.Transport{}
notifyClient = &http.Client{
@@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
// result[0], 32 bytes hex encoded current block header pow-hash
// result[1], 32 bytes hex encoded seed hash used for DAG
// result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
+ // result[3], hex encoded block number
makeWork := func(block *types.Block) {
hash := ethash.SealHash(block.Header())
currentWork[0] = hash.Hex()
currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex()
+ currentWork[3] = hexutil.EncodeBig(block.Number())
// Trace the seal work fetched by remote sealer.
currentBlock = block
diff --git a/core/block_validator.go b/core/block_validator.go
index 1329f62427..3b9496fece 100644
--- a/core/block_validator.go
+++ b/core/block_validator.go
@@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
return ErrKnownBlock
}
- if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
- if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
- return consensus.ErrUnknownAncestor
- }
- return consensus.ErrPrunedAncestor
- }
// Header validity is known at this point, check the uncles and transactions
header := block.Header()
if err := v.engine.VerifyUncles(v.bc, block); err != nil {
@@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
}
+ if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+ if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
+ return consensus.ErrUnknownAncestor
+ }
+ return consensus.ErrPrunedAncestor
+ }
return nil
}
diff --git a/core/blockchain.go b/core/blockchain.go
index f4a818f4c0..bcd53d4987 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -68,9 +68,10 @@ const (
// CacheConfig contains the configuration values for the trie caching/pruning
// that's resident in a blockchain.
type CacheConfig struct {
- Disabled bool // Whether to disable trie write caching (archive node)
- TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk
- TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
+ Disabled bool // Whether to disable trie write caching (archive node)
+ TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
+ TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
+ TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
}
// BlockChain represents the canonical chain given a database with a genesis
@@ -140,8 +141,9 @@ type BlockChain struct {
func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
if cacheConfig == nil {
cacheConfig = &CacheConfig{
- TrieNodeLimit: 256 * 1024 * 1024,
- TrieTimeLimit: 5 * time.Minute,
+ TrieCleanLimit: 256,
+ TrieDirtyLimit: 256,
+ TrieTimeLimit: 5 * time.Minute,
}
}
bodyCache, _ := lru.New(bodyCacheLimit)
@@ -156,7 +158,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
cacheConfig: cacheConfig,
db: db,
triegc: prque.New(nil),
- stateCache: state.NewDatabase(db),
+ stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
quit: make(chan struct{}),
shouldPreserve: shouldPreserve,
bodyCache: bodyCache,
@@ -393,6 +395,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
return state.New(root, bc.stateCache)
}
+// StateCache returns the caching database underpinning the blockchain instance.
+func (bc *BlockChain) StateCache() state.Database {
+ return bc.stateCache
+}
+
// Reset purges the entire blockchain, restoring it to its genesis state.
func (bc *BlockChain) Reset() error {
return bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -438,7 +445,11 @@ func (bc *BlockChain) repair(head **types.Block) error {
return nil
}
// Otherwise rewind one block and recheck state availability there
- (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+ block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+ if block == nil {
+ return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
+ }
+ (*head) = block
}
}
@@ -554,6 +565,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
return rawdb.HasBody(bc.db, hash, number)
}
+// HasFastBlock checks if a fast block is fully present in the database or not.
+func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
+ if !bc.HasBlock(hash, number) {
+ return false
+ }
+ if bc.receiptsCache.Contains(hash) {
+ return true
+ }
+ return rawdb.HasReceipts(bc.db, hash, number)
+}
+
// HasState checks if state trie is fully present in the database or not.
func (bc *BlockChain) HasState(hash common.Hash) bool {
_, err := bc.stateCache.OpenTrie(hash)
@@ -611,12 +633,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
if receipts, ok := bc.receiptsCache.Get(hash); ok {
return receipts.(types.Receipts)
}
-
number := rawdb.ReadHeaderNumber(bc.db, hash)
if number == nil {
return nil
}
-
receipts := rawdb.ReadReceipts(bc.db, hash, *number)
bc.receiptsCache.Add(hash, receipts)
return receipts
@@ -938,7 +958,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
// If we exceeded our memory allowance, flush matured singleton nodes to disk
var (
nodes, imgs = triedb.Size()
- limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
+ limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
)
if nodes > limit || imgs > 4*1024*1024 {
triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1002,7 +1022,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
}
// Write the positional metadata for transaction/receipt lookups and preimages
rawdb.WriteTxLookupEntries(batch, block)
- rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages())
+ rawdb.WritePreimages(batch, state.Preimages())
status = CanonStatTy
} else {
@@ -1020,6 +1040,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
return status, nil
}
+// addFutureBlock checks if the block is within the max allowed window to get
+// accepted for future processing, and returns an error if the block is too far
+// ahead and was not added.
+func (bc *BlockChain) addFutureBlock(block *types.Block) error {
+ max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
+ if block.Time().Cmp(max) > 0 {
+ return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
+ }
+ bc.futureBlocks.Add(block.Hash(), block)
+ return nil
+}
+
// InsertChain attempts to insert the given batch of blocks in to the canonical
// chain or, otherwise, create a fork. If an error is returned it will return
// the index number of the failing block as well an error describing what went
@@ -1027,18 +1059,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
//
// After insertion is done, all accumulated events will be fired.
func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
- n, events, logs, err := bc.insertChain(chain)
- bc.PostChainEvents(events, logs)
- return n, err
-}
-
-// insertChain will execute the actual chain insertion and event aggregation. The
-// only reason this method exists as a separate one is to make locking cleaner
-// with deferred statements.
-func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) {
// Sanity check that we have something meaningful to import
if len(chain) == 0 {
- return 0, nil, nil, nil
+ return 0, nil
}
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(chain); i++ {
@@ -1047,16 +1070,36 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
- return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
+ return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
}
}
// Pre-checks passed, start the full block imports
bc.wg.Add(1)
- defer bc.wg.Done()
-
bc.chainmu.Lock()
- defer bc.chainmu.Unlock()
+ n, events, logs, err := bc.insertChain(chain, true)
+ bc.chainmu.Unlock()
+ bc.wg.Done()
+
+ bc.PostChainEvents(events, logs)
+ return n, err
+}
+
+// insertChain is the internal implementation of insertChain, which assumes that
+// 1) chains are contiguous, and 2) The chain mutex is held.
+//
+// This method is split out so that import batches that require re-injecting
+// historical blocks can do so without releasing the lock, which could lead to
+// racey behaviour. If a sidechain import is in progress, and the historic state
+// is imported, but then new canon-head is added before the actual sidechain
+// completes, then the historic state could be pruned again
+func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) {
+ // If the chain is terminating, don't even bother starting u
+ if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+ return 0, nil, nil, nil
+ }
+ // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
+ senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
// A queued approach to delivering events. This is generally
// faster than direct delivery and requires much less mutex
@@ -1073,16 +1116,56 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
for i, block := range chain {
headers[i] = block.Header()
- seals[i] = true
+ seals[i] = verifySeals
}
abort, results := bc.engine.VerifyHeaders(bc, headers, seals)
defer close(abort)
- // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
- senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
+ // Peek the error for the first block to decide the directing import logic
+ it := newInsertIterator(chain, results, bc.Validator())
- // Iterate over the blocks and insert when the verifier permits
- for i, block := range chain {
+ block, err := it.next()
+ switch {
+ // First block is pruned, insert as sidechain and reorg only if TD grows enough
+ case err == consensus.ErrPrunedAncestor:
+ return bc.insertSidechain(it)
+
+ // First block is future, shove it (and all children) to the future queue (unknown ancestor)
+ case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
+ for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) {
+ if err := bc.addFutureBlock(block); err != nil {
+ return it.index, events, coalescedLogs, err
+ }
+ block, err = it.next()
+ }
+ stats.queued += it.processed()
+ stats.ignored += it.remaining()
+
+ // If there are any still remaining, mark as ignored
+ return it.index, events, coalescedLogs, err
+
+ // First block (and state) is known
+ // 1. We did a roll-back, and should now do a re-import
+ // 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
+ // from the canonical chain, which has not been verified.
+ case err == ErrKnownBlock:
+ // Skip all known blocks that behind us
+ current := bc.CurrentBlock().NumberU64()
+
+ for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
+ stats.ignored++
+ block, err = it.next()
+ }
+ // Falls through to the block import
+
+ // Some other error occurred, abort
+ case err != nil:
+ stats.ignored += len(it.chain)
+ bc.reportBlock(block, nil, err)
+ return it.index, events, coalescedLogs, err
+ }
+ // No validation errors for the first block (or chain prefix skipped)
+ for ; block != nil && err == nil; block, err = it.next() {
// If the chain is terminating, stop processing blocks
if atomic.LoadInt32(&bc.procInterrupt) == 1 {
log.Debug("Premature abort during blocks processing")
@@ -1091,115 +1174,45 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
// If the header is a banned one, straight out abort
if BadHashes[block.Hash()] {
bc.reportBlock(block, nil, ErrBlacklistedHash)
- return i, events, coalescedLogs, ErrBlacklistedHash
+ return it.index, events, coalescedLogs, ErrBlacklistedHash
}
- // Wait for the block's verification to complete
- bstart := time.Now()
+ // Retrieve the parent block and it's state to execute on top
+ start := time.Now()
- err := <-results
- if err == nil {
- err = bc.Validator().ValidateBody(block)
- }
- switch {
- case err == ErrKnownBlock:
- // Block and state both already known. However if the current block is below
- // this number we did a rollback and we should reimport it nonetheless.
- if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
- stats.ignored++
- continue
- }
-
- case err == consensus.ErrFutureBlock:
- // Allow up to MaxFuture second in the future blocks. If this limit is exceeded
- // the chain is discarded and processed at a later time if given.
- max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
- if block.Time().Cmp(max) > 0 {
- return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
- }
- bc.futureBlocks.Add(block.Hash(), block)
- stats.queued++
- continue
-
- case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
- bc.futureBlocks.Add(block.Hash(), block)
- stats.queued++
- continue
-
- case err == consensus.ErrPrunedAncestor:
- // Block competing with the canonical chain, store in the db, but don't process
- // until the competitor TD goes above the canonical TD
- currentBlock := bc.CurrentBlock()
- localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
- externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
- if localTd.Cmp(externTd) > 0 {
- if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
- return i, events, coalescedLogs, err
- }
- continue
- }
- // Competitor chain beat canonical, gather all blocks from the common ancestor
- var winner []*types.Block
-
- parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
- for !bc.HasState(parent.Root()) {
- winner = append(winner, parent)
- parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
- }
- for j := 0; j < len(winner)/2; j++ {
- winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
- }
- // Import all the pruned blocks to make the state available
- bc.chainmu.Unlock()
- _, evs, logs, err := bc.insertChain(winner)
- bc.chainmu.Lock()
- events, coalescedLogs = evs, logs
-
- if err != nil {
- return i, events, coalescedLogs, err
- }
-
- case err != nil:
- bc.reportBlock(block, nil, err)
- return i, events, coalescedLogs, err
- }
- // Create a new statedb using the parent block and report an
- // error if it fails.
- var parent *types.Block
- if i == 0 {
+ parent := it.previous()
+ if parent == nil {
parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
- } else {
- parent = chain[i-1]
}
state, err := state.New(parent.Root(), bc.stateCache)
if err != nil {
- return i, events, coalescedLogs, err
+ return it.index, events, coalescedLogs, err
}
// Process block using the parent state as reference point.
receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig)
if err != nil {
bc.reportBlock(block, receipts, err)
- return i, events, coalescedLogs, err
+ return it.index, events, coalescedLogs, err
}
// Validate the state using the default validator
- err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas)
- if err != nil {
+ if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil {
bc.reportBlock(block, receipts, err)
- return i, events, coalescedLogs, err
+ return it.index, events, coalescedLogs, err
}
- proctime := time.Since(bstart)
+ proctime := time.Since(start)
// Write the block to the chain and get the status.
status, err := bc.WriteBlockWithState(block, receipts, state)
if err != nil {
- return i, events, coalescedLogs, err
+ return it.index, events, coalescedLogs, err
}
switch status {
case CanonStatTy:
- log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()),
- "txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart)))
+ log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(),
+ "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(),
+ "elapsed", common.PrettyDuration(time.Since(start)),
+ "root", block.Root())
coalescedLogs = append(coalescedLogs, logs...)
- blockInsertTimer.UpdateSince(bstart)
events = append(events, ChainEvent{block, block.Hash(), logs})
lastCanon = block
@@ -1207,78 +1220,153 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
bc.gcproc += proctime
case SideStatTy:
- log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
- common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
-
- blockInsertTimer.UpdateSince(bstart)
+ log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(),
+ "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
+ "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
+ "root", block.Root())
events = append(events, ChainSideEvent{block})
}
+ blockInsertTimer.UpdateSince(start)
stats.processed++
stats.usedGas += usedGas
cache, _ := bc.stateCache.TrieDB().Size()
- stats.report(chain, i, cache)
+ stats.report(chain, it.index, cache)
}
+ // Any blocks remaining here? The only ones we care about are the future ones
+ if block != nil && err == consensus.ErrFutureBlock {
+ if err := bc.addFutureBlock(block); err != nil {
+ return it.index, events, coalescedLogs, err
+ }
+ block, err = it.next()
+
+ for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() {
+ if err := bc.addFutureBlock(block); err != nil {
+ return it.index, events, coalescedLogs, err
+ }
+ stats.queued++
+ }
+ }
+ stats.ignored += it.remaining()
+
// Append a single chain head event if we've progressed the chain
if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
events = append(events, ChainHeadEvent{lastCanon})
}
- return 0, events, coalescedLogs, nil
+ return it.index, events, coalescedLogs, err
}
-// insertStats tracks and reports on block insertion.
-type insertStats struct {
- queued, processed, ignored int
- usedGas uint64
- lastIndex int
- startTime mclock.AbsTime
-}
-
-// statsReportLimit is the time limit during import and export after which we
-// always print out progress. This avoids the user wondering what's going on.
-const statsReportLimit = 8 * time.Second
-
-// report prints statistics if some number of blocks have been processed
-// or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
- // Fetch the timings for the batch
+// insertSidechain is called when an import batch hits upon a pruned ancestor
+// error, which happens when a sidechain with a sufficiently old fork-block is
+// found.
+//
+// The method writes all (header-and-body-valid) blocks to disk, then tries to
+// switch over to the new chain if the TD exceeded the current chain.
+func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
var (
- now = mclock.Now()
- elapsed = time.Duration(now) - time.Duration(st.startTime)
+ externTd *big.Int
+ current = bc.CurrentBlock().NumberU64()
)
- // If we're at the last block of the batch or report period reached, log
- if index == len(chain)-1 || elapsed >= statsReportLimit {
- var (
- end = chain[index]
- txs = countTransactions(chain[st.lastIndex : index+1])
- )
- context := []interface{}{
- "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
- "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
- "number", end.Number(), "hash", end.Hash(),
- }
- if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
- context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
- }
- context = append(context, []interface{}{"cache", cache}...)
+ // The first sidechain block error is already verified to be ErrPrunedAncestor.
+ // Since we don't import them here, we expect ErrUnknownAncestor for the remaining
+ // ones. Any other errors means that the block is invalid, and should not be written
+ // to disk.
+ block, err := it.current(), consensus.ErrPrunedAncestor
+ for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
+ // Check the canonical state root for that number
+ if number := block.NumberU64(); current >= number {
+ if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
+ // This is most likely a shadow-state attack. When a fork is imported into the
+ // database, and it eventually reaches a block height which is not pruned, we
+ // just found that the state already exist! This means that the sidechain block
+ // refers to a state which already exists in our canon chain.
+ //
+ // If left unchecked, we would now proceed importing the blocks, without actually
+ // having verified the state of the previous blocks.
+ log.Warn("Sidechain ghost-state attack detected", "number", block.NumberU64(), "sideroot", block.Root(), "canonroot", canonical.Root())
- if st.queued > 0 {
- context = append(context, []interface{}{"queued", st.queued}...)
+ // If someone legitimately side-mines blocks, they would still be imported as usual. However,
+ // we cannot risk writing unverified blocks to disk when they obviously target the pruning
+ // mechanism.
+ return it.index, nil, nil, errors.New("sidechain ghost-state attack")
+ }
}
- if st.ignored > 0 {
- context = append(context, []interface{}{"ignored", st.ignored}...)
+ if externTd == nil {
+ externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1)
}
- log.Info("Imported new chain segment", context...)
+ externTd = new(big.Int).Add(externTd, block.Difficulty())
- *st = insertStats{startTime: now, lastIndex: index + 1}
+ if !bc.HasBlock(block.Hash(), block.NumberU64()) {
+ start := time.Now()
+ if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
+ return it.index, nil, nil, err
+ }
+ log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
+ "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
+ "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
+ "root", block.Root())
+ }
}
-}
-
-func countTransactions(chain []*types.Block) (c int) {
- for _, b := range chain {
- c += len(b.Transactions())
+ // At this point, we've written all sidechain blocks to database. Loop ended
+ // either on some other error or all were processed. If there was some other
+ // error, we can ignore the rest of those blocks.
+ //
+ // If the externTd was larger than our local TD, we now need to reimport the previous
+ // blocks to regenerate the required state
+ localTd := bc.GetTd(bc.CurrentBlock().Hash(), current)
+ if localTd.Cmp(externTd) > 0 {
+ log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd)
+ return it.index, nil, nil, err
}
- return c
+ // Gather all the sidechain hashes (full blocks may be memory heavy)
+ var (
+ hashes []common.Hash
+ numbers []uint64
+ )
+ parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64())
+ for parent != nil && !bc.HasState(parent.Root) {
+ hashes = append(hashes, parent.Hash())
+ numbers = append(numbers, parent.Number.Uint64())
+
+ parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
+ }
+ if parent == nil {
+ return it.index, nil, nil, errors.New("missing parent")
+ }
+ // Import all the pruned blocks to make the state available
+ var (
+ blocks []*types.Block
+ memory common.StorageSize
+ )
+ for i := len(hashes) - 1; i >= 0; i-- {
+ // Append the next block to our batch
+ block := bc.GetBlock(hashes[i], numbers[i])
+
+ blocks = append(blocks, block)
+ memory += block.Size()
+
+ // If memory use grew too large, import and continue. Sadly we need to discard
+ // all raised events and logs from notifications since we're too heavy on the
+ // memory here.
+ if len(blocks) >= 2048 || memory > 64*1024*1024 {
+ log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64())
+ if _, _, _, err := bc.insertChain(blocks, false); err != nil {
+ return 0, nil, nil, err
+ }
+ blocks, memory = blocks[:0], 0
+
+ // If the chain is terminating, stop processing blocks
+ if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+ log.Debug("Premature abort during blocks processing")
+ return 0, nil, nil, nil
+ }
+ }
+ }
+ if len(blocks) > 0 {
+ log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64())
+ return bc.insertChain(blocks, false)
+ }
+ return 0, nil, nil, nil
}
// reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them
@@ -1453,8 +1541,10 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e
bc.addBadBlock(block)
var receiptString string
- for _, receipt := range receipts {
- receiptString += fmt.Sprintf("\t%v\n", receipt)
+ for i, receipt := range receipts {
+ receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n",
+ i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(),
+ receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState)
}
log.Error(fmt.Sprintf(`
########## BAD BLOCK #########
diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go
new file mode 100644
index 0000000000..70bea35445
--- /dev/null
+++ b/core/blockchain_insert.go
@@ -0,0 +1,143 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package core
+
+import (
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/mclock"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+// insertStats tracks and reports on block insertion.
+type insertStats struct {
+ queued, processed, ignored int
+ usedGas uint64
+ lastIndex int
+ startTime mclock.AbsTime
+}
+
+// statsReportLimit is the time limit during import and export after which we
+// always print out progress. This avoids the user wondering what's going on.
+const statsReportLimit = 8 * time.Second
+
+// report prints statistics if some number of blocks have been processed
+// or more than a few seconds have passed since the last message.
+func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
+ // Fetch the timings for the batch
+ var (
+ now = mclock.Now()
+ elapsed = time.Duration(now) - time.Duration(st.startTime)
+ )
+ // If we're at the last block of the batch or report period reached, log
+ if index == len(chain)-1 || elapsed >= statsReportLimit {
+ // Count the number of transactions in this segment
+ var txs int
+ for _, block := range chain[st.lastIndex : index+1] {
+ txs += len(block.Transactions())
+ }
+ end := chain[index]
+
+ // Assemble the log context and send it to the logger
+ context := []interface{}{
+ "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
+ "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+ "number", end.Number(), "hash", end.Hash(),
+ }
+ if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
+ context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
+ }
+ context = append(context, []interface{}{"cache", cache}...)
+
+ if st.queued > 0 {
+ context = append(context, []interface{}{"queued", st.queued}...)
+ }
+ if st.ignored > 0 {
+ context = append(context, []interface{}{"ignored", st.ignored}...)
+ }
+ log.Info("Imported new chain segment", context...)
+
+ // Bump the stats reported to the next section
+ *st = insertStats{startTime: now, lastIndex: index + 1}
+ }
+}
+
+// insertIterator is a helper to assist during chain import.
+type insertIterator struct {
+ chain types.Blocks
+ results <-chan error
+ index int
+ validator Validator
+}
+
+// newInsertIterator creates a new iterator based on the given blocks, which are
+// assumed to be a contiguous chain.
+func newInsertIterator(chain types.Blocks, results <-chan error, validator Validator) *insertIterator {
+ return &insertIterator{
+ chain: chain,
+ results: results,
+ index: -1,
+ validator: validator,
+ }
+}
+
+// next returns the next block in the iterator, along with any potential validation
+// error for that block. When the end is reached, it will return (nil, nil).
+func (it *insertIterator) next() (*types.Block, error) {
+ if it.index+1 >= len(it.chain) {
+ it.index = len(it.chain)
+ return nil, nil
+ }
+ it.index++
+ if err := <-it.results; err != nil {
+ return it.chain[it.index], err
+ }
+ return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
+}
+
+// current returns the current block that's being processed.
+func (it *insertIterator) current() *types.Block {
+ if it.index < 0 || it.index+1 >= len(it.chain) {
+ return nil
+ }
+ return it.chain[it.index]
+}
+
+// previous returns the previous block was being processed, or nil
+func (it *insertIterator) previous() *types.Block {
+ if it.index < 1 {
+ return nil
+ }
+ return it.chain[it.index-1]
+}
+
+// first returns the first block in the it.
+func (it *insertIterator) first() *types.Block {
+ return it.chain[0]
+}
+
+// remaining returns the number of remaining blocks.
+func (it *insertIterator) remaining() int {
+ return len(it.chain) - it.index
+}
+
+// processed returns the number of processed blocks.
+func (it *insertIterator) processed() int {
+ return it.index + 1
+}
diff --git a/core/blockchain_test.go b/core/blockchain_test.go
index aef810050b..5ab29e205f 100644
--- a/core/blockchain_test.go
+++ b/core/blockchain_test.go
@@ -579,11 +579,11 @@ func testInsertNonceError(t *testing.T, full bool) {
blockchain.hc.engine = blockchain.engine
failRes, err = blockchain.InsertHeaderChain(headers, 1)
}
- // Check that the returned error indicates the failure.
+ // Check that the returned error indicates the failure
if failRes != failAt {
- t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
+ t.Errorf("test %d: failure (%v) index mismatch: have %d, want %d", i, err, failRes, failAt)
}
- // Check that all no blocks after the failing block have been inserted.
+ // Check that all blocks after the failing block have been inserted
for j := 0; j < i-failAt; j++ {
if full {
if block := blockchain.GetBlockByNumber(failNum + uint64(j)); block != nil {
@@ -1345,7 +1345,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
t.Fatalf("failed to insert shared chain: %v", err)
}
if _, err := chain.InsertChain(original); err != nil {
- t.Fatalf("failed to insert shared chain: %v", err)
+ t.Fatalf("failed to insert original chain: %v", err)
}
// Ensure that the state associated with the forking point is pruned away
if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {
diff --git a/core/chain_makers.go b/core/chain_makers.go
index 0bc453fdf3..e3a5537a40 100644
--- a/core/chain_makers.go
+++ b/core/chain_makers.go
@@ -33,12 +33,11 @@ import (
// BlockGen creates blocks for testing.
// See GenerateChain for a detailed explanation.
type BlockGen struct {
- i int
- parent *types.Block
- chain []*types.Block
- chainReader consensus.ChainReader
- header *types.Header
- statedb *state.StateDB
+ i int
+ parent *types.Block
+ chain []*types.Block
+ header *types.Header
+ statedb *state.StateDB
gasPool *GasPool
txs []*types.Transaction
@@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) {
// For index -1, PrevBlock returns the parent block given to GenerateChain.
func (b *BlockGen) PrevBlock(index int) *types.Block {
if index >= b.i {
- panic("block index out of range")
+ panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
}
if index == -1 {
return b.parent
@@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) {
if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
panic("block time out of range")
}
- b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header())
+ chainreader := &fakeChainReader{config: b.config}
+ b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header())
}
// GenerateChain creates a chain of n blocks. The first block's
@@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
config = params.TestChainConfig
}
blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+ chainreader := &fakeChainReader{config: config}
genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
- // TODO(karalabe): This is needed for clique, which depends on multiple blocks.
- // It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
- blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
- defer blockchain.Stop()
-
- b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
- b.header = makeHeader(b.chainReader, parent, statedb, b.engine)
+ b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
+ b.header = makeHeader(chainreader, parent, statedb, b.engine)
// Mutate the state and block according to any hard-fork specs
if daoBlock := config.DAOForkBlock; daoBlock != nil {
@@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
}
if b.engine != nil {
// Finalize and seal the block
- block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+ block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
// Write state changes to db
root, err := statedb.Commit(config.IsEIP158(b.header.Number))
@@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd
})
return blocks
}
+
+type fakeChainReader struct {
+ config *params.ChainConfig
+ genesis *types.Block
+}
+
+// Config returns the chain configuration.
+func (cr *fakeChainReader) Config() *params.ChainConfig {
+ return cr.config
+}
+
+func (cr *fakeChainReader) CurrentHeader() *types.Header { return nil }
+func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil }
+func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil }
diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go
index 6660e17de1..491a125c65 100644
--- a/core/rawdb/accessors_chain.go
+++ b/core/rawdb/accessors_chain.go
@@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
}
}
+// HasReceipts verifies the existence of all the transaction receipts belonging
+// to a block.
+func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
+ if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
+ return false
+ }
+ return true
+}
+
// ReadReceipts retrieves all the transaction receipts belonging to a block.
func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
// Retrieve the flattened receipt slice
diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go
index 514328e87b..3b6e6548d3 100644
--- a/core/rawdb/accessors_metadata.go
+++ b/core/rawdb/accessors_metadata.go
@@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte {
return data
}
-// WritePreimages writes the provided set of preimages to the database. `number` is the
-// current block number, and is used for debug messages only.
-func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) {
+// WritePreimages writes the provided set of preimages to the database.
+func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) {
for hash, preimage := range preimages {
if err := db.Put(preimageKey(hash), preimage); err != nil {
log.Crit("Failed to store trie preimage", "err", err)
diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go
index ef597ef309..8a9921ef40 100644
--- a/core/rawdb/schema.go
+++ b/core/rawdb/schema.go
@@ -35,7 +35,7 @@ var (
// headBlockKey tracks the latest know full block's hash.
headBlockKey = []byte("LastBlock")
- // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync.
+ // headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
headFastBlockKey = []byte("LastFast")
// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
diff --git a/core/state/database.go b/core/state/database.go
index c1b630991c..f6ea144b9b 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -72,13 +72,19 @@ type Trie interface {
}
// NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains cached trie nodes in memory. The pool is an optional
-// intermediate trie-node memory pool between the low level storage layer and the
-// high level trie abstraction.
+// concurrent use and retains a few recent expanded trie nodes in memory. To keep
+// more historical state in memory, use the NewDatabaseWithCache constructor.
func NewDatabase(db ethdb.Database) Database {
+ return NewDatabaseWithCache(db, 0)
+}
+
+// NewDatabase creates a backing store for state. The returned database is safe for
+// concurrent use and retains both a few recent expanded trie nodes in memory, as
+// well as a lot of collapsed RLP trie nodes in a large memory cache.
+func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{
- db: trie.NewDatabase(db),
+ db: trie.NewDatabaseWithCache(db, cache),
codeSizeCache: csc,
}
}
diff --git a/core/state/statedb.go b/core/state/statedb.go
index f0d7cdb6eb..76e67d839d 100644
--- a/core/state/statedb.go
+++ b/core/state/statedb.go
@@ -22,7 +22,6 @@ import (
"fmt"
"math/big"
"sort"
- "sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@@ -87,8 +86,6 @@ type StateDB struct {
journal *journal
validRevisions []revision
nextRevisionId int
-
- lock sync.Mutex
}
// Create a new state from a given trie.
@@ -496,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common
// Copy creates a deep, independent copy of the state.
// Snapshots of the copied state cannot be applied to the copy.
func (self *StateDB) Copy() *StateDB {
- self.lock.Lock()
- defer self.lock.Unlock()
-
// Copy all the basic fields, initialize the memory ones
state := &StateDB{
db: self.db,
diff --git a/core/tx_pool.go b/core/tx_pool.go
index f6da5da2a7..fc35d1f243 100644
--- a/core/tx_pool.go
+++ b/core/tx_pool.go
@@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
// addTxsLocked attempts to queue a batch of transactions if they are valid,
// whilst assuming the transaction pool lock is already held.
func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
- // Add the batch of transaction, tracking the accepted ones
+ // Add the batch of transactions, tracking the accepted ones
dirty := make(map[common.Address]struct{})
errs := make([]error, len(txs))
diff --git a/core/types/block.go b/core/types/block.go
index 8a21bba1e7..9d11f60d83 100644
--- a/core/types/block.go
+++ b/core/types/block.go
@@ -81,8 +81,8 @@ type Header struct {
GasUsed uint64 `json:"gasUsed" gencodec:"required"`
Time *big.Int `json:"timestamp" gencodec:"required"`
Extra []byte `json:"extraData" gencodec:"required"`
- MixDigest common.Hash `json:"mixHash" gencodec:"required"`
- Nonce BlockNonce `json:"nonce" gencodec:"required"`
+ MixDigest common.Hash `json:"mixHash"`
+ Nonce BlockNonce `json:"nonce"`
}
// field type overrides for gencodec
diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go
index 1b92cd9cf4..59a1c9c439 100644
--- a/core/types/gen_header_json.go
+++ b/core/types/gen_header_json.go
@@ -13,6 +13,7 @@ import (
var _ = (*headerMarshaling)(nil)
+// MarshalJSON marshals as JSON.
func (h Header) MarshalJSON() ([]byte, error) {
type Header struct {
ParentHash common.Hash `json:"parentHash" gencodec:"required"`
@@ -28,8 +29,8 @@ func (h Header) MarshalJSON() ([]byte, error) {
GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
Extra hexutil.Bytes `json:"extraData" gencodec:"required"`
- MixDigest common.Hash `json:"mixHash" gencodec:"required"`
- Nonce BlockNonce `json:"nonce" gencodec:"required"`
+ MixDigest common.Hash `json:"mixHash"`
+ Nonce BlockNonce `json:"nonce"`
Hash common.Hash `json:"hash"`
}
var enc Header
@@ -52,6 +53,7 @@ func (h Header) MarshalJSON() ([]byte, error) {
return json.Marshal(&enc)
}
+// UnmarshalJSON unmarshals from JSON.
func (h *Header) UnmarshalJSON(input []byte) error {
type Header struct {
ParentHash *common.Hash `json:"parentHash" gencodec:"required"`
@@ -67,8 +69,8 @@ func (h *Header) UnmarshalJSON(input []byte) error {
GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
Extra *hexutil.Bytes `json:"extraData" gencodec:"required"`
- MixDigest *common.Hash `json:"mixHash" gencodec:"required"`
- Nonce *BlockNonce `json:"nonce" gencodec:"required"`
+ MixDigest *common.Hash `json:"mixHash"`
+ Nonce *BlockNonce `json:"nonce"`
}
var dec Header
if err := json.Unmarshal(input, &dec); err != nil {
@@ -126,13 +128,11 @@ func (h *Header) UnmarshalJSON(input []byte) error {
return errors.New("missing required field 'extraData' for Header")
}
h.Extra = *dec.Extra
- if dec.MixDigest == nil {
- return errors.New("missing required field 'mixHash' for Header")
+ if dec.MixDigest != nil {
+ h.MixDigest = *dec.MixDigest
}
- h.MixDigest = *dec.MixDigest
- if dec.Nonce == nil {
- return errors.New("missing required field 'nonce' for Header")
+ if dec.Nonce != nil {
+ h.Nonce = *dec.Nonce
}
- h.Nonce = *dec.Nonce
return nil
}
diff --git a/core/vm/instructions.go b/core/vm/instructions.go
index b7c3ca5323..6696c6e3d1 100644
--- a/core/vm/instructions.go
+++ b/core/vm/instructions.go
@@ -544,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract,
// this account should be regarded as a non-existent account and zero should be returned.
func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
slot := stack.peek()
- slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes())
+ address := common.BigToAddress(slot)
+ if interpreter.evm.StateDB.Empty(address) {
+ slot.SetUint64(0)
+ } else {
+ slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes())
+ }
return nil, nil
}
diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go
index cba7c7a0ef..2ea7535a79 100644
--- a/core/vm/logger_test.go
+++ b/core/vm/logger_test.go
@@ -46,7 +46,7 @@ type dummyStatedb struct {
state.StateDB
}
-func (dummyStatedb) GetRefund() uint64 { return 1337 }
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
func TestStoreCapture(t *testing.T) {
var (
diff --git a/eth/api.go b/eth/api.go
index 3ec3afb81e..816b9cd335 100644
--- a/eth/api.go
+++ b/eth/api.go
@@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
}
+ triedb := api.eth.BlockChain().StateCache().TrieDB()
- oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+ oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0)
if err != nil {
return nil, err
}
- newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
+ newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0)
if err != nil {
return nil, err
}
-
diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}))
iter := trie.NewIterator(diff)
diff --git a/eth/api_tracer.go b/eth/api_tracer.go
index 80552ada8c..2ebbcc5fd9 100644
--- a/eth/api_tracer.go
+++ b/eth/api_tracer.go
@@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
// Ensure we have a valid starting state before doing any work
origin := start.NumberU64()
- database := state.NewDatabase(api.eth.ChainDb())
+ database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis
if number := start.NumberU64(); number > 0 {
start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
@@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
}
// Otherwise try to reexec blocks until we find a state or reach our limit
origin := block.NumberU64()
- database := state.NewDatabase(api.eth.ChainDb())
+ database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16)
for i := uint64(0); i < reexec; i++ {
block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
diff --git a/eth/backend.go b/eth/backend.go
index b555b064ad..4721408425 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
EWASMInterpreter: config.EWASMInterpreter,
EVMInterpreter: config.EVMInterpreter,
}
- cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
+ cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout}
)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
if err != nil {
diff --git a/eth/config.go b/eth/config.go
index efbaafb6a2..601f4735e3 100644
--- a/eth/config.go
+++ b/eth/config.go
@@ -43,15 +43,16 @@ var DefaultConfig = Config{
DatasetsInMem: 1,
DatasetsOnDisk: 2,
},
- NetworkId: 1,
- LightPeers: 100,
- DatabaseCache: 768,
- TrieCache: 256,
- TrieTimeout: 60 * time.Minute,
- MinerGasFloor: 8000000,
- MinerGasCeil: 8000000,
- MinerGasPrice: big.NewInt(params.GWei),
- MinerRecommit: 3 * time.Second,
+ NetworkId: 1,
+ LightPeers: 100,
+ DatabaseCache: 512,
+ TrieCleanCache: 256,
+ TrieDirtyCache: 256,
+ TrieTimeout: 60 * time.Minute,
+ MinerGasFloor: 8000000,
+ MinerGasCeil: 8000000,
+ MinerGasPrice: big.NewInt(params.GWei),
+ MinerRecommit: 3 * time.Second,
TxPool: core.DefaultTxPoolConfig,
GPO: gasprice.Config{
@@ -94,7 +95,8 @@ type Config struct {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
- TrieCache int
+ TrieCleanCache int
+ TrieDirtyCache int
TrieTimeout time.Duration
// Mining-related options
@@ -122,7 +124,7 @@ type Config struct {
// Miscellaneous options
DocRoot string `toml:"-"`
- // Type of the EWASM interpreter ("" for detault)
+ // Type of the EWASM interpreter ("" for default)
EWASMInterpreter string
// Type of the EVM interpreter ("" for default)
EVMInterpreter string
diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go
index f01a8fdbd8..f81a5cbac2 100644
--- a/eth/downloader/downloader.go
+++ b/eth/downloader/downloader.go
@@ -181,6 +181,9 @@ type BlockChain interface {
// HasBlock verifies a block's presence in the local chain.
HasBlock(common.Hash, uint64) bool
+ // HasFastBlock verifies a fast block's presence in the local chain.
+ HasFastBlock(common.Hash, uint64) bool
+
// GetBlockByHash retrieves a block from the local chain.
GetBlockByHash(common.Hash) *types.Block
@@ -430,7 +433,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
}
height := latest.Number.Uint64()
- origin, err := d.findAncestor(p, height)
+ origin, err := d.findAncestor(p, latest)
if err != nil {
return err
}
@@ -587,41 +590,86 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) {
}
}
+// calculateRequestSpan calculates what headers to request from a peer when trying to determine the
+// common ancestor.
+// It returns parameters to be used for peer.RequestHeadersByNumber:
+// from - starting block number
+// count - number of headers to request
+// skip - number of headers to skip
+// and also returns 'max', the last block which is expected to be returned by the remote peers,
+// given the (from,count,skip)
+func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) {
+ var (
+ from int
+ count int
+ MaxCount = MaxHeaderFetch / 16
+ )
+ // requestHead is the highest block that we will ask for. If requestHead is not offset,
+ // the highest block that we will get is 16 blocks back from head, which means we
+ // will fetch 14 or 15 blocks unnecessarily in the case the height difference
+ // between us and the peer is 1-2 blocks, which is most common
+ requestHead := int(remoteHeight) - 1
+ if requestHead < 0 {
+ requestHead = 0
+ }
+ // requestBottom is the lowest block we want included in the query
+ // Ideally, we want to include just below own head
+ requestBottom := int(localHeight - 1)
+ if requestBottom < 0 {
+ requestBottom = 0
+ }
+ totalSpan := requestHead - requestBottom
+ span := 1 + totalSpan/MaxCount
+ if span < 2 {
+ span = 2
+ }
+ if span > 16 {
+ span = 16
+ }
+
+ count = 1 + totalSpan/span
+ if count > MaxCount {
+ count = MaxCount
+ }
+ if count < 2 {
+ count = 2
+ }
+ from = requestHead - (count-1)*span
+ if from < 0 {
+ from = 0
+ }
+ max := from + (count-1)*span
+ return int64(from), count, span - 1, uint64(max)
+}
+
// findAncestor tries to locate the common ancestor link of the local chain and
// a remote peers blockchain. In the general case when our node was in sync and
// on the correct chain, checking the top N links should already get us a match.
// In the rare scenario when we ended up on a long reorganisation (i.e. none of
// the head links match), we do a binary search to find the common ancestor.
-func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) {
+func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) {
// Figure out the valid ancestor range to prevent rewrite attacks
- floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
+ var (
+ floor = int64(-1)
+ localHeight uint64
+ remoteHeight = remoteHeader.Number.Uint64()
+ )
+ switch d.mode {
+ case FullSync:
+ localHeight = d.blockchain.CurrentBlock().NumberU64()
+ case FastSync:
+ localHeight = d.blockchain.CurrentFastBlock().NumberU64()
+ default:
+ localHeight = d.lightchain.CurrentHeader().Number.Uint64()
+ }
+ p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight)
+ if localHeight >= MaxForkAncestry {
+ floor = int64(localHeight - MaxForkAncestry)
+ }
+ from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight)
- if d.mode == FullSync {
- ceil = d.blockchain.CurrentBlock().NumberU64()
- } else if d.mode == FastSync {
- ceil = d.blockchain.CurrentFastBlock().NumberU64()
- }
- if ceil >= MaxForkAncestry {
- floor = int64(ceil - MaxForkAncestry)
- }
- p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
-
- // Request the topmost blocks to short circuit binary ancestor lookup
- head := ceil
- if head > height {
- head = height
- }
- from := int64(head) - int64(MaxHeaderFetch)
- if from < 0 {
- from = 0
- }
- // Span out with 15 block gaps into the future to catch bad head reports
- limit := 2 * MaxHeaderFetch / 16
- count := 1 + int((int64(ceil)-from)/16)
- if count > limit {
- count = limit
- }
- go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false)
+ p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip)
+ go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false)
// Wait for the remote response to the head fetch
number, hash := uint64(0), common.Hash{}
@@ -647,9 +695,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
return 0, errEmptyHeaderSet
}
// Make sure the peer's reply conforms to the request
- for i := 0; i < len(headers); i++ {
- if number := headers[i].Number.Int64(); number != from+int64(i)*16 {
- p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number)
+ for i, header := range headers {
+ expectNumber := from + int64(i)*int64((skip+1))
+ if number := header.Number.Int64(); number != expectNumber {
+ p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
return 0, errInvalidChain
}
}
@@ -657,20 +706,24 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
finished = true
for i := len(headers) - 1; i >= 0; i-- {
// Skip any headers that underflow/overflow our requested set
- if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil {
+ if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max {
continue
}
// Otherwise check if we already know the header or not
h := headers[i].Hash()
n := headers[i].Number.Uint64()
- if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) {
- number, hash = n, h
- // If every header is known, even future ones, the peer straight out lied about its head
- if number > height && i == limit-1 {
- p.log.Warn("Lied about chain head", "reported", height, "found", number)
- return 0, errStallingPeer
- }
+ var known bool
+ switch d.mode {
+ case FullSync:
+ known = d.blockchain.HasBlock(h, n)
+ case FastSync:
+ known = d.blockchain.HasFastBlock(h, n)
+ default:
+ known = d.lightchain.HasHeader(h, n)
+ }
+ if known {
+ number, hash = n, h
break
}
}
@@ -694,10 +747,12 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
return number, nil
}
// Ancestor not found, we need to binary search over our chain
- start, end := uint64(0), head
+ start, end := uint64(0), remoteHeight
if floor > 0 {
start = uint64(floor)
}
+ p.log.Trace("Binary searching for common ancestor", "start", start, "end", end)
+
for start+1 < end {
// Split our chain interval in two, and request the hash to cross check
check := (start + end) / 2
@@ -730,7 +785,17 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
// Modify the search interval based on the response
h := headers[0].Hash()
n := headers[0].Number.Uint64()
- if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) {
+
+ var known bool
+ switch d.mode {
+ case FullSync:
+ known = d.blockchain.HasBlock(h, n)
+ case FastSync:
+ known = d.blockchain.HasFastBlock(h, n)
+ default:
+ known = d.lightchain.HasHeader(h, n)
+ }
+ if !known {
end = check
break
}
@@ -740,6 +805,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
return 0, errBadPeer
}
start = check
+ hash = h
case <-timeout:
p.log.Debug("Waiting for search header timed out", "elapsed", ttl)
diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go
index dad626e89a..1a42965d39 100644
--- a/eth/downloader/downloader_test.go
+++ b/eth/downloader/downloader_test.go
@@ -20,27 +20,20 @@ import (
"errors"
"fmt"
"math/big"
+ "strings"
"sync"
"sync/atomic"
"testing"
"time"
+ ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/consensus/ethash"
- "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
- "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
- "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
-var (
- testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
- testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
-)
-
// Reduce some of the parameters to make the tester faster.
func init() {
MaxForkAncestry = uint64(10000)
@@ -55,6 +48,7 @@ type downloadTester struct {
genesis *types.Block // Genesis blocks used by the tester and peers
stateDb ethdb.Database // Database used by the tester for syncing from peers
peerDb ethdb.Database // Database of the peers containing all data
+ peers map[string]*downloadTesterPeer
ownHashes []common.Hash // Hash chain belonging to the tester
ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester
@@ -62,129 +56,27 @@ type downloadTester struct {
ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester
ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain
- peerHashes map[string][]common.Hash // Hash chain belonging to different test peers
- peerHeaders map[string]map[common.Hash]*types.Header // Headers belonging to different test peers
- peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers
- peerReceipts map[string]map[common.Hash]types.Receipts // Receipts belonging to different test peers
- peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains
-
- peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return
-
lock sync.RWMutex
}
// newTester creates a new downloader test mocker.
func newTester() *downloadTester {
- testdb := ethdb.NewMemDatabase()
- genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000))
-
tester := &downloadTester{
- genesis: genesis,
- peerDb: testdb,
- ownHashes: []common.Hash{genesis.Hash()},
- ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()},
- ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis},
- ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil},
- ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()},
- peerHashes: make(map[string][]common.Hash),
- peerHeaders: make(map[string]map[common.Hash]*types.Header),
- peerBlocks: make(map[string]map[common.Hash]*types.Block),
- peerReceipts: make(map[string]map[common.Hash]types.Receipts),
- peerChainTds: make(map[string]map[common.Hash]*big.Int),
- peerMissingStates: make(map[string]map[common.Hash]bool),
+ genesis: testGenesis,
+ peerDb: testDB,
+ peers: make(map[string]*downloadTesterPeer),
+ ownHashes: []common.Hash{testGenesis.Hash()},
+ ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()},
+ ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis},
+ ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil},
+ ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()},
}
tester.stateDb = ethdb.NewMemDatabase()
- tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00})
-
+ tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00})
tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer)
-
return tester
}
-// makeChain creates a chain of n blocks starting at and including parent.
-// the returned hash chain is ordered head->parent. In addition, every 3rd block
-// contains a transaction and every 5th an uncle to allow testing correct block
-// reassembly.
-func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts, heavy bool) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) {
- // Generate the block chain
- blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), dl.peerDb, n, func(i int, block *core.BlockGen) {
- block.SetCoinbase(common.Address{seed})
-
- // If a heavy chain is requested, delay blocks to raise difficulty
- if heavy {
- block.OffsetTime(-1)
- }
- // If the block number is multiple of 3, send a bonus transaction to the miner
- if parent == dl.genesis && i%3 == 0 {
- signer := types.MakeSigner(params.TestChainConfig, block.Number())
- tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey)
- if err != nil {
- panic(err)
- }
- block.AddTx(tx)
- }
- // If the block number is a multiple of 5, add a bonus uncle to the block
- if i > 0 && i%5 == 0 {
- block.AddUncle(&types.Header{
- ParentHash: block.PrevBlock(i - 1).Hash(),
- Number: big.NewInt(block.Number().Int64() - 1),
- })
- }
- })
- // Convert the block-chain into a hash-chain and header/block maps
- hashes := make([]common.Hash, n+1)
- hashes[len(hashes)-1] = parent.Hash()
-
- headerm := make(map[common.Hash]*types.Header, n+1)
- headerm[parent.Hash()] = parent.Header()
-
- blockm := make(map[common.Hash]*types.Block, n+1)
- blockm[parent.Hash()] = parent
-
- receiptm := make(map[common.Hash]types.Receipts, n+1)
- receiptm[parent.Hash()] = parentReceipts
-
- for i, b := range blocks {
- hashes[len(hashes)-i-2] = b.Hash()
- headerm[b.Hash()] = b.Header()
- blockm[b.Hash()] = b
- receiptm[b.Hash()] = receipts[i]
- }
- return hashes, headerm, blockm, receiptm
-}
-
-// makeChainFork creates two chains of length n, such that h1[:f] and
-// h2[:f] are different but have a common suffix of length n-f.
-func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentReceipts types.Receipts, balanced bool) ([]common.Hash, []common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]*types.Block, map[common.Hash]types.Receipts, map[common.Hash]types.Receipts) {
- // Create the common suffix
- hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false)
-
- // Create the forks, making the second heavier if non balanced forks were requested
- hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false)
- hashes1 = append(hashes1, hashes[1:]...)
-
- heavy := false
- if !balanced {
- heavy = true
- }
- hashes2, headers2, blocks2, receipts2 := dl.makeChain(f, 2, blocks[hashes[0]], receipts[hashes[0]], heavy)
- hashes2 = append(hashes2, hashes[1:]...)
-
- for hash, header := range headers {
- headers1[hash] = header
- headers2[hash] = header
- }
- for hash, block := range blocks {
- blocks1[hash] = block
- blocks2[hash] = block
- }
- for hash, receipt := range receipts {
- receipts1[hash] = receipt
- receipts2[hash] = receipt
- }
- return hashes1, hashes2, headers1, headers2, blocks1, blocks2, receipts1, receipts2
-}
-
// terminate aborts any operations on the embedded downloader and releases all
// held resources.
func (dl *downloadTester) terminate() {
@@ -194,13 +86,10 @@ func (dl *downloadTester) terminate() {
// sync starts synchronizing with a remote peer, blocking until it completes.
func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error {
dl.lock.RLock()
- hash := dl.peerHashes[id][0]
+ hash := dl.peers[id].chain.headBlock().Hash()
// If no particular TD was requested, load from the peer's blockchain
if td == nil {
- td = big.NewInt(1)
- if diff, ok := dl.peerChainTds[id][hash]; ok {
- td = diff
- }
+ td = dl.peers[id].chain.td(hash)
}
dl.lock.RUnlock()
@@ -226,6 +115,15 @@ func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool {
return dl.GetBlockByHash(hash) != nil
}
+// HasFastBlock checks if a block is present in the testers canonical chain.
+func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool {
+ dl.lock.RLock()
+ defer dl.lock.RUnlock()
+
+ _, ok := dl.ownReceipts[hash]
+ return ok
+}
+
// GetHeader retrieves a header from the testers canonical chain.
func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header {
dl.lock.RLock()
@@ -302,7 +200,7 @@ func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int {
}
// InsertHeaderChain injects a new batch of headers into the simulated chain.
-func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (int, error) {
+func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (i int, err error) {
dl.lock.Lock()
defer dl.lock.Unlock()
@@ -331,7 +229,7 @@ func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq i
}
// InsertChain injects a new batch of blocks into the simulated chain.
-func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) {
+func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) {
dl.lock.Lock()
defer dl.lock.Unlock()
@@ -346,6 +244,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) {
dl.ownHeaders[block.Hash()] = block.Header()
}
dl.ownBlocks[block.Hash()] = block
+ dl.ownReceipts[block.Hash()] = make(types.Receipts, 0)
dl.stateDb.Put(block.Root().Bytes(), []byte{0x00})
dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty())
}
@@ -353,7 +252,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) {
}
// InsertReceiptChain injects a new batch of receipts into the simulated chain.
-func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (int, error) {
+func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (i int, err error) {
dl.lock.Lock()
defer dl.lock.Unlock()
@@ -387,60 +286,13 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) {
}
// newPeer registers a new block download source into the downloader.
-func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error {
- return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0)
-}
-
-// newSlowPeer registers a new block download source into the downloader, with a
-// specific delay time on processing the network packets sent to it, simulating
-// potentially slow network IO.
-func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts, delay time.Duration) error {
+func (dl *downloadTester) newPeer(id string, version int, chain *testChain) error {
dl.lock.Lock()
defer dl.lock.Unlock()
- var err = dl.downloader.RegisterPeer(id, version, &downloadTesterPeer{dl: dl, id: id, delay: delay})
- if err == nil {
- // Assign the owned hashes, headers and blocks to the peer (deep copy)
- dl.peerHashes[id] = make([]common.Hash, len(hashes))
- copy(dl.peerHashes[id], hashes)
-
- dl.peerHeaders[id] = make(map[common.Hash]*types.Header)
- dl.peerBlocks[id] = make(map[common.Hash]*types.Block)
- dl.peerReceipts[id] = make(map[common.Hash]types.Receipts)
- dl.peerChainTds[id] = make(map[common.Hash]*big.Int)
- dl.peerMissingStates[id] = make(map[common.Hash]bool)
-
- genesis := hashes[len(hashes)-1]
- if header := headers[genesis]; header != nil {
- dl.peerHeaders[id][genesis] = header
- dl.peerChainTds[id][genesis] = header.Difficulty
- }
- if block := blocks[genesis]; block != nil {
- dl.peerBlocks[id][genesis] = block
- dl.peerChainTds[id][genesis] = block.Difficulty()
- }
-
- for i := len(hashes) - 2; i >= 0; i-- {
- hash := hashes[i]
-
- if header, ok := headers[hash]; ok {
- dl.peerHeaders[id][hash] = header
- if _, ok := dl.peerHeaders[id][header.ParentHash]; ok {
- dl.peerChainTds[id][hash] = new(big.Int).Add(header.Difficulty, dl.peerChainTds[id][header.ParentHash])
- }
- }
- if block, ok := blocks[hash]; ok {
- dl.peerBlocks[id][hash] = block
- if _, ok := dl.peerBlocks[id][block.ParentHash()]; ok {
- dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][block.ParentHash()])
- }
- }
- if receipt, ok := receipts[hash]; ok {
- dl.peerReceipts[id][hash] = receipt
- }
- }
- }
- return err
+ peer := &downloadTesterPeer{dl: dl, id: id, chain: chain}
+ dl.peers[id] = peer
+ return dl.downloader.RegisterPeer(id, version, peer)
}
// dropPeer simulates a hard peer removal from the connection pool.
@@ -448,89 +300,48 @@ func (dl *downloadTester) dropPeer(id string) {
dl.lock.Lock()
defer dl.lock.Unlock()
- delete(dl.peerHashes, id)
- delete(dl.peerHeaders, id)
- delete(dl.peerBlocks, id)
- delete(dl.peerChainTds, id)
-
+ delete(dl.peers, id)
dl.downloader.UnregisterPeer(id)
}
type downloadTesterPeer struct {
- dl *downloadTester
- id string
- delay time.Duration
- lock sync.RWMutex
-}
-
-// setDelay is a thread safe setter for the network delay value.
-func (dlp *downloadTesterPeer) setDelay(delay time.Duration) {
- dlp.lock.Lock()
- defer dlp.lock.Unlock()
-
- dlp.delay = delay
-}
-
-// waitDelay is a thread safe way to sleep for the configured time.
-func (dlp *downloadTesterPeer) waitDelay() {
- dlp.lock.RLock()
- delay := dlp.delay
- dlp.lock.RUnlock()
-
- time.Sleep(delay)
+ dl *downloadTester
+ id string
+ lock sync.RWMutex
+ chain *testChain
+ missingStates map[common.Hash]bool // State entries that fast sync should not return
}
// Head constructs a function to retrieve a peer's current head hash
// and total difficulty.
func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) {
- dlp.dl.lock.RLock()
- defer dlp.dl.lock.RUnlock()
-
- return dlp.dl.peerHashes[dlp.id][0], nil
+ b := dlp.chain.headBlock()
+ return b.Hash(), dlp.chain.td(b.Hash())
}
// RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed
// origin; associated with a particular peer in the download tester. The returned
// function can be used to retrieve batches of headers from the particular peer.
func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error {
- // Find the canonical number of the hash
- dlp.dl.lock.RLock()
- number := uint64(0)
- for num, hash := range dlp.dl.peerHashes[dlp.id] {
- if hash == origin {
- number = uint64(len(dlp.dl.peerHashes[dlp.id]) - num - 1)
- break
- }
+ if reverse {
+ panic("reverse header requests not supported")
}
- dlp.dl.lock.RUnlock()
- // Use the absolute header fetcher to satisfy the query
- return dlp.RequestHeadersByNumber(number, amount, skip, reverse)
+ result := dlp.chain.headersByHash(origin, amount, skip)
+ go dlp.dl.downloader.DeliverHeaders(dlp.id, result)
+ return nil
}
// RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered
// origin; associated with a particular peer in the download tester. The returned
// function can be used to retrieve batches of headers from the particular peer.
func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error {
- dlp.waitDelay()
-
- dlp.dl.lock.RLock()
- defer dlp.dl.lock.RUnlock()
-
- // Gather the next batch of headers
- hashes := dlp.dl.peerHashes[dlp.id]
- headers := dlp.dl.peerHeaders[dlp.id]
- result := make([]*types.Header, 0, amount)
- for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ {
- if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok {
- result = append(result, header)
- }
+ if reverse {
+ panic("reverse header requests not supported")
}
- // Delay delivery a bit to allow attacks to unfold
- go func() {
- time.Sleep(time.Millisecond)
- dlp.dl.downloader.DeliverHeaders(dlp.id, result)
- }()
+
+ result := dlp.chain.headersByNumber(origin, amount, skip)
+ go dlp.dl.downloader.DeliverHeaders(dlp.id, result)
return nil
}
@@ -538,24 +349,8 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int,
// peer in the download tester. The returned function can be used to retrieve
// batches of block bodies from the particularly requested peer.
func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error {
- dlp.waitDelay()
-
- dlp.dl.lock.RLock()
- defer dlp.dl.lock.RUnlock()
-
- blocks := dlp.dl.peerBlocks[dlp.id]
-
- transactions := make([][]*types.Transaction, 0, len(hashes))
- uncles := make([][]*types.Header, 0, len(hashes))
-
- for _, hash := range hashes {
- if block, ok := blocks[hash]; ok {
- transactions = append(transactions, block.Transactions())
- uncles = append(uncles, block.Uncles())
- }
- }
- go dlp.dl.downloader.DeliverBodies(dlp.id, transactions, uncles)
-
+ txs, uncles := dlp.chain.bodies(hashes)
+ go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles)
return nil
}
@@ -563,21 +358,8 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error {
// peer in the download tester. The returned function can be used to retrieve
// batches of block receipts from the particularly requested peer.
func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error {
- dlp.waitDelay()
-
- dlp.dl.lock.RLock()
- defer dlp.dl.lock.RUnlock()
-
- receipts := dlp.dl.peerReceipts[dlp.id]
-
- results := make([][]*types.Receipt, 0, len(hashes))
- for _, hash := range hashes {
- if receipt, ok := receipts[hash]; ok {
- results = append(results, receipt)
- }
- }
- go dlp.dl.downloader.DeliverReceipts(dlp.id, results)
-
+ receipts := dlp.chain.receipts(hashes)
+ go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts)
return nil
}
@@ -585,49 +367,46 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error {
// peer in the download tester. The returned function can be used to retrieve
// batches of node state data from the particularly requested peer.
func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error {
- dlp.waitDelay()
-
dlp.dl.lock.RLock()
defer dlp.dl.lock.RUnlock()
results := make([][]byte, 0, len(hashes))
for _, hash := range hashes {
if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil {
- if !dlp.dl.peerMissingStates[dlp.id][hash] {
+ if !dlp.missingStates[hash] {
results = append(results, data)
}
}
}
go dlp.dl.downloader.DeliverNodeData(dlp.id, results)
-
return nil
}
// assertOwnChain checks if the local chain contains the correct number of items
// of the various chain components.
func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
+ // Mark this method as a helper to report errors at callsite, not in here
+ t.Helper()
+
assertOwnForkedChain(t, tester, 1, []int{length})
}
// assertOwnForkedChain checks if the local forked chain contains the correct
// number of items of the various chain components.
func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
- // Initialize the counters for the first fork
- headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks
+ // Mark this method as a helper to report errors at callsite, not in here
+ t.Helper()
+
+ // Initialize the counters for the first fork
+ headers, blocks, receipts := lengths[0], lengths[0], lengths[0]
- if receipts < 0 {
- receipts = 1
- }
// Update the counters for each subsequent fork
for _, length := range lengths[1:] {
headers += length - common
blocks += length - common
- receipts += length - common - fsMinFullBlocks
+ receipts += length - common
}
- switch tester.downloader.mode {
- case FullSync:
- receipts = 1
- case LightSync:
+ if tester.downloader.mode == LightSync {
blocks, receipts = 1, 1
}
if hs := len(tester.ownHeaders); hs != headers {
@@ -639,21 +418,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
if rs := len(tester.ownReceipts); rs != receipts {
t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts)
}
- // Verify the state trie too for fast syncs
- /*if tester.downloader.mode == FastSync {
- pivot := uint64(0)
- var index int
- if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common {
- index = pivot
- } else {
- index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
- }
- if index > 0 {
- if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil {
- t.Fatalf("state reconstruction failed: %v", err)
- }
- }
- }*/
}
// Tests that simple synchronization against a canonical chain works correctly.
@@ -673,16 +437,14 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
-
- tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
+ chain := testChainBase.shorten(blockCacheItems - 15)
+ tester.newPeer("peer", protocol, chain)
// Synchronise with the peer and make sure all relevant data was retrieved
if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
+ assertOwnChain(t, tester, chain.len())
}
// Tests that if a large batch of blocks are being downloaded, it is throttled
@@ -699,10 +461,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a long block chain to download and the tester
- targetBlocks := 8 * blockCacheItems
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
-
- tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
+ targetBlocks := testChainBase.len() - 1
+ tester.newPeer("peer", protocol, testChainBase)
// Wrap the importer to allow stepping
blocked, proceed := uint32(0), make(chan struct{})
@@ -734,9 +494,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
cached = len(tester.downloader.queue.blockDonePool)
if mode == FastSync {
if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached {
- //if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot {
cached = receipts
- //}
}
}
frozen = int(atomic.LoadUint32(&blocked))
@@ -786,24 +544,22 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- // Create a long enough forked chain
- common, fork := MaxHashFetch, 2*MaxHashFetch
- hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true)
-
- tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA)
- tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB)
+ chainA := testChainForkLightA.shorten(testChainBase.len() + 80)
+ chainB := testChainForkLightB.shorten(testChainBase.len() + 80)
+ tester.newPeer("fork A", protocol, chainA)
+ tester.newPeer("fork B", protocol, chainB)
// Synchronise with the peer and make sure all blocks were retrieved
if err := tester.sync("fork A", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, common+fork+1)
+ assertOwnChain(t, tester, chainA.len())
// Synchronise with the second peer and make sure that fork is pulled too
if err := tester.sync("fork B", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1})
+ assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()})
}
// Tests that synchronising against a much shorter but much heavyer fork works
@@ -821,24 +577,22 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- // Create a long enough forked chain
- common, fork := MaxHashFetch, 4*MaxHashFetch
- hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false)
-
- tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA)
- tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB)
+ chainA := testChainForkLightA.shorten(testChainBase.len() + 80)
+ chainB := testChainForkHeavy.shorten(testChainBase.len() + 80)
+ tester.newPeer("light", protocol, chainA)
+ tester.newPeer("heavy", protocol, chainB)
// Synchronise with the peer and make sure all blocks were retrieved
if err := tester.sync("light", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, common+fork+1)
+ assertOwnChain(t, tester, chainA.len())
// Synchronise with the second peer and make sure that fork is pulled too
if err := tester.sync("heavy", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1})
+ assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()})
}
// Tests that chain forks are contained within a certain interval of the current
@@ -857,18 +611,16 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- // Create a long enough forked chain
- common, fork := 13, int(MaxForkAncestry+17)
- hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true)
-
- tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA)
- tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB)
+ chainA := testChainForkLightA
+ chainB := testChainForkLightB
+ tester.newPeer("original", protocol, chainA)
+ tester.newPeer("rewriter", protocol, chainB)
// Synchronise with the peer and make sure all blocks were retrieved
if err := tester.sync("original", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, common+fork+1)
+ assertOwnChain(t, tester, chainA.len())
// Synchronise with the second peer and ensure that the fork is rejected to being too old
if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor {
@@ -893,17 +645,16 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a long enough forked chain
- common, fork := 13, int(MaxForkAncestry+17)
- hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false)
-
- tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA)
- tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit
+ chainA := testChainForkLightA
+ chainB := testChainForkHeavy
+ tester.newPeer("original", protocol, chainA)
+ tester.newPeer("heavy-rewriter", protocol, chainB)
// Synchronise with the peer and make sure all blocks were retrieved
if err := tester.sync("original", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, common+fork+1)
+ assertOwnChain(t, tester, chainA.len())
// Synchronise with the second peer and ensure that the fork is rejected to being too old
if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor {
@@ -924,7 +675,7 @@ func TestInactiveDownloader62(t *testing.T) {
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
}
if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive {
- t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
+ t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
}
}
@@ -962,17 +713,8 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- // Create a small enough block chain to download and the tester
- targetBlocks := blockCacheItems - 15
- if targetBlocks >= MaxHashFetch {
- targetBlocks = MaxHashFetch - 15
- }
- if targetBlocks >= MaxHeaderFetch {
- targetBlocks = MaxHeaderFetch - 15
- }
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
-
- tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
+ chain := testChainBase.shorten(MaxHeaderFetch)
+ tester.newPeer("peer", protocol, chain)
// Make sure canceling works with a pristine downloader
tester.downloader.Cancel()
@@ -1005,17 +747,16 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) {
// Create various peers with various parts of the chain
targetPeers := 8
- targetBlocks := targetPeers*blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(targetPeers * 100)
for i := 0; i < targetPeers; i++ {
id := fmt.Sprintf("peer #%d", i)
- tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts)
+ tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1)))
}
if err := tester.sync("peer #0", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
+ assertOwnChain(t, tester, chain.len())
}
// Tests that synchronisations behave well in multi-version protocol environments
@@ -1034,24 +775,23 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(blockCacheItems - 15)
// Create peers of every type
- tester.newPeer("peer 62", 62, hashes, headers, blocks, nil)
- tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts)
- tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts)
+ tester.newPeer("peer 62", 62, chain)
+ tester.newPeer("peer 63", 63, chain)
+ tester.newPeer("peer 64", 64, chain)
// Synchronise with the requested peer and make sure all blocks were retrieved
if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
+ assertOwnChain(t, tester, chain.len())
// Check that no peers have been dropped off
for _, version := range []int{62, 63, 64} {
peer := fmt.Sprintf("peer %d", version)
- if _, ok := tester.peerHashes[peer]; !ok {
+ if _, ok := tester.peers[peer]; !ok {
t.Errorf("%s dropped", peer)
}
}
@@ -1073,10 +813,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a block chain to download
- targetBlocks := 2*blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
-
- tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
+ chain := testChainBase
+ tester.newPeer("peer", protocol, chain)
// Instrument the downloader to signal body requests
bodiesHave, receiptsHave := int32(0), int32(0)
@@ -1090,16 +828,16 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
+ assertOwnChain(t, tester, chain.len())
// Validate the number of block bodies that should have been requested
bodiesNeeded, receiptsNeeded := 0, 0
- for _, block := range blocks {
+ for _, block := range chain.blockm {
if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) {
bodiesNeeded++
}
}
- for _, receipt := range receipts {
+ for _, receipt := range chain.receiptm {
if mode == FastSync && len(receipt) > 0 {
receiptsNeeded++
}
@@ -1127,24 +865,20 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- // Create a small enough block chain to download
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
-
- // Attempt a full sync with an attacker feeding gapped headers
- tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
- missing := targetBlocks / 2
- delete(tester.peerHeaders["attack"], hashes[missing])
+ chain := testChainBase.shorten(blockCacheItems - 15)
+ brokenChain := chain.shorten(chain.len())
+ delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2])
+ tester.newPeer("attack", protocol, brokenChain)
if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
- tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ tester.newPeer("valid", protocol, chain)
if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
+ assertOwnChain(t, tester, chain.len())
}
// Tests that if requested headers are shifted (i.e. first is missing), the queue
@@ -1162,25 +896,24 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- // Create a small enough block chain to download
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(blockCacheItems - 15)
// Attempt a full sync with an attacker feeding shifted headers
- tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
- delete(tester.peerHeaders["attack"], hashes[len(hashes)-2])
- delete(tester.peerBlocks["attack"], hashes[len(hashes)-2])
- delete(tester.peerReceipts["attack"], hashes[len(hashes)-2])
-
+ brokenChain := chain.shorten(chain.len())
+ delete(brokenChain.headerm, brokenChain.chain[1])
+ delete(brokenChain.blockm, brokenChain.chain[1])
+ delete(brokenChain.receiptm, brokenChain.chain[1])
+ tester.newPeer("attack", protocol, brokenChain)
if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
+
// Synchronise with the valid peer and make sure sync succeeds
- tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ tester.newPeer("valid", protocol, chain)
if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
+ assertOwnChain(t, tester, chain.len())
}
// Tests that upon detecting an invalid header, the recent ones are rolled back
@@ -1198,13 +931,14 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(targetBlocks)
// Attempt to sync with an attacker that feeds junk during the fast sync phase.
// This should result in the last fsHeaderSafetyNet headers being rolled back.
- tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts)
missing := fsHeaderSafetyNet + MaxHeaderFetch + 1
- delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing])
+ fastAttackChain := chain.shorten(chain.len())
+ delete(fastAttackChain.headerm, fastAttackChain.chain[missing])
+ tester.newPeer("fast-attack", protocol, fastAttackChain)
if err := tester.sync("fast-attack", nil, mode); err == nil {
t.Fatalf("succeeded fast attacker synchronisation")
@@ -1212,13 +946,15 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch {
t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch)
}
+
// Attempt to sync with an attacker that feeds junk during the block import phase.
// This should result in both the last fsHeaderSafetyNet number of headers being
// rolled back, and also the pivot point being reverted to a non-block status.
- tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts)
missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
- delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in
- delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing])
+ blockAttackChain := chain.shorten(chain.len())
+ delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in
+ delete(blockAttackChain.headerm, blockAttackChain.chain[missing])
+ tester.newPeer("block-attack", protocol, blockAttackChain)
if err := tester.sync("block-attack", nil, mode); err == nil {
t.Fatalf("succeeded block attacker synchronisation")
@@ -1231,19 +967,18 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
t.Errorf("fast sync pivot block #%d not rolled back", head)
}
}
+
// Attempt to sync with an attacker that withholds promised blocks after the
// fast sync pivot point. This could be a trial to leave the node with a bad
// but already imported pivot block.
- tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
- missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
-
+ withholdAttackChain := chain.shorten(chain.len())
+ tester.newPeer("withhold-attack", protocol, withholdAttackChain)
tester.downloader.syncInitHook = func(uint64, uint64) {
- for i := missing; i <= len(hashes); i++ {
- delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
+ for i := missing; i < withholdAttackChain.len(); i++ {
+ delete(withholdAttackChain.headerm, withholdAttackChain.chain[i])
}
tester.downloader.syncInitHook = nil
}
-
if err := tester.sync("withhold-attack", nil, mode); err == nil {
t.Fatalf("succeeded withholding attacker synchronisation")
}
@@ -1255,20 +990,21 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
t.Errorf("fast sync pivot block #%d not rolled back", head)
}
}
- // Synchronise with the valid peer and make sure sync succeeds. Since the last
- // rollback should also disable fast syncing for this process, verify that we
- // did a fresh full sync. Note, we can't assert anything about the receipts
- // since we won't purge the database of them, hence we can't use assertOwnChain.
- tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+
+ // synchronise with the valid peer and make sure sync succeeds. Since the last rollback
+ // should also disable fast syncing for this process, verify that we did a fresh full
+ // sync. Note, we can't assert anything about the receipts since we won't purge the
+ // database of them, hence we can't use assertOwnChain.
+ tester.newPeer("valid", protocol, chain)
if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if hs := len(tester.ownHeaders); hs != len(headers) {
- t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers))
+ if hs := len(tester.ownHeaders); hs != chain.len() {
+ t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len())
}
if mode != LightSync {
- if bs := len(tester.ownBlocks); bs != len(blocks) {
- t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks))
+ if bs := len(tester.ownBlocks); bs != chain.len() {
+ t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len())
}
}
}
@@ -1288,9 +1024,8 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
- hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false)
- tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts)
-
+ chain := testChainBase.shorten(1)
+ tester.newPeer("attack", protocol, chain)
if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer {
t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer)
}
@@ -1333,21 +1068,22 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
// Run the tests and check disconnection status
tester := newTester()
defer tester.terminate()
+ chain := testChainBase.shorten(1)
for i, tt := range tests {
// Register a new peer and ensure it's presence
id := fmt.Sprintf("test %d", i)
- if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil {
+ if err := tester.newPeer(id, protocol, chain); err != nil {
t.Fatalf("test %d: failed to register new peer: %v", i, err)
}
- if _, ok := tester.peerHashes[id]; !ok {
+ if _, ok := tester.peers[id]; !ok {
t.Fatalf("test %d: registered peer not found", i)
}
// Simulate a synchronisation and check the required result
tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result }
tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync)
- if _, ok := tester.peerHashes[id]; !ok != tt.drop {
+ if _, ok := tester.peers[id]; !ok != tt.drop {
t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop)
}
}
@@ -1367,10 +1103,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
-
- // Create a small enough block chain to download
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(blockCacheItems - 15)
// Set a sync init hook to catch progress changes
starting := make(chan struct{})
@@ -1380,12 +1113,10 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync progress and ensure they are zero (pristine sync)
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 {
- t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0)
- }
+ checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{})
+
// Synchronise half the blocks and check initial progress
- tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts)
+ tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2))
pending := new(sync.WaitGroup)
pending.Add(1)
@@ -1396,16 +1127,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) {
- t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1)
- }
+ checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{
+ HighestBlock: uint64(chain.len()/2 - 1),
+ })
progress <- struct{}{}
pending.Wait()
// Synchronise all the blocks and check continuation progress
- tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts)
+ tester.newPeer("peer-full", protocol, chain)
pending.Add(1)
-
go func() {
defer pending.Done()
if err := tester.sync("peer-full", nil, mode); err != nil {
@@ -1413,15 +1143,31 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks)
- }
- progress <- struct{}{}
- pending.Wait()
+ checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{
+ StartingBlock: uint64(chain.len()/2 - 1),
+ CurrentBlock: uint64(chain.len()/2 - 1),
+ HighestBlock: uint64(chain.len() - 1),
+ })
// Check final progress after successful sync
- if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks)
+ progress <- struct{}{}
+ pending.Wait()
+ checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{
+ StartingBlock: uint64(chain.len()/2 - 1),
+ CurrentBlock: uint64(chain.len() - 1),
+ HighestBlock: uint64(chain.len() - 1),
+ })
+}
+
+func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) {
+ // Mark this method as a helper to report errors at callsite, not in here
+ t.Helper()
+
+ p := d.Progress()
+ p.KnownStates, p.PulledStates = 0, 0
+ want.KnownStates, want.PulledStates = 0, 0
+ if p != want {
+ t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want)
}
}
@@ -1440,10 +1186,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
-
- // Create a forked chain to simulate origin revertal
- common, fork := MaxHashFetch, 2*MaxHashFetch
- hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true)
+ chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHashFetch)
+ chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHashFetch)
// Set a sync init hook to catch progress changes
starting := make(chan struct{})
@@ -1453,15 +1197,12 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync progress and ensure they are zero (pristine sync)
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 {
- t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0)
- }
+ checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{})
+
// Synchronise with one of the forks and check progress
- tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA)
+ tester.newPeer("fork A", protocol, chainA)
pending := new(sync.WaitGroup)
pending.Add(1)
-
go func() {
defer pending.Done()
if err := tester.sync("fork A", nil, mode); err != nil {
@@ -1469,9 +1210,10 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) {
- t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1)
- }
+
+ checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{
+ HighestBlock: uint64(chainA.len() - 1),
+ })
progress <- struct{}{}
pending.Wait()
@@ -1479,9 +1221,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight
// Synchronise with the second fork and check progress resets
- tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB)
+ tester.newPeer("fork B", protocol, chainB)
pending.Add(1)
-
go func() {
defer pending.Done()
if err := tester.sync("fork B", nil, mode); err != nil {
@@ -1489,16 +1230,20 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) {
- t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1)
- }
- progress <- struct{}{}
- pending.Wait()
+ checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{
+ StartingBlock: uint64(testChainBase.len()) - 1,
+ CurrentBlock: uint64(chainA.len() - 1),
+ HighestBlock: uint64(chainB.len() - 1),
+ })
// Check final progress after successful sync
- if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) {
- t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1)
- }
+ progress <- struct{}{}
+ pending.Wait()
+ checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{
+ StartingBlock: uint64(testChainBase.len()) - 1,
+ CurrentBlock: uint64(chainB.len() - 1),
+ HighestBlock: uint64(chainB.len() - 1),
+ })
}
// Tests that if synchronisation is aborted due to some failure, then the progress
@@ -1516,10 +1261,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
-
- // Create a small enough block chain to download
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(blockCacheItems - 15)
// Set a sync init hook to catch progress changes
starting := make(chan struct{})
@@ -1529,20 +1271,18 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync progress and ensure they are zero (pristine sync)
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 {
- t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0)
- }
+ checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{})
+
// Attempt a full sync with a faulty peer
- tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts)
- missing := targetBlocks / 2
- delete(tester.peerHeaders["faulty"], hashes[missing])
- delete(tester.peerBlocks["faulty"], hashes[missing])
- delete(tester.peerReceipts["faulty"], hashes[missing])
+ brokenChain := chain.shorten(chain.len())
+ missing := brokenChain.len() / 2
+ delete(brokenChain.headerm, brokenChain.chain[missing])
+ delete(brokenChain.blockm, brokenChain.chain[missing])
+ delete(brokenChain.receiptm, brokenChain.chain[missing])
+ tester.newPeer("faulty", protocol, brokenChain)
pending := new(sync.WaitGroup)
pending.Add(1)
-
go func() {
defer pending.Done()
if err := tester.sync("faulty", nil, mode); err == nil {
@@ -1550,16 +1290,17 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks)
- }
+ checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{
+ HighestBlock: uint64(brokenChain.len() - 1),
+ })
progress <- struct{}{}
pending.Wait()
+ afterFailedSync := tester.downloader.Progress()
- // Synchronise with a good peer and check that the progress origin remind the same after a failure
- tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ // Synchronise with a good peer and check that the progress origin remind the same
+ // after a failure
+ tester.newPeer("valid", protocol, chain)
pending.Add(1)
-
go func() {
defer pending.Done()
if err := tester.sync("valid", nil, mode); err != nil {
@@ -1567,16 +1308,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks)
- }
- progress <- struct{}{}
- pending.Wait()
+ checkProgress(t, tester.downloader, "completing", afterFailedSync)
// Check final progress after successful sync
- if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks)
- }
+ progress <- struct{}{}
+ pending.Wait()
+ checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{
+ CurrentBlock: uint64(chain.len() - 1),
+ HighestBlock: uint64(chain.len() - 1),
+ })
}
// Tests that if an attacker fakes a chain height, after the attack is detected,
@@ -1593,34 +1333,27 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
defer tester.terminate()
-
- // Create a small block chain
- targetBlocks := blockCacheItems - 15
- hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false)
+ chain := testChainBase.shorten(blockCacheItems - 15)
// Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
-
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync progress and ensure they are zero (pristine sync)
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 {
- t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0)
- }
- // Create and sync with an attacker that promises a higher chain than available
- tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
- for i := 1; i < 3; i++ {
- delete(tester.peerHeaders["attack"], hashes[i])
- delete(tester.peerBlocks["attack"], hashes[i])
- delete(tester.peerReceipts["attack"], hashes[i])
+ checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{})
+
+ // Create and sync with an attacker that promises a higher chain than available.
+ brokenChain := chain.shorten(chain.len())
+ numMissing := 5
+ for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- {
+ delete(brokenChain.headerm, brokenChain.chain[i])
}
+ tester.newPeer("attack", protocol, brokenChain)
pending := new(sync.WaitGroup)
pending.Add(1)
-
go func() {
defer pending.Done()
if err := tester.sync("attack", nil, mode); err == nil {
@@ -1628,14 +1361,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) {
- t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3)
- }
+ checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{
+ HighestBlock: uint64(brokenChain.len() - 1),
+ })
progress <- struct{}{}
pending.Wait()
+ afterFailedSync := tester.downloader.Progress()
- // Synchronise with a good peer and check that the progress height has been reduced to the true value
- tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts)
+ // Synchronise with a good peer and check that the progress height has been reduced to
+ // the true value.
+ validChain := chain.shorten(chain.len() - numMissing)
+ tester.newPeer("valid", protocol, validChain)
pending.Add(1)
go func() {
@@ -1645,23 +1381,25 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
}()
<-starting
- if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks)
- }
+ checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{
+ CurrentBlock: afterFailedSync.CurrentBlock,
+ HighestBlock: uint64(validChain.len() - 1),
+ })
+
+ // Check final progress after successful sync.
progress <- struct{}{}
pending.Wait()
-
- // Check final progress after successful sync
- if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) {
- t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks)
- }
+ checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{
+ CurrentBlock: uint64(validChain.len() - 1),
+ HighestBlock: uint64(validChain.len() - 1),
+ })
}
// This test reproduces an issue where unexpected deliveries would
// block indefinitely if they arrived at the right time.
-// We use data driven subtests to manage this so that it will be parallel on its own
-// and not with the other tests, avoiding intermittent failures.
func TestDeliverHeadersHang(t *testing.T) {
+ t.Parallel()
+
testCases := []struct {
protocol int
syncMode SyncMode
@@ -1675,15 +1413,38 @@ func TestDeliverHeadersHang(t *testing.T) {
}
for _, tc := range testCases {
t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) {
+ t.Parallel()
testDeliverHeadersHang(t, tc.protocol, tc.syncMode)
})
}
}
+func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) {
+ master := newTester()
+ defer master.terminate()
+ chain := testChainBase.shorten(15)
+
+ for i := 0; i < 200; i++ {
+ tester := newTester()
+ tester.peerDb = master.peerDb
+ tester.newPeer("peer", protocol, chain)
+
+ // Whenever the downloader requests headers, flood it with
+ // a lot of unrequested header deliveries.
+ tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{
+ peer: tester.downloader.peers.peers["peer"].peer,
+ tester: tester,
+ }
+ if err := tester.sync("peer", nil, mode); err != nil {
+ t.Errorf("test %d: sync failed: %v", i, err)
+ }
+ tester.terminate()
+ }
+}
+
type floodingTestPeer struct {
peer Peer
tester *downloadTester
- pend sync.WaitGroup
}
func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() }
@@ -1702,23 +1463,29 @@ func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error {
func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error {
deliveriesDone := make(chan struct{}, 500)
- for i := 0; i < cap(deliveriesDone); i++ {
+ for i := 0; i < cap(deliveriesDone)-1; i++ {
peer := fmt.Sprintf("fake-peer%d", i)
- ftp.pend.Add(1)
-
go func() {
ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}})
deliveriesDone <- struct{}{}
- ftp.pend.Done()
}()
}
- // Deliver the actual requested headers.
- go ftp.peer.RequestHeadersByNumber(from, count, skip, reverse)
+
// None of the extra deliveries should block.
timeout := time.After(60 * time.Second)
+ launched := false
for i := 0; i < cap(deliveriesDone); i++ {
select {
case <-deliveriesDone:
+ if !launched {
+ // Start delivering the requested headers
+ // after one of the flooding responses has arrived.
+ go func() {
+ ftp.peer.RequestHeadersByNumber(from, count, skip, reverse)
+ deliveriesDone <- struct{}{}
+ }()
+ launched = true
+ }
case <-timeout:
panic("blocked")
}
@@ -1726,30 +1493,77 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int
return nil
}
-func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) {
- t.Parallel()
-
- master := newTester()
- defer master.terminate()
-
- hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false)
- for i := 0; i < 200; i++ {
- tester := newTester()
- tester.peerDb = master.peerDb
-
- tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
- // Whenever the downloader requests headers, flood it with
- // a lot of unrequested header deliveries.
- tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{
- peer: tester.downloader.peers.peers["peer"].peer,
- tester: tester,
+func TestRemoteHeaderRequestSpan(t *testing.T) {
+ testCases := []struct {
+ remoteHeight uint64
+ localHeight uint64
+ expected []int
+ }{
+ // Remote is way higher. We should ask for the remote head and go backwards
+ {1500, 1000,
+ []int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499},
+ },
+ {15000, 13006,
+ []int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999},
+ },
+ //Remote is pretty close to us. We don't have to fetch as many
+ {1200, 1150,
+ []int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199},
+ },
+ // Remote is equal to us (so on a fork with higher td)
+ // We should get the closest couple of ancestors
+ {1500, 1500,
+ []int{1497, 1499},
+ },
+ // We're higher than the remote! Odd
+ {1000, 1500,
+ []int{997, 999},
+ },
+ // Check some weird edgecases that it behaves somewhat rationally
+ {0, 1500,
+ []int{0, 2},
+ },
+ {6000000, 0,
+ []int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999},
+ },
+ {0, 0,
+ []int{0, 2},
+ },
+ }
+ reqs := func(from, count, span int) []int {
+ var r []int
+ num := from
+ for len(r) < count {
+ r = append(r, num)
+ num += span + 1
}
- if err := tester.sync("peer", nil, mode); err != nil {
- t.Errorf("test %d: sync failed: %v", i, err)
- }
- tester.terminate()
+ return r
+ }
+ for i, tt := range testCases {
+ from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight)
+ data := reqs(int(from), count, span)
- // Flush all goroutines to prevent messing with subsequent tests
- tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait()
+ if max != uint64(data[len(data)-1]) {
+ t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max)
+ }
+ failed := false
+ if len(data) != len(tt.expected) {
+ failed = true
+ t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data))
+ } else {
+ for j, n := range data {
+ if n != tt.expected[j] {
+ failed = true
+ break
+ }
+ }
+ }
+ if failed {
+ res := strings.Replace(fmt.Sprint(data), " ", ",", -1)
+ exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1)
+ fmt.Printf("got: %v\n", res)
+ fmt.Printf("exp: %v\n", exp)
+ t.Errorf("test %d: wrong values", i)
+ }
}
}
diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go
index c6b635aff1..7c33953811 100644
--- a/eth/downloader/queue.go
+++ b/eth/downloader/queue.go
@@ -325,7 +325,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
}
// Make sure no duplicate requests are executed
if _, ok := q.blockTaskPool[hash]; ok {
- log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash)
+ log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash)
continue
}
if _, ok := q.receiptTaskPool[hash]; ok {
@@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest,
}
// Add the peer to the expiry report along the number of failed requests
expiries[id] = len(request.Headers)
+
+ // Remove the expired requests from the pending pool directly
+ delete(pendPool, id)
}
}
- // Remove the expired requests from the pending pool
- for id := range expiries {
- delete(pendPool, id)
- }
return expiries
}
diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go
index 8d33dfec74..29d5ee4ddb 100644
--- a/eth/downloader/statesync.go
+++ b/eth/downloader/statesync.go
@@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) {
s.d.dropPeer(req.peer.id)
}
// Process all the received blobs and check for stale delivery
- if err = s.process(req); err != nil {
+ delivered, err := s.process(req)
+ if err != nil {
log.Warn("Node data write error", "err", err)
return err
}
- req.peer.SetNodeDataIdle(len(req.response))
+ req.peer.SetNodeDataIdle(delivered)
}
}
return nil
@@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) {
// process iterates over a batch of delivered state data, injecting each item
// into a running state sync, re-queuing any items that were requested but not
// delivered.
-func (s *stateSync) process(req *stateReq) error {
+// Returns whether the peer actually managed to deliver anything of value,
+// and any error that occurred
+func (s *stateSync) process(req *stateReq) (int, error) {
// Collect processing stats and update progress if valid data was received
- duplicate, unexpected := 0, 0
+ duplicate, unexpected, successful := 0, 0, 0
defer func(start time.Time) {
if duplicate > 0 || unexpected > 0 {
@@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error {
// Iterate over all the delivered data and inject one-by-one into the trie
progress := false
-
for _, blob := range req.response {
prog, hash, err := s.processNodeData(blob)
switch err {
@@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error {
s.numUncommitted++
s.bytesUncommitted += len(blob)
progress = progress || prog
+ successful++
case trie.ErrNotRequested:
unexpected++
case trie.ErrAlreadyProcessed:
duplicate++
default:
- return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
+ return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
}
if _, ok := req.tasks[hash]; ok {
delete(req.tasks, hash)
@@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error {
// If we've requested the node too many times already, it may be a malicious
// sync where nobody has the right data. Abort.
if len(task.attempts) >= npeers {
- return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
+ return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
}
// Missing item, place into the retry queue.
s.tasks[hash] = task
}
- return nil
+ return successful, nil
}
// processNodeData tries to inject a trie node data blob delivered from a remote
diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go
new file mode 100644
index 0000000000..0b5a214257
--- /dev/null
+++ b/eth/downloader/testchain_test.go
@@ -0,0 +1,221 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package downloader
+
+import (
+ "fmt"
+ "math/big"
+ "sync"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/consensus/ethash"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/params"
+)
+
+// Test chain parameters.
+var (
+ testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
+ testDB = ethdb.NewMemDatabase()
+ testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000))
+)
+
+// The common prefix of all test chains:
+var testChainBase = newTestChain(blockCacheItems+200, testGenesis)
+
+// Different forks on top of the base chain:
+var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain
+
+func init() {
+ var forkLen = int(MaxForkAncestry + 50)
+ var wg sync.WaitGroup
+ wg.Add(3)
+ go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }()
+ go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }()
+ go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }()
+ wg.Wait()
+}
+
+type testChain struct {
+ genesis *types.Block
+ chain []common.Hash
+ headerm map[common.Hash]*types.Header
+ blockm map[common.Hash]*types.Block
+ receiptm map[common.Hash][]*types.Receipt
+ tdm map[common.Hash]*big.Int
+}
+
+// newTestChain creates a blockchain of the given length.
+func newTestChain(length int, genesis *types.Block) *testChain {
+ tc := new(testChain).copy(length)
+ tc.genesis = genesis
+ tc.chain = append(tc.chain, genesis.Hash())
+ tc.headerm[tc.genesis.Hash()] = tc.genesis.Header()
+ tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty()
+ tc.blockm[tc.genesis.Hash()] = tc.genesis
+ tc.generate(length-1, 0, genesis, false)
+ return tc
+}
+
+// makeFork creates a fork on top of the test chain.
+func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain {
+ fork := tc.copy(tc.len() + length)
+ fork.generate(length, seed, tc.headBlock(), heavy)
+ return fork
+}
+
+// shorten creates a copy of the chain with the given length. It panics if the
+// length is longer than the number of available blocks.
+func (tc *testChain) shorten(length int) *testChain {
+ if length > tc.len() {
+ panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len()))
+ }
+ return tc.copy(length)
+}
+
+func (tc *testChain) copy(newlen int) *testChain {
+ cpy := &testChain{
+ genesis: tc.genesis,
+ headerm: make(map[common.Hash]*types.Header, newlen),
+ blockm: make(map[common.Hash]*types.Block, newlen),
+ receiptm: make(map[common.Hash][]*types.Receipt, newlen),
+ tdm: make(map[common.Hash]*big.Int, newlen),
+ }
+ for i := 0; i < len(tc.chain) && i < newlen; i++ {
+ hash := tc.chain[i]
+ cpy.chain = append(cpy.chain, tc.chain[i])
+ cpy.tdm[hash] = tc.tdm[hash]
+ cpy.blockm[hash] = tc.blockm[hash]
+ cpy.headerm[hash] = tc.headerm[hash]
+ cpy.receiptm[hash] = tc.receiptm[hash]
+ }
+ return cpy
+}
+
+// generate creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent. In addition, every 22th block
+// contains a transaction and every 5th an uncle to allow testing correct block
+// reassembly.
+func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) {
+ // start := time.Now()
+ // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }()
+
+ blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) {
+ block.SetCoinbase(common.Address{seed})
+ // If a heavy chain is requested, delay blocks to raise difficulty
+ if heavy {
+ block.OffsetTime(-1)
+ }
+ // Include transactions to the miner to make blocks more interesting.
+ if parent == tc.genesis && i%22 == 0 {
+ signer := types.MakeSigner(params.TestChainConfig, block.Number())
+ tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey)
+ if err != nil {
+ panic(err)
+ }
+ block.AddTx(tx)
+ }
+ // if the block number is a multiple of 5, add a bonus uncle to the block
+ if i > 0 && i%5 == 0 {
+ block.AddUncle(&types.Header{
+ ParentHash: block.PrevBlock(i - 1).Hash(),
+ Number: big.NewInt(block.Number().Int64() - 1),
+ })
+ }
+ })
+
+ // Convert the block-chain into a hash-chain and header/block maps
+ td := new(big.Int).Set(tc.td(parent.Hash()))
+ for i, b := range blocks {
+ td := td.Add(td, b.Difficulty())
+ hash := b.Hash()
+ tc.chain = append(tc.chain, hash)
+ tc.blockm[hash] = b
+ tc.headerm[hash] = b.Header()
+ tc.receiptm[hash] = receipts[i]
+ tc.tdm[hash] = new(big.Int).Set(td)
+ }
+}
+
+// len returns the total number of blocks in the chain.
+func (tc *testChain) len() int {
+ return len(tc.chain)
+}
+
+// headBlock returns the head of the chain.
+func (tc *testChain) headBlock() *types.Block {
+ return tc.blockm[tc.chain[len(tc.chain)-1]]
+}
+
+// td returns the total difficulty of the given block.
+func (tc *testChain) td(hash common.Hash) *big.Int {
+ return tc.tdm[hash]
+}
+
+// headersByHash returns headers in ascending order from the given hash.
+func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header {
+ num, _ := tc.hashToNumber(origin)
+ return tc.headersByNumber(num, amount, skip)
+}
+
+// headersByNumber returns headers in ascending order from the given number.
+func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header {
+ result := make([]*types.Header, 0, amount)
+ for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 {
+ if header, ok := tc.headerm[tc.chain[int(num)]]; ok {
+ result = append(result, header)
+ }
+ }
+ return result
+}
+
+// receipts returns the receipts of the given block hashes.
+func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt {
+ results := make([][]*types.Receipt, 0, len(hashes))
+ for _, hash := range hashes {
+ if receipt, ok := tc.receiptm[hash]; ok {
+ results = append(results, receipt)
+ }
+ }
+ return results
+}
+
+// bodies returns the block bodies of the given block hashes.
+func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) {
+ transactions := make([][]*types.Transaction, 0, len(hashes))
+ uncles := make([][]*types.Header, 0, len(hashes))
+ for _, hash := range hashes {
+ if block, ok := tc.blockm[hash]; ok {
+ transactions = append(transactions, block.Transactions())
+ uncles = append(uncles, block.Uncles())
+ }
+ }
+ return transactions, uncles
+}
+
+func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) {
+ for num, hash := range tc.chain {
+ if hash == target {
+ return uint64(num), true
+ }
+ }
+ return 0, false
+}
diff --git a/eth/gen_config.go b/eth/gen_config.go
index d401a917d2..2777aa9e83 100644
--- a/eth/gen_config.go
+++ b/eth/gen_config.go
@@ -28,7 +28,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
- TrieCache int
+ TrieCleanCache int
+ TrieDirtyCache int
TrieTimeout time.Duration
Etherbase common.Address `toml:",omitempty"`
MinerNotify []string `toml:",omitempty"`
@@ -43,6 +44,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
GPO gasprice.Config
EnablePreimageRecording bool
DocRoot string `toml:"-"`
+ EWASMInterpreter string
+ EVMInterpreter string
}
var enc Config
enc.Genesis = c.Genesis
@@ -54,7 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.SkipBcVersionCheck = c.SkipBcVersionCheck
enc.DatabaseHandles = c.DatabaseHandles
enc.DatabaseCache = c.DatabaseCache
- enc.TrieCache = c.TrieCache
+ enc.TrieCleanCache = c.TrieCleanCache
+ enc.TrieDirtyCache = c.TrieDirtyCache
enc.TrieTimeout = c.TrieTimeout
enc.Etherbase = c.Etherbase
enc.MinerNotify = c.MinerNotify
@@ -69,6 +73,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.GPO = c.GPO
enc.EnablePreimageRecording = c.EnablePreimageRecording
enc.DocRoot = c.DocRoot
+ enc.EWASMInterpreter = c.EWASMInterpreter
+ enc.EVMInterpreter = c.EVMInterpreter
return &enc, nil
}
@@ -84,7 +90,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
SkipBcVersionCheck *bool `toml:"-"`
DatabaseHandles *int `toml:"-"`
DatabaseCache *int
- TrieCache *int
+ TrieCleanCache *int
+ TrieDirtyCache *int
TrieTimeout *time.Duration
Etherbase *common.Address `toml:",omitempty"`
MinerNotify []string `toml:",omitempty"`
@@ -99,6 +106,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
GPO *gasprice.Config
EnablePreimageRecording *bool
DocRoot *string `toml:"-"`
+ EWASMInterpreter *string
+ EVMInterpreter *string
}
var dec Config
if err := unmarshal(&dec); err != nil {
@@ -131,8 +140,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.DatabaseCache != nil {
c.DatabaseCache = *dec.DatabaseCache
}
- if dec.TrieCache != nil {
- c.TrieCache = *dec.TrieCache
+ if dec.TrieCleanCache != nil {
+ c.TrieCleanCache = *dec.TrieCleanCache
+ }
+ if dec.TrieDirtyCache != nil {
+ c.TrieDirtyCache = *dec.TrieDirtyCache
}
if dec.TrieTimeout != nil {
c.TrieTimeout = *dec.TrieTimeout
@@ -176,5 +188,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.DocRoot != nil {
c.DocRoot = *dec.DocRoot
}
+ if dec.EWASMInterpreter != nil {
+ c.EWASMInterpreter = *dec.EWASMInterpreter
+ }
+ if dec.EVMInterpreter != nil {
+ c.EVMInterpreter = *dec.EVMInterpreter
+ }
return nil
}
diff --git a/eth/handler.go b/eth/handler.go
index 1f62d820ec..741fc9d5a5 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -653,12 +653,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
trueHead = request.Block.ParentHash()
trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty())
)
- // Update the peers total difficulty if better than the previous
+ // Update the peer's total difficulty if better than the previous
if _, td := p.Head(); trueTD.Cmp(td) > 0 {
p.SetHead(trueHead, trueTD)
// Schedule a sync if above ours. Note, this will not fire a sync for a gap of
- // a singe block (as the true TD is below the propagated block), however this
+ // a single block (as the true TD is below the propagated block), however this
// scenario should easily be covered by the fetcher.
currentBlock := pm.blockchain.CurrentBlock()
if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 {
diff --git a/eth/handler_test.go b/eth/handler_test.go
index 7811cd4804..44824fd0b7 100644
--- a/eth/handler_test.go
+++ b/eth/handler_test.go
@@ -585,7 +585,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
}
}(peer)
}
- timeoutCh := time.NewTimer(time.Millisecond * 100).C
+ timeout := time.After(300 * time.Millisecond)
var receivedCount int
outer:
for {
@@ -597,7 +597,7 @@ outer:
if receivedCount == totalPeers {
break outer
}
- case <-timeoutCh:
+ case <-timeout:
break outer
}
}
diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go
index 52f29c83fc..a45a121159 100644
--- a/eth/tracers/tracer_test.go
+++ b/eth/tracers/tracer_test.go
@@ -48,7 +48,7 @@ type dummyStatedb struct {
state.StateDB
}
-func (dummyStatedb) GetRefund() uint64 { return 1337 }
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
func runTrace(tracer *Tracer) (json.RawMessage, error) {
env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go
index b40837c8cc..f3163e19b3 100644
--- a/ethclient/ethclient.go
+++ b/ethclient/ethclient.go
@@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb
// FilterLogs executes a filter query.
func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
var result []types.Log
- err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q))
+ arg, err := toFilterArg(q)
+ if err != nil {
+ return nil, err
+ }
+ err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg)
return result, err
}
// SubscribeFilterLogs subscribes to the results of a streaming filter query.
func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
- return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q))
+ arg, err := toFilterArg(q)
+ if err != nil {
+ return nil, err
+ }
+ return ec.c.EthSubscribe(ctx, ch, "logs", arg)
}
-func toFilterArg(q ethereum.FilterQuery) interface{} {
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
arg := map[string]interface{}{
- "fromBlock": toBlockNumArg(q.FromBlock),
- "toBlock": toBlockNumArg(q.ToBlock),
- "address": q.Addresses,
- "topics": q.Topics,
+ "address": q.Addresses,
+ "topics": q.Topics,
}
- if q.FromBlock == nil {
- arg["fromBlock"] = "0x0"
+ if q.BlockHash != nil {
+ arg["blockHash"] = *q.BlockHash
+ if q.FromBlock != nil || q.ToBlock != nil {
+ return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+ }
+ } else {
+ if q.FromBlock == nil {
+ arg["fromBlock"] = "0x0"
+ } else {
+ arg["fromBlock"] = toBlockNumArg(q.FromBlock)
+ }
+ arg["toBlock"] = toBlockNumArg(q.ToBlock)
}
- return arg
+ return arg, nil
}
// Pending State
diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go
index 178eb2be98..3e8bf974c2 100644
--- a/ethclient/ethclient_test.go
+++ b/ethclient/ethclient_test.go
@@ -16,7 +16,15 @@
package ethclient
-import "github.com/ethereum/go-ethereum"
+import (
+ "fmt"
+ "math/big"
+ "reflect"
+ "testing"
+
+ "github.com/ethereum/go-ethereum"
+ "github.com/ethereum/go-ethereum/common"
+)
// Verify that Client implements the ethereum interfaces.
var (
@@ -32,3 +40,113 @@ var (
// _ = ethereum.PendingStateEventer(&Client{})
_ = ethereum.PendingContractCaller(&Client{})
)
+
+func TestToFilterArg(t *testing.T) {
+ blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+ addresses := []common.Address{
+ common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"),
+ }
+ blockHash := common.HexToHash(
+ "0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb",
+ )
+
+ for _, testCase := range []struct {
+ name string
+ input ethereum.FilterQuery
+ output interface{}
+ err error
+ }{
+ {
+ "without BlockHash",
+ ethereum.FilterQuery{
+ Addresses: addresses,
+ FromBlock: big.NewInt(1),
+ ToBlock: big.NewInt(2),
+ Topics: [][]common.Hash{},
+ },
+ map[string]interface{}{
+ "address": addresses,
+ "fromBlock": "0x1",
+ "toBlock": "0x2",
+ "topics": [][]common.Hash{},
+ },
+ nil,
+ },
+ {
+ "with nil fromBlock and nil toBlock",
+ ethereum.FilterQuery{
+ Addresses: addresses,
+ Topics: [][]common.Hash{},
+ },
+ map[string]interface{}{
+ "address": addresses,
+ "fromBlock": "0x0",
+ "toBlock": "latest",
+ "topics": [][]common.Hash{},
+ },
+ nil,
+ },
+ {
+ "with blockhash",
+ ethereum.FilterQuery{
+ Addresses: addresses,
+ BlockHash: &blockHash,
+ Topics: [][]common.Hash{},
+ },
+ map[string]interface{}{
+ "address": addresses,
+ "blockHash": blockHash,
+ "topics": [][]common.Hash{},
+ },
+ nil,
+ },
+ {
+ "with blockhash and from block",
+ ethereum.FilterQuery{
+ Addresses: addresses,
+ BlockHash: &blockHash,
+ FromBlock: big.NewInt(1),
+ Topics: [][]common.Hash{},
+ },
+ nil,
+ blockHashErr,
+ },
+ {
+ "with blockhash and to block",
+ ethereum.FilterQuery{
+ Addresses: addresses,
+ BlockHash: &blockHash,
+ ToBlock: big.NewInt(1),
+ Topics: [][]common.Hash{},
+ },
+ nil,
+ blockHashErr,
+ },
+ {
+ "with blockhash and both from / to block",
+ ethereum.FilterQuery{
+ Addresses: addresses,
+ BlockHash: &blockHash,
+ FromBlock: big.NewInt(1),
+ ToBlock: big.NewInt(2),
+ Topics: [][]common.Hash{},
+ },
+ nil,
+ blockHashErr,
+ },
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ output, err := toFilterArg(testCase.input)
+ if (testCase.err == nil) != (err == nil) {
+ t.Fatalf("expected error %v but got %v", testCase.err, err)
+ }
+ if testCase.err != nil {
+ if testCase.err.Error() != err.Error() {
+ t.Fatalf("expected error %v but got %v", testCase.err, err)
+ }
+ } else if !reflect.DeepEqual(testCase.output, output) {
+ t.Fatalf("expected filter arg %v but got %v", testCase.output, output)
+ }
+ })
+ }
+}
diff --git a/event/event_test.go b/event/event_test.go
index a12945a471..2be357ba2d 100644
--- a/event/event_test.go
+++ b/event/event_test.go
@@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) {
}
}
-func emptySubscriber(mux *TypeMux, types ...interface{}) {
+func emptySubscriber(mux *TypeMux) {
s := mux.Subscribe(testEvent(0))
go func() {
for range s.Chan() {
@@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) {
func BenchmarkPostConcurrent(b *testing.B) {
var mux = new(TypeMux)
defer mux.Stop()
- emptySubscriber(mux, testEvent(0))
- emptySubscriber(mux, testEvent(0))
- emptySubscriber(mux, testEvent(0))
+ emptySubscriber(mux)
+ emptySubscriber(mux)
+ emptySubscriber(mux)
var wg sync.WaitGroup
poster := func() {
diff --git a/event/filter/filter.go b/event/filter/filter.go
deleted file mode 100644
index a6fe46d6a0..0000000000
--- a/event/filter/filter.go
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// Package filter implements event filters.
-package filter
-
-import "reflect"
-
-type Filter interface {
- Compare(Filter) bool
- Trigger(data interface{})
-}
-
-type FilterEvent struct {
- filter Filter
- data interface{}
-}
-
-type Filters struct {
- id int
- watchers map[int]Filter
- ch chan FilterEvent
-
- quit chan struct{}
-}
-
-func New() *Filters {
- return &Filters{
- ch: make(chan FilterEvent),
- watchers: make(map[int]Filter),
- quit: make(chan struct{}),
- }
-}
-
-func (f *Filters) Start() {
- go f.loop()
-}
-
-func (f *Filters) Stop() {
- close(f.quit)
-}
-
-func (f *Filters) Notify(filter Filter, data interface{}) {
- f.ch <- FilterEvent{filter, data}
-}
-
-func (f *Filters) Install(watcher Filter) int {
- f.watchers[f.id] = watcher
- f.id++
-
- return f.id - 1
-}
-
-func (f *Filters) Uninstall(id int) {
- delete(f.watchers, id)
-}
-
-func (f *Filters) loop() {
-out:
- for {
- select {
- case <-f.quit:
- break out
- case event := <-f.ch:
- for _, watcher := range f.watchers {
- if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) {
- if watcher.Compare(event.filter) {
- watcher.Trigger(event.data)
- }
- }
- }
- }
- }
-}
-
-func (f *Filters) Match(a, b Filter) bool {
- return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b)
-}
-
-func (f *Filters) Get(i int) Filter {
- return f.watchers[i]
-}
diff --git a/event/filter/filter_test.go b/event/filter/filter_test.go
deleted file mode 100644
index dcc9112453..0000000000
--- a/event/filter/filter_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package filter
-
-import (
- "testing"
- "time"
-)
-
-// Simple test to check if baseline matching/mismatching filtering works.
-func TestFilters(t *testing.T) {
- fm := New()
- fm.Start()
-
- // Register two filters to catch posted data
- first := make(chan struct{})
- fm.Install(Generic{
- Str1: "hello",
- Fn: func(data interface{}) {
- first <- struct{}{}
- },
- })
- second := make(chan struct{})
- fm.Install(Generic{
- Str1: "hello1",
- Str2: "hello",
- Fn: func(data interface{}) {
- second <- struct{}{}
- },
- })
- // Post an event that should only match the first filter
- fm.Notify(Generic{Str1: "hello"}, true)
- fm.Stop()
-
- // Ensure only the mathcing filters fire
- select {
- case <-first:
- case <-time.After(100 * time.Millisecond):
- t.Error("matching filter timed out")
- }
- select {
- case <-second:
- t.Error("mismatching filter fired")
- case <-time.After(100 * time.Millisecond):
- }
-}
diff --git a/event/filter/generic_filter.go b/event/filter/generic_filter.go
deleted file mode 100644
index d679b8bfa8..0000000000
--- a/event/filter/generic_filter.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package filter
-
-type Generic struct {
- Str1, Str2, Str3 string
- Data map[string]struct{}
-
- Fn func(data interface{})
-}
-
-// self = registered, f = incoming
-func (self Generic) Compare(f Filter) bool {
- var strMatch, dataMatch = true, true
-
- filter := f.(Generic)
- if (len(self.Str1) > 0 && filter.Str1 != self.Str1) ||
- (len(self.Str2) > 0 && filter.Str2 != self.Str2) ||
- (len(self.Str3) > 0 && filter.Str3 != self.Str3) {
- strMatch = false
- }
-
- for k := range self.Data {
- if _, ok := filter.Data[k]; !ok {
- return false
- }
- }
-
- return strMatch && dataMatch
-}
-
-func (self Generic) Trigger(data interface{}) {
- self.Fn(data)
-}
diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go
index 20e82ec2a9..f8b1b43c01 100644
--- a/internal/cmdtest/test_cmd.go
+++ b/internal/cmdtest/test_cmd.go
@@ -27,6 +27,7 @@ import (
"regexp"
"strings"
"sync"
+ "syscall"
"testing"
"text/template"
"time"
@@ -50,6 +51,8 @@ type TestCmd struct {
stdout *bufio.Reader
stdin io.WriteCloser
stderr *testlogger
+ // Err will contain the process exit error or interrupt signal error
+ Err error
}
// Run exec's the current binary using name as argv[0] which will trigger the
@@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() {
}
func (tt *TestCmd) WaitExit() {
- tt.cmd.Wait()
+ tt.Err = tt.cmd.Wait()
}
func (tt *TestCmd) Interrupt() {
- tt.cmd.Process.Signal(os.Interrupt)
+ tt.Err = tt.cmd.Process.Signal(os.Interrupt)
+}
+
+// ExitStatus exposes the process' OS exit code
+// It will only return a valid value after the process has finished.
+func (tt *TestCmd) ExitStatus() int {
+ if tt.Err != nil {
+ exitErr := tt.Err.(*exec.ExitError)
+ if exitErr != nil {
+ if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+ return status.ExitStatus()
+ }
+ }
+ }
+ return 0
}
// StderrText returns any stderr output written so far.
diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go
index 0c751c3282..43a33e9921 100644
--- a/internal/ethapi/api.go
+++ b/internal/ethapi/api.go
@@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
return fetchKeystore(s.am).Lock(addr) == nil
}
-// signTransactions sets defaults and signs the given transaction
+// signTransaction sets defaults and signs the given transaction
// NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
// and release it after the transaction has been submitted to the tx pool
func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) {
diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go
index addf3c7660..a5f3196535 100644
--- a/internal/web3ext/web3ext.go
+++ b/internal/web3ext/web3ext.go
@@ -481,6 +481,12 @@ web3._extend({
params: 2,
inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex]
}),
+ new web3._extend.Method({
+ name: 'getProof',
+ call: 'eth_getProof',
+ params: 3,
+ inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter]
+ }),
],
properties: [
new web3._extend.Property({
diff --git a/les/fetcher.go b/les/fetcher.go
index f0d3b188db..2615f69df9 100644
--- a/les/fetcher.go
+++ b/les/fetcher.go
@@ -141,36 +141,39 @@ func (f *lightFetcher) syncLoop() {
s := requesting
requesting = false
var (
- rq *distReq
- reqID uint64
+ rq *distReq
+ reqID uint64
+ syncing bool
)
if !f.syncing && !(newAnnounce && s) {
- rq, reqID = f.nextRequest()
+ rq, reqID, syncing = f.nextRequest()
}
- syncing := f.syncing
f.lock.Unlock()
if rq != nil {
requesting = true
- _, ok := <-f.pm.reqDist.queue(rq)
- if !ok {
+ if _, ok := <-f.pm.reqDist.queue(rq); ok {
+ if syncing {
+ f.lock.Lock()
+ f.syncing = true
+ f.lock.Unlock()
+ } else {
+ go func() {
+ time.Sleep(softRequestTimeout)
+ f.reqMu.Lock()
+ req, ok := f.requested[reqID]
+ if ok {
+ req.timeout = true
+ f.requested[reqID] = req
+ }
+ f.reqMu.Unlock()
+ // keep starting new requests while possible
+ f.requestChn <- false
+ }()
+ }
+ } else {
f.requestChn <- false
}
-
- if !syncing {
- go func() {
- time.Sleep(softRequestTimeout)
- f.reqMu.Lock()
- req, ok := f.requested[reqID]
- if ok {
- req.timeout = true
- f.requested[reqID] = req
- }
- f.reqMu.Unlock()
- // keep starting new requests while possible
- f.requestChn <- false
- }()
- }
}
case reqID := <-f.timeoutChn:
f.reqMu.Lock()
@@ -209,6 +212,7 @@ func (f *lightFetcher) syncLoop() {
f.checkSyncedHeaders(p)
f.syncing = false
f.lock.Unlock()
+ f.requestChn <- false
}
}
}
@@ -405,7 +409,7 @@ func (f *lightFetcher) requestedID(reqID uint64) bool {
// nextRequest selects the peer and announced head to be requested next, amount
// to be downloaded starting from the head backwards is also returned
-func (f *lightFetcher) nextRequest() (*distReq, uint64) {
+func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) {
var (
bestHash common.Hash
bestAmount uint64
@@ -427,14 +431,12 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
}
}
if bestTd == f.maxConfirmedTd {
- return nil, 0
+ return nil, 0, false
}
- f.syncing = bestSyncing
-
var rq *distReq
reqID := genReqID()
- if f.syncing {
+ if bestSyncing {
rq = &distReq{
getCost: func(dp distPeer) uint64 {
return 0
@@ -500,7 +502,7 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
},
}
}
- return rq, reqID
+ return rq, reqID, bestSyncing
}
// deliverHeaders delivers header download request responses for processing
diff --git a/les/serverpool.go b/les/serverpool.go
index 0fe6e49b61..52b54b371f 100644
--- a/les/serverpool.go
+++ b/les/serverpool.go
@@ -683,7 +683,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
}
func encodePubkey64(pub *ecdsa.PublicKey) []byte {
- return crypto.FromECDSAPub(pub)[:1]
+ return crypto.FromECDSAPub(pub)[1:]
}
func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {
diff --git a/light/postprocess.go b/light/postprocess.go
index 1cfd7535e7..dd1b74a7be 100644
--- a/light/postprocess.go
+++ b/light/postprocess.go
@@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co
diskdb: db,
odr: odr,
trieTable: trieTable,
- triedb: trie.NewDatabase(trieTable),
+ triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
sectionSize: size,
}
return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht")
@@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
diskdb: db,
odr: odr,
trieTable: trieTable,
- triedb: trie.NewDatabase(trieTable),
+ triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
parentSize: parentSize,
size: size,
}
diff --git a/light/trie.go b/light/trie.go
index c07e99461c..ab4e18b43e 100644
--- a/light/trie.go
+++ b/light/trie.go
@@ -108,7 +108,7 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) {
func (t *odrTrie) TryUpdate(key, value []byte) error {
key = crypto.Keccak256(key)
return t.do(key, func() error {
- return t.trie.TryDelete(key)
+ return t.trie.TryUpdate(key, value)
})
}
diff --git a/metrics/registry.go b/metrics/registry.go
index cc34c9dfd2..c1cf7906ce 100644
--- a/metrics/registry.go
+++ b/metrics/registry.go
@@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() {
r.underlying.UnregisterAll()
}
-var DefaultRegistry Registry = NewRegistry()
+var (
+ DefaultRegistry = NewRegistry()
+ EphemeralRegistry = NewRegistry()
+)
// Call the given function for each registered metric.
func Each(f func(string, interface{})) {
diff --git a/miner/stress_clique.go b/miner/stress_clique.go
index 8961091d56..7e19975aec 100644
--- a/miner/stress_clique.go
+++ b/miner/stress_clique.go
@@ -22,7 +22,6 @@ package main
import (
"bytes"
"crypto/ecdsa"
- "fmt"
"io/ioutil"
"math/big"
"math/rand"
@@ -40,7 +39,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
- "github.com/ethereum/go-ethereum/p2p/discover"
+ "github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params"
)
@@ -62,11 +61,11 @@ func main() {
var (
nodes []*node.Node
- enodes []string
+ enodes []*enode.Node
)
for _, sealer := range sealers {
// Start the node and wait until it's up
- node, err := makeSealer(genesis, enodes)
+ node, err := makeSealer(genesis)
if err != nil {
panic(err)
}
@@ -76,18 +75,12 @@ func main() {
time.Sleep(250 * time.Millisecond)
}
// Connect the node to al the previous ones
- for _, enode := range enodes {
- enode, err := discover.ParseNode(enode)
- if err != nil {
- panic(err)
- }
- node.Server().AddPeer(enode)
+ for _, n := range enodes {
+ node.Server().AddPeer(n)
}
- // Start tracking the node and it's enode url
+ // Start tracking the node and it's enode
nodes = append(nodes, node)
-
- enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
- enodes = append(enodes, enode)
+ enodes = append(enodes, node.Server().Self())
// Inject the signer key and start sealing with it
store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core
return genesis
}
-func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeSealer(genesis *core.Genesis) (*node.Node, error) {
// Define the basic configurations for the Ethereum node
datadir, _ := ioutil.TempDir("", "")
diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go
index 5ed11d73a5..044ca9a218 100644
--- a/miner/stress_ethash.go
+++ b/miner/stress_ethash.go
@@ -21,7 +21,6 @@ package main
import (
"crypto/ecdsa"
- "fmt"
"io/ioutil"
"math/big"
"math/rand"
@@ -41,7 +40,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
- "github.com/ethereum/go-ethereum/p2p/discover"
+ "github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params"
)
@@ -62,11 +61,11 @@ func main() {
var (
nodes []*node.Node
- enodes []string
+ enodes []*enode.Node
)
for i := 0; i < 4; i++ {
// Start the node and wait until it's up
- node, err := makeMiner(genesis, enodes)
+ node, err := makeMiner(genesis)
if err != nil {
panic(err)
}
@@ -76,18 +75,12 @@ func main() {
time.Sleep(250 * time.Millisecond)
}
// Connect the node to al the previous ones
- for _, enode := range enodes {
- enode, err := discover.ParseNode(enode)
- if err != nil {
- panic(err)
- }
- node.Server().AddPeer(enode)
+ for _, n := range enodes {
+ node.Server().AddPeer(n)
}
- // Start tracking the node and it's enode url
+ // Start tracking the node and it's enode
nodes = append(nodes, node)
-
- enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
- enodes = append(enodes, enode)
+ enodes = append(enodes, node.Server().Self())
// Inject the signer key and start sealing with it
store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis {
return genesis
}
-func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeMiner(genesis *core.Genesis) (*node.Node, error) {
// Define the basic configurations for the Ethereum node
datadir, _ := ioutil.TempDir("", "")
diff --git a/node/node.go b/node/node.go
index 85299dba74..8461008391 100644
--- a/node/node.go
+++ b/node/node.go
@@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error {
if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
return err
}
- n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace)
+ n.log.Debug("InProc registered", "namespace", api.Namespace)
}
n.inprocHandler = handler
return nil
diff --git a/p2p/discover/table.go b/p2p/discover/table.go
index afd4c9a272..9f7f1d41b9 100644
--- a/p2p/discover/table.go
+++ b/p2p/discover/table.go
@@ -434,7 +434,7 @@ func (tab *Table) loadSeedNodes() {
for i := range seeds {
seed := seeds[i]
age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID())) }}
- log.Debug("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
+ log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
tab.add(seed)
}
}
diff --git a/p2p/metrics.go b/p2p/metrics.go
index 6a7c0bad33..8df82bb074 100644
--- a/p2p/metrics.go
+++ b/p2p/metrics.go
@@ -47,8 +47,8 @@ var (
egressConnectMeter = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections
egressTrafficMeter = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil) // Meter metering the cumulative egress traffic
- PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress
- PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.DefaultRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress
+ PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress
+ PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress
meteredPeerFeed event.Feed // Event feed for peer metrics
meteredPeerCount int32 // Actually stored peer connection count
@@ -74,7 +74,7 @@ const (
type MeteredPeerEvent struct {
Type MeteredPeerEventType // Type of peer event
IP net.IP // IP address of the peer
- ID string // NodeID of the peer
+ ID enode.ID // NodeID of the peer
Elapsed time.Duration // Time elapsed between the connection and the handshake/disconnection
Ingress uint64 // Ingress count at the moment of the event
Egress uint64 // Egress count at the moment of the event
@@ -93,7 +93,7 @@ type meteredConn struct {
connected time.Time // Connection time of the peer
ip net.IP // IP address of the peer
- id string // NodeID of the peer
+ id enode.ID // NodeID of the peer
// trafficMetered denotes if the peer is registered in the traffic registries.
// Its value is true if the metered peer count doesn't reach the limit in the
@@ -160,8 +160,7 @@ func (c *meteredConn) Write(b []byte) (n int, err error) {
// handshakeDone is called when a peer handshake is done. Registers the peer to
// the ingress and the egress traffic registries using the peer's IP and node ID,
// also emits connect event.
-func (c *meteredConn) handshakeDone(nodeID enode.ID) {
- id := nodeID.String()
+func (c *meteredConn) handshakeDone(id enode.ID) {
if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit {
// Don't register the peer in the traffic registries.
atomic.AddInt32(&meteredPeerCount, -1)
@@ -170,7 +169,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) {
c.lock.Unlock()
log.Warn("Metered peer count reached the limit")
} else {
- key := fmt.Sprintf("%s/%s", c.ip, id)
+ key := fmt.Sprintf("%s/%s", c.ip, id.String())
c.lock.Lock()
c.id, c.trafficMetered = id, true
c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry)
@@ -190,7 +189,7 @@ func (c *meteredConn) handshakeDone(nodeID enode.ID) {
func (c *meteredConn) Close() error {
err := c.Conn.Close()
c.lock.RLock()
- if c.id == "" {
+ if c.id == (enode.ID{}) {
// If the peer disconnects before the handshake.
c.lock.RUnlock()
meteredPeerFeed.Send(MeteredPeerEvent{
diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go
index 2874af48d2..a26222cd8b 100644
--- a/p2p/protocols/protocol_test.go
+++ b/p2p/protocols/protocol_test.go
@@ -318,7 +318,7 @@ func TestProtocolHook(t *testing.T) {
<-testHook.waitC
time.Sleep(100 * time.Millisecond)
- err = tester.TestDisconnected(&p2ptest.Disconnect{tester.Nodes[1].ID(), testHook.err})
+ err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: tester.Nodes[1].ID(), Error: testHook.err})
if err != nil {
t.Fatalf("Expected a specific disconnect error, but got different one: %v", err)
}
diff --git a/p2p/rlpx.go b/p2p/rlpx.go
index a105720a44..22a27dd96a 100644
--- a/p2p/rlpx.go
+++ b/p2p/rlpx.go
@@ -151,7 +151,7 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake,
}
if msg.Code == discMsg {
// Disconnect before protocol handshake is valid according to the
- // spec and we send it ourself if the posthanshake checks fail.
+ // spec and we send it ourself if the post-handshake checks fail.
// We can't return the reason directly, though, because it is echoed
// back otherwise. Wrap it in a string instead.
var reason [1]DiscReason
diff --git a/p2p/server.go b/p2p/server.go
index 38a881f7bd..6678608634 100644
--- a/p2p/server.go
+++ b/p2p/server.go
@@ -228,7 +228,7 @@ type transport interface {
MsgReadWriter
// transports must provide Close because we use MsgPipe in some of
// the tests. Closing the actual network connection doesn't do
- // anything in those tests because NsgPipe doesn't use it.
+ // anything in those tests because MsgPipe doesn't use it.
close(err error)
}
diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go
index 7bdbad1b53..d9513caaa0 100644
--- a/p2p/simulations/http_test.go
+++ b/p2p/simulations/http_test.go
@@ -18,6 +18,7 @@ package simulations
import (
"context"
+ "flag"
"fmt"
"math/rand"
"net/http/httptest"
@@ -28,13 +29,26 @@ import (
"time"
"github.com/ethereum/go-ethereum/event"
+ "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/rpc"
+ colorable "github.com/mattn/go-colorable"
)
+var (
+ loglevel = flag.Int("loglevel", 2, "verbosity of logs")
+)
+
+func init() {
+ flag.Parse()
+
+ log.PrintOrigins(true)
+ log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+}
+
// testService implements the node.Service interface and provides protocols
// and APIs which are useful for testing nodes in a simulation network
type testService struct {
@@ -584,9 +598,26 @@ func TestHTTPNodeRPC(t *testing.T) {
// TestHTTPSnapshot tests creating and loading network snapshots
func TestHTTPSnapshot(t *testing.T) {
// start the server
- _, s := testHTTPServer(t)
+ network, s := testHTTPServer(t)
defer s.Close()
+ var eventsDone = make(chan struct{})
+ count := 1
+ eventsDoneChan := make(chan *Event)
+ eventSub := network.Events().Subscribe(eventsDoneChan)
+ go func() {
+ defer eventSub.Unsubscribe()
+ for event := range eventsDoneChan {
+ if event.Type == EventTypeConn && !event.Control {
+ count--
+ if count == 0 {
+ eventsDone <- struct{}{}
+ return
+ }
+ }
+ }
+ }()
+
// create a two-node network
client := NewClient(s.URL)
nodeCount := 2
@@ -620,7 +651,7 @@ func TestHTTPSnapshot(t *testing.T) {
}
states[i] = state
}
-
+ <-eventsDone
// create a snapshot
snap, err := client.CreateSnapshot()
if err != nil {
@@ -634,9 +665,23 @@ func TestHTTPSnapshot(t *testing.T) {
}
// create another network
- _, s = testHTTPServer(t)
+ network2, s := testHTTPServer(t)
defer s.Close()
client = NewClient(s.URL)
+ count = 1
+ eventSub = network2.Events().Subscribe(eventsDoneChan)
+ go func() {
+ defer eventSub.Unsubscribe()
+ for event := range eventsDoneChan {
+ if event.Type == EventTypeConn && !event.Control {
+ count--
+ if count == 0 {
+ eventsDone <- struct{}{}
+ return
+ }
+ }
+ }
+ }()
// subscribe to events so we can check them later
events := make(chan *Event, 100)
@@ -651,6 +696,7 @@ func TestHTTPSnapshot(t *testing.T) {
if err := client.LoadSnapshot(snap); err != nil {
t.Fatalf("error loading snapshot: %s", err)
}
+ <-eventsDone
// check the nodes and connection exists
net, err := client.GetNetwork()
@@ -676,6 +722,9 @@ func TestHTTPSnapshot(t *testing.T) {
if conn.Other.String() != nodes[1].ID {
t.Fatalf("expected connection to have other=%q, got other=%q", nodes[1].ID, conn.Other)
}
+ if !conn.Up {
+ t.Fatal("should be up")
+ }
// check the node states were restored
for i, node := range nodes {
diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go
index 200015ff39..92ccfde817 100644
--- a/p2p/simulations/network.go
+++ b/p2p/simulations/network.go
@@ -644,11 +644,18 @@ type NodeSnapshot struct {
// Snapshot creates a network snapshot
func (net *Network) Snapshot() (*Snapshot, error) {
+ return net.snapshot(nil, nil)
+}
+
+func (net *Network) SnapshotWithServices(addServices []string, removeServices []string) (*Snapshot, error) {
+ return net.snapshot(addServices, removeServices)
+}
+
+func (net *Network) snapshot(addServices []string, removeServices []string) (*Snapshot, error) {
net.lock.Lock()
defer net.lock.Unlock()
snap := &Snapshot{
Nodes: make([]NodeSnapshot, len(net.Nodes)),
- Conns: make([]Conn, len(net.Conns)),
}
for i, node := range net.Nodes {
snap.Nodes[i] = NodeSnapshot{Node: *node}
@@ -660,9 +667,40 @@ func (net *Network) Snapshot() (*Snapshot, error) {
return nil, err
}
snap.Nodes[i].Snapshots = snapshots
+ for _, addSvc := range addServices {
+ haveSvc := false
+ for _, svc := range snap.Nodes[i].Node.Config.Services {
+ if svc == addSvc {
+ haveSvc = true
+ break
+ }
+ }
+ if !haveSvc {
+ snap.Nodes[i].Node.Config.Services = append(snap.Nodes[i].Node.Config.Services, addSvc)
+ }
+ }
+ if len(removeServices) > 0 {
+ var cleanedServices []string
+ for _, svc := range snap.Nodes[i].Node.Config.Services {
+ haveSvc := false
+ for _, rmSvc := range removeServices {
+ if rmSvc == svc {
+ haveSvc = true
+ break
+ }
+ }
+ if !haveSvc {
+ cleanedServices = append(cleanedServices, svc)
+ }
+
+ }
+ snap.Nodes[i].Node.Config.Services = cleanedServices
+ }
}
- for i, conn := range net.Conns {
- snap.Conns[i] = *conn
+ for _, conn := range net.Conns {
+ if conn.Up {
+ snap.Conns = append(snap.Conns, *conn)
+ }
}
return snap, nil
}
diff --git a/params/config.go b/params/config.go
index ba719b873b..3570cb204f 100644
--- a/params/config.go
+++ b/params/config.go
@@ -49,10 +49,10 @@ var (
// MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network.
MainnetTrustedCheckpoint = &TrustedCheckpoint{
Name: "mainnet",
- SectionIndex: 195,
- SectionHead: common.HexToHash("0x1cdd2a84cf6c1261ffccc88f6bcefb513abd7934a96c1e909fbf74767560f16b"),
- CHTRoot: common.HexToHash("0xe453333c20391d16b91b6fe11c104704f62c8dba15f69db73b4cdf7e100105eb"),
- BloomRoot: common.HexToHash("0x47f30069473072e00d2cdca146dce40f0aad243dfc8221bf810822c091674efe"),
+ SectionIndex: 203,
+ SectionHead: common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"),
+ CHTRoot: common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"),
+ BloomRoot: common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"),
}
// TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network.
@@ -73,10 +73,10 @@ var (
// TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network.
TestnetTrustedCheckpoint = &TrustedCheckpoint{
Name: "testnet",
- SectionIndex: 126,
- SectionHead: common.HexToHash("0x48f7dd4c9c60be04bf15fd4d0bcac46ddd8caf6b01d6fb8f8e1f7955cdd1337a"),
- CHTRoot: common.HexToHash("0x6e54cb80a1884881ea1a114243af9012c95e0296b47f103b5ab124313968508e"),
- BloomRoot: common.HexToHash("0xb55accf6dce6455b47db8510d15eff38d0ed7378829f3036d26b48e7d15da3f6"),
+ SectionIndex: 134,
+ SectionHead: common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"),
+ CHTRoot: common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"),
+ BloomRoot: common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"),
}
// RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network.
@@ -100,10 +100,10 @@ var (
// RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network.
RinkebyTrustedCheckpoint = &TrustedCheckpoint{
Name: "rinkeby",
- SectionIndex: 93,
- SectionHead: common.HexToHash("0xdefb94aa217ab38f2919f7318d1d5476bd2aabf1ec9148047fe03e555615e0b4"),
- CHTRoot: common.HexToHash("0x52c98c2fe508a8332c27dc10538f3fead43306e2b22b597587763c2fe6586da6"),
- BloomRoot: common.HexToHash("0x93d83be0c1b12f732b1a027ecdfb16f39b0d020b8c10bfb90e76f3b01adfc5b6"),
+ SectionIndex: 100,
+ SectionHead: common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"),
+ CHTRoot: common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"),
+ BloomRoot: common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"),
}
// AllEthashProtocolChanges contains every protocol change (EIPs) introduced
@@ -111,16 +111,16 @@ var (
//
// This configuration is intentionally not using keyed fields to force anyone
// adding flags to the config to also have to set these fields.
- AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
+ AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
// AllCliqueProtocolChanges contains every protocol change (EIPs) introduced
// and accepted by the Ethereum core developers into the Clique consensus.
//
// This configuration is intentionally not using keyed fields to force anyone
// adding flags to the config to also have to set these fields.
- AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
+ AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
- TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
+ TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
TestRules = TestChainConfig.Rules(new(big.Int))
)
diff --git a/params/version.go b/params/version.go
index 9e2f540c32..b9dcc2a842 100644
--- a/params/version.go
+++ b/params/version.go
@@ -23,7 +23,7 @@ import (
const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 8 // Minor version component of the current release
- VersionPatch = 18 // Patch version component of the current release
+ VersionPatch = 19 // Patch version component of the current release
VersionMeta = "unstable" // Version metadata to append to the version string
)
diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go
index 9c21c12d5d..3bb8717b80 100644
--- a/rpc/client_example_test.go
+++ b/rpc/client_example_test.go
@@ -25,7 +25,7 @@ import (
"github.com/ethereum/go-ethereum/rpc"
)
-// In this example, our client whishes to track the latest 'block number'
+// In this example, our client wishes to track the latest 'block number'
// known to the server. The server supports two methods:
//
// eth_getBlockByNumber("latest", {})
diff --git a/rpc/doc.go b/rpc/doc.go
index 9a6c4abbce..c60381b5ae 100644
--- a/rpc/doc.go
+++ b/rpc/doc.go
@@ -32,7 +32,7 @@ An example method:
func (s *CalcService) Add(a, b int) (int, error)
When the returned error isn't nil the returned integer is ignored and the error is
-send back to the client. Otherwise the returned integer is send back to the client.
+sent back to the client. Otherwise the returned integer is sent back to the client.
Optional arguments are supported by accepting pointer values as arguments. E.g.
if we want to do the addition in an optional finite field we can accept a mod
diff --git a/signer/core/abihelper_test.go b/signer/core/abihelper_test.go
index 2afeec73e6..878210be1f 100644
--- a/signer/core/abihelper_test.go
+++ b/signer/core/abihelper_test.go
@@ -38,13 +38,13 @@ func verify(t *testing.T, jsondata, calldata string, exp []interface{}) {
cd := common.Hex2Bytes(calldata)
sigdata, argdata := cd[:4], cd[4:]
method, err := abispec.MethodById(sigdata)
-
if err != nil {
t.Fatal(err)
}
-
data, err := method.Inputs.UnpackValues(argdata)
-
+ if err != nil {
+ t.Fatal(err)
+ }
if len(data) != len(exp) {
t.Fatalf("Mismatched length, expected %d, got %d", len(exp), len(data))
}
diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go
index 55614077ca..0b520a15bf 100644
--- a/signer/rules/rules_test.go
+++ b/signer/rules/rules_test.go
@@ -151,7 +151,7 @@ func TestListRequest(t *testing.T) {
t.Errorf("Couldn't create evaluator %v", err)
return
}
- resp, err := r.ApproveListing(&core.ListRequest{
+ resp, _ := r.ApproveListing(&core.ListRequest{
Accounts: accs,
Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"},
})
@@ -515,7 +515,7 @@ func TestLimitWindow(t *testing.T) {
r.OnApprovedTx(response)
}
// Fourth should fail
- resp, err := r.ApproveTx(dummyTx(h))
+ resp, _ := r.ApproveTx(dummyTx(h))
if resp.Approved {
t.Errorf("Expected check to resolve to 'Reject'")
}
@@ -609,8 +609,8 @@ func TestContextIsCleared(t *testing.T) {
t.Fatalf("Failed to load bootstrap js: %v", err)
}
tx := dummyTxWithV(0)
- r1, err := r.ApproveTx(tx)
- r2, err := r.ApproveTx(tx)
+ r1, _ := r.ApproveTx(tx)
+ r2, _ := r.ApproveTx(tx)
if r1.Approved != r2.Approved {
t.Errorf("Expected execution context to be cleared between executions")
}
diff --git a/swarm/api/act.go b/swarm/api/act.go
index 52d9098271..e54369f9a8 100644
--- a/swarm/api/act.go
+++ b/swarm/api/act.go
@@ -458,6 +458,9 @@ func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees
return nil, nil, nil, err
}
sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt)
+ if err != nil {
+ return nil, nil, nil, err
+ }
hasher := sha3.NewKeccak256()
hasher.Write(append(sessionKey, 0))
diff --git a/swarm/api/api.go b/swarm/api/api.go
index 7bb631967c..099a469396 100644
--- a/swarm/api/api.go
+++ b/swarm/api/api.go
@@ -472,7 +472,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
// no entry found
status = http.StatusNotFound
apiGetNotFound.Inc(1)
- err = fmt.Errorf("manifest entry for '%s' not found", path)
+ err = fmt.Errorf("Not found: could not find resource '%s'", path)
log.Trace("manifest entry not found", "key", contentAddr, "path", path)
}
return
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index 03c6cbb28c..76b3493978 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -33,10 +33,9 @@ import (
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
- "github.com/ethereum/go-ethereum/swarm/testutil"
)
-func serverFunc(api *api.API) testutil.TestServer {
+func serverFunc(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}
@@ -49,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) {
}
func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
client := NewClient(srv.URL)
@@ -90,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
}
func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
client := NewClient(srv.URL)
@@ -188,7 +187,7 @@ func newTestDirectory(t *testing.T) string {
// TestClientUploadDownloadDirectory tests uploading and downloading a
// directory of files to a swarm manifest
func TestClientUploadDownloadDirectory(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
dir := newTestDirectory(t)
@@ -254,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) {
}
func testClientFileList(toEncrypt bool, t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
dir := newTestDirectory(t)
@@ -312,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
// TestClientMultipartUpload tests uploading files to swarm using a multipart
// upload
func TestClientMultipartUpload(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// define an uploader which uploads testDirFiles with some data
@@ -378,7 +377,7 @@ func TestClientCreateFeedMultihash(t *testing.T) {
signer, _ := newTestSigner()
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
client := NewClient(srv.URL)
defer srv.Close()
@@ -440,7 +439,7 @@ func TestClientCreateUpdateFeed(t *testing.T) {
signer, _ := newTestSigner()
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
client := NewClient(srv.URL)
defer srv.Close()
@@ -457,6 +456,9 @@ func TestClientCreateUpdateFeed(t *testing.T) {
}
feedManifestHash, err := client.CreateFeedWithManifest(createRequest)
+ if err != nil {
+ t.Fatal(err)
+ }
correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882"
if feedManifestHash != correctManifestAddrHex {
diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go
index 43695efc1e..266ef71bec 100644
--- a/swarm/api/filesystem.go
+++ b/swarm/api/filesystem.go
@@ -122,6 +122,10 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error
var wait func(context.Context) error
ctx := context.TODO()
hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt)
+ if err != nil {
+ errors[i] = err
+ return
+ }
if hash != nil {
list[i].Hash = hash.Hex()
}
diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go
index ccc040c54d..f5f70138b3 100644
--- a/swarm/api/http/middleware.go
+++ b/swarm/api/http/middleware.go
@@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writer := newLoggingResponseWriter(w)
h.ServeHTTP(writer, r)
- log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
+ log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
})
}
diff --git a/swarm/api/http/response_test.go b/swarm/api/http/response_test.go
index 50a704be6d..486c19ab0e 100644
--- a/swarm/api/http/response_test.go
+++ b/swarm/api/http/response_test.go
@@ -24,12 +24,10 @@ import (
"testing"
"golang.org/x/net/html"
-
- "github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestError(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@@ -55,7 +53,7 @@ func TestError(t *testing.T) {
}
func Test404Page(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@@ -81,7 +79,7 @@ func Test404Page(t *testing.T) {
}
func Test500Page(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@@ -106,7 +104,7 @@ func Test500Page(t *testing.T) {
}
}
func Test500PageWith0xHashPrefix(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) {
}
func TestJsonResponse(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index e9005104ee..3c6735a73e 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -366,7 +366,7 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api
}
var size int64
- var reader io.Reader = part
+ var reader io.Reader
if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
size, err = strconv.ParseInt(contentLength, 10, 64)
if err != nil {
@@ -484,7 +484,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
return
}
- if updateRequest.IsUpdate() {
+ switch {
+ case updateRequest.IsUpdate():
// Verify that the signature is intact and that the signer is authorized
// to update this feed
// Check this early, to avoid creating a feed and then not being able to set its first update.
@@ -497,9 +498,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
- }
-
- if query.Get("manifest") == "1" {
+ fallthrough
+ case query.Get("manifest") == "1":
// we create a manifest so we can retrieve feed updates with bzz:// later
// this manifest has a special "feed type" manifest, and saves the
// feed identification used to retrieve feed updates later
@@ -519,6 +519,8 @@ func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, string(outdata))
w.Header().Add("Content-type", "application/json")
+ default:
+ respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest)
}
}
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 1cf7ff5770..1ef3deece2 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -20,7 +20,6 @@ import (
"archive/tar"
"bytes"
"context"
- "crypto/rand"
"encoding/json"
"errors"
"flag"
@@ -58,7 +57,7 @@ func init() {
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
-func serverFunc(api *api.API) testutil.TestServer {
+func serverFunc(api *api.API) TestServer {
return NewServer(api, "")
}
@@ -79,7 +78,7 @@ func TestBzzFeedMultihash(t *testing.T) {
signer, _ := newTestSigner()
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// add the data our multihash aliased manifest will point to
@@ -167,26 +166,19 @@ func TestBzzFeedMultihash(t *testing.T) {
// Test Swarm feeds using the raw update methods
func TestBzzFeed(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
signer, _ := newTestSigner()
defer srv.Close()
// data of update 1
- update1Data := make([]byte, 666)
+ update1Data := testutil.RandomBytes(1, 666)
update1Timestamp := srv.CurrentTime
- _, err := rand.Read(update1Data)
- if err != nil {
- t.Fatal(err)
- }
//data for update 2
update2Data := []byte("foo")
topic, _ := feed.NewTopic("foo.eth", nil)
updateRequest := feed.NewFirstRequest(topic)
- if err != nil {
- t.Fatal(err)
- }
updateRequest.SetData(update1Data)
if err := updateRequest.Sign(signer); err != nil {
@@ -263,7 +255,7 @@ func TestBzzFeed(t *testing.T) {
if resp.StatusCode == http.StatusOK {
t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK")
}
- b, err = ioutil.ReadAll(resp.Body)
+ _, err = ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
@@ -333,15 +325,45 @@ func TestBzzFeed(t *testing.T) {
}
urlQuery = testUrl.Query()
body = updateRequest.AppendValues(urlQuery) // this adds all query parameters
+ goodQueryParameters := urlQuery.Encode() // save the query parameters for a second attempt
+
+ // create bad query parameters in which the signature is missing
+ urlQuery.Del("signature")
testUrl.RawQuery = urlQuery.Encode()
+ // 1st attempt with bad query parameters in which the signature is missing
resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- t.Fatalf("Update returned %s", resp.Status)
+ expectedCode := http.StatusBadRequest
+ if resp.StatusCode != expectedCode {
+ t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode)
+ }
+
+ // 2nd attempt with bad query parameters in which the signature is of incorrect length
+ urlQuery.Set("signature", "0xabcd") // should be 130 hex chars
+ resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ expectedCode = http.StatusBadRequest
+ if resp.StatusCode != expectedCode {
+ t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode)
+ }
+
+ // 3rd attempt, with good query parameters:
+ testUrl.RawQuery = goodQueryParameters
+ resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ expectedCode = http.StatusOK
+ if resp.StatusCode != expectedCode {
+ t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode)
}
// get latest update through bzz-feed directly
@@ -420,7 +442,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
addr := [3]storage.Address{}
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
for i, mf := range testmanifest {
@@ -461,6 +483,9 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
}
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("Error while reading response body: %v", err)
+ }
if string(respbody) != testmanifest[v] {
isexpectedfailrequest := false
@@ -655,7 +680,7 @@ func TestBzzTar(t *testing.T) {
}
func testBzzTar(encrypted bool, t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"}
fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
@@ -790,7 +815,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) {
}
func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// create a manifest with some data at the root path
@@ -845,7 +870,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
}
func TestMethodsNotAllowed(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
databytes := "bar"
for _, c := range []struct {
@@ -904,7 +929,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string
}
func TestGet(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
for _, testCase := range []struct {
@@ -987,7 +1012,7 @@ func TestGet(t *testing.T) {
}
func TestModify(t *testing.T) {
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
swarmClient := swarm.NewClient(srv.URL)
@@ -1088,7 +1113,7 @@ func TestMultiPartUpload(t *testing.T) {
// POST /bzz:/ Content-Type: multipart/form-data
verbose := false
// Setup Swarm
- srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
+ srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
url := fmt.Sprintf("%s/bzz:/", srv.URL)
@@ -1119,7 +1144,7 @@ func TestMultiPartUpload(t *testing.T) {
// TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver
func TestBzzGetFileWithResolver(t *testing.T) {
resolver := newTestResolveValidator("")
- srv := testutil.NewTestSwarmServer(t, serverFunc, resolver)
+ srv := NewTestSwarmServer(t, serverFunc, resolver)
defer srv.Close()
fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"}
fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
diff --git a/swarm/testutil/http.go b/swarm/api/http/test_server.go
similarity index 99%
rename from swarm/testutil/http.go
rename to swarm/api/http/test_server.go
index cdf9239d06..9245c9c5b8 100644
--- a/swarm/testutil/http.go
+++ b/swarm/api/http/test_server.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see .
-package testutil
+package http
import (
"io/ioutil"
diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go
index 7c4cc88e47..890ed88bd4 100644
--- a/swarm/api/manifest.go
+++ b/swarm/api/manifest.go
@@ -557,7 +557,6 @@ func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manif
if path != entry.Path {
return nil, 0
}
- pos = epl
}
}
return nil, 0
diff --git a/swarm/bmt/bmt_test.go b/swarm/bmt/bmt_test.go
index 760aa11d8b..683ba4f5b2 100644
--- a/swarm/bmt/bmt_test.go
+++ b/swarm/bmt/bmt_test.go
@@ -18,10 +18,8 @@ package bmt
import (
"bytes"
- crand "crypto/rand"
"encoding/binary"
"fmt"
- "io"
"math/rand"
"sync"
"sync/atomic"
@@ -29,6 +27,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/crypto/sha3"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
// the actual data length generated (could be longer than max datalength of the BMT)
@@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) {
})
// run the tests
- for _, x := range tests {
+ for i, x := range tests {
for segmentCount := x.from; segmentCount <= x.to; segmentCount++ {
for length := 1; length <= segmentCount*32; length++ {
t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) {
- data := make([]byte, length)
- if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF {
- t.Fatal(err)
- }
+ data := testutil.RandomBytes(i, length)
expected := x.expected(data)
actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data)
if !bytes.Equal(actual, expected) {
@@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) {
// tests sequential write with entire max size written in one go
func TestSyncHasherCorrectness(t *testing.T) {
- data := newData(BufferSize)
+ data := testutil.RandomBytes(1, BufferSize)
hasher := sha3.NewKeccak256
size := hasher().Size()
@@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) {
// tests order-neutral concurrent writes with entire max size written in one go
func TestAsyncCorrectness(t *testing.T) {
- data := newData(BufferSize)
+ data := testutil.RandomBytes(1, BufferSize)
hasher := sha3.NewKeccak256
size := hasher().Size()
whs := []whenHash{first, last, random}
@@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) {
bmt := New(pool)
for i := 0; i < 100; i++ {
- data := newData(BufferSize)
+ data := testutil.RandomBytes(1, BufferSize)
n := rand.Intn(bmt.Size())
err := testHasherCorrectness(bmt, hasher, data, n, segmentCount)
if err != nil {
@@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) {
for i := 0; i < cycles; i++ {
go func() {
bmt := New(pool)
- data := newData(BufferSize)
+ data := testutil.RandomBytes(1, BufferSize)
n := rand.Intn(bmt.Size())
errc <- testHasherCorrectness(bmt, hasher, data, n, 128)
}()
@@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) {
defer pool.Drain(0)
n := count * 32
bmt := New(pool)
- data := newData(n)
+ data := testutil.RandomBytes(1, n)
rbmt := NewRefHasher(hasher, count)
refHash := rbmt.Hash(data)
expHash := syncHash(bmt, nil, data)
@@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) {
// benchmarks simple sha3 hash on chunks
func benchmarkSHA3(t *testing.B, n int) {
- data := newData(n)
+ data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
h := hasher()
@@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) {
func benchmarkBMTBaseline(t *testing.B, n int) {
hasher := sha3.NewKeccak256
hashSize := hasher().Size()
- data := newData(hashSize)
+ data := testutil.RandomBytes(1, hashSize)
t.ReportAllocs()
t.ResetTimer()
@@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) {
// benchmarks BMT Hasher
func benchmarkBMT(t *testing.B, n int) {
- data := newData(n)
+ data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
bmt := New(pool)
@@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) {
// benchmarks BMT hasher with asynchronous concurrent segment/section writes
func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
- data := newData(n)
+ data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
bmt := New(pool).NewAsyncWriter(double)
idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
- shuffle(len(idxs), func(i int, j int) {
+ rand.Shuffle(len(idxs), func(i int, j int) {
idxs[i], idxs[j] = idxs[j], idxs[i]
})
@@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
// benchmarks 100 concurrent bmt hashes with pool capacity
func benchmarkPool(t *testing.B, poolsize, n int) {
- data := newData(n)
+ data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
pool := NewTreePool(hasher, segmentCount, poolsize)
cycles := 100
@@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) {
// benchmarks the reference hasher
func benchmarkRefHasher(t *testing.B, n int) {
- data := newData(n)
+ data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
rbmt := NewRefHasher(hasher, 128)
@@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) {
}
}
-func newData(bufferSize int) []byte {
- data := make([]byte, bufferSize)
- _, err := io.ReadFull(crand.Reader, data)
- if err != nil {
- panic(err.Error())
- }
- return data
-}
-
// Hash hashes the data and the span using the bmt hasher
func syncHash(h *Hasher, span, data []byte) []byte {
h.ResetWithLength(span)
@@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) {
section := data[i*secsize : end]
segments = append(segments, section)
}
- shuffle(n, func(i int, j int) {
+ rand.Shuffle(n, func(i int, j int) {
idxs[i], idxs[j] = idxs[j], idxs[i]
})
return idxs, segments
@@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s
}
return <-c
}
-
-// this is also in swarm/network_test.go
-// shuffle pseudo-randomizes the order of elements.
-// n is the number of elements. Shuffle panics if n < 0.
-// swap swaps the elements with indexes i and j.
-func shuffle(n int, swap func(i, j int)) {
- if n < 0 {
- panic("invalid argument to Shuffle")
- }
-
- // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
- // Shuffle really ought not be called with n that doesn't fit in 32 bits.
- // Not only will it take a very long time, but with 2³¹! possible permutations,
- // there's no way that any PRNG can have a big enough internal state to
- // generate even a minuscule percentage of the possible permutations.
- // Nevertheless, the right API signature accepts an int n, so handle it as best we can.
- i := n - 1
- for ; i > 1<<31-1-1; i-- {
- j := int(rand.Int63n(int64(i + 1)))
- swap(i, j)
- }
- for ; i > 0; i-- {
- j := int(rand.Int31n(int32(i + 1)))
- swap(i, j)
- }
-}
diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go
index 87d9185505..460e31c4e9 100644
--- a/swarm/fuse/swarmfs_test.go
+++ b/swarm/fuse/swarmfs_test.go
@@ -20,20 +20,19 @@ package fuse
import (
"bytes"
- "crypto/rand"
"flag"
"fmt"
"io"
"io/ioutil"
+ "math/rand"
"os"
"path/filepath"
"testing"
+ "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage"
-
- "github.com/ethereum/go-ethereum/log"
-
+ "github.com/ethereum/go-ethereum/swarm/testutil"
colorable "github.com/mattn/go-colorable"
)
@@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) {
}
}
-func getRandomBytes(size int) []byte {
- contents := make([]byte, size)
- rand.Read(contents)
- return contents
-}
-
func isDirEmpty(name string) bool {
f, err := os.Open(name)
if err != nil {
@@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) {
dat.testMountDir = filepath.Join(dat.testDir, "testMountDir")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)}
- dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)}
- dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)}
- dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)}
- dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)}
- dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)}
- dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)}
- dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
- dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
- dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
- dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
- dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)}
- dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)}
- dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)}
- dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)}
+ dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)}
+ dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)}
+ dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)}
+ dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)}
+ dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)}
+ dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)}
+ dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)}
+ dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)}
+ dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)}
+ dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)}
+ dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)}
+ dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)}
+ dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount1")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount2")
- dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount3")
- dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount4")
- dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount5")
- dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
if err != nil {
t.Fatalf("Couldn't create upload dir 6: %v", err)
}
- dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
testMountDir6 := filepath.Join(dat.testDir, "max-mount6")
err = os.MkdirAll(testMountDir6, 0777)
if err != nil {
@@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
}
// mount a different hash in already mounted point
- dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2")
if err3 != nil {
t.Fatalf("Error creating second upload dir: %v", err3)
@@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
//we need to manually close the file before mount for this test
//but let's defer too in case of errors
defer d.Close()
- _, err = d.Write(getRandomBytes(10))
+ _, err = d.Write(testutil.RandomBytes(1, 10))
if err != nil {
t.Fatalf("Couldn't write to file: %v", err)
}
@@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "seek-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "create-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
}
defer d.Close()
log.Debug("Opened file")
- contents := make([]byte, 11)
- _, err = rand.Read(contents)
- if err != nil {
- t.Fatalf("Could not rand read contents %v", err)
- }
+ contents := testutil.RandomBytes(1, 11)
log.Debug("content read")
_, err = d.Write(contents)
if err != nil {
@@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount")
dat.files = make(map[string]fileInfo)
- dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
}
defer d.Close()
log.Debug("File opened")
- contents := make([]byte, 11)
- _, err = rand.Read(contents)
- if err != nil {
- t.Fatalf("Error filling random bytes into byte array %v", err)
- }
+ contents := testutil.RandomBytes(1, 11)
log.Debug("Content read")
_, err = d.Write(contents)
if err != nil {
@@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
}
defer d.Close()
log.Debug("File opened")
- contents := make([]byte, 11)
- _, err = rand.Read(contents)
- if err != nil {
- t.Fatalf("Error writing random bytes to byte array: %v", err)
- }
+ contents := testutil.RandomBytes(1, 11)
log.Debug("content read")
_, err = d.Write(contents)
if err != nil {
@@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload")
dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
}
defer d.Close()
log.Debug("file opened")
- contents := make([]byte, 11)
- _, err = rand.Read(contents)
- if err != nil {
- t.Fatalf("Error writing random bytes to byte array: %v", err)
- }
+ contents := testutil.RandomBytes(1, 11)
log.Debug("content read")
_, err = d.Write(contents)
if err != nil {
@@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload")
dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
dat.files = make(map[string]fileInfo)
- dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
dat.files = make(map[string]fileInfo)
- dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount")
dat.files = make(map[string]fileInfo)
- dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
- dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
+ dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
+ dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
+ dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
+ dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)}
+ dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)}
+ dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount")
dat.files = make(map[string]fileInfo)
- line1 := make([]byte, 10)
- _, err = rand.Read(line1)
- if err != nil {
- t.Fatalf("Error writing random bytes to byte array: %v", err)
- }
+ line1 := testutil.RandomBytes(1, 10)
dat.files["1.txt"] = fileInfo{0700, 333, 444, line1}
@@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
}
defer fd.Close()
log.Debug("file opened")
- line2 := make([]byte, 5)
- _, err = rand.Read(line2)
- if err != nil {
- t.Fatalf("Error writing random bytes to byte array: %v", err)
- }
+ line2 := testutil.RandomBytes(1, 5)
log.Debug("line read")
_, err = fd.Seek(int64(len(line1)), 0)
if err != nil {
diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go
index 059c3dc96d..56adc5a8e9 100644
--- a/swarm/network/hive_test.go
+++ b/swarm/network/hive_test.go
@@ -70,6 +70,9 @@ func TestHiveStatePersistance(t *testing.T) {
defer os.RemoveAll(dir)
store, err := state.NewDBStore(dir) //start the hive with an empty dbstore
+ if err != nil {
+ t.Fatal(err)
+ }
params := NewHiveParams()
s, pp := newHiveTester(t, params, 5, store)
@@ -90,6 +93,9 @@ func TestHiveStatePersistance(t *testing.T) {
store.Close()
persistedStore, err := state.NewDBStore(dir) //start the hive with an empty dbstore
+ if err != nil {
+ t.Fatal(err)
+ }
s1, pp := newHiveTester(t, params, 1, persistedStore)
diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go
index 55a0c6f13d..cd94741be3 100644
--- a/swarm/network/kademlia.go
+++ b/swarm/network/kademlia.go
@@ -261,7 +261,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) {
// found among live peers, do nothing
return v
})
- if ins {
+ if ins && !p.BzzPeer.LightNode {
a := newEntry(p.BzzAddr)
a.conn = p
// insert new online peer into addrs
@@ -329,14 +329,18 @@ func (k *Kademlia) Off(p *Peer) {
k.lock.Lock()
defer k.lock.Unlock()
var del bool
- k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val {
- // v cannot be nil, must check otherwise we overwrite entry
- if v == nil {
- panic(fmt.Sprintf("connected peer not found %v", p))
- }
+ if !p.BzzPeer.LightNode {
+ k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val {
+ // v cannot be nil, must check otherwise we overwrite entry
+ if v == nil {
+ panic(fmt.Sprintf("connected peer not found %v", p))
+ }
+ del = true
+ return newEntry(p.BzzAddr)
+ })
+ } else {
del = true
- return newEntry(p.BzzAddr)
- })
+ }
if del {
k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val {
diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go
index 903c8dbdac..d2e051f457 100644
--- a/swarm/network/kademlia_test.go
+++ b/swarm/network/kademlia_test.go
@@ -46,19 +46,19 @@ func newTestKademlia(b string) *Kademlia {
return NewKademlia(base, params)
}
-func newTestKadPeer(k *Kademlia, s string) *Peer {
- return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k)
+func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer {
+ return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k)
}
func On(k *Kademlia, ons ...string) {
for _, s := range ons {
- k.On(newTestKadPeer(k, s))
+ k.On(newTestKadPeer(k, s, false))
}
}
func Off(k *Kademlia, offs ...string) {
for _, s := range offs {
- k.Off(newTestKadPeer(k, s))
+ k.Off(newTestKadPeer(k, s, false))
}
}
@@ -254,6 +254,56 @@ func TestSuggestPeerFindPeers(t *testing.T) {
}
+// a node should stay in the address book if it's removed from the kademlia
+func TestOffEffectingAddressBookNormalNode(t *testing.T) {
+ k := newTestKademlia("00000000")
+ // peer added to kademlia
+ k.On(newTestKadPeer(k, "01000000", false))
+ // peer should be in the address book
+ if k.addrs.Size() != 1 {
+ t.Fatal("known peer addresses should contain 1 entry")
+ }
+ // peer should be among live connections
+ if k.conns.Size() != 1 {
+ t.Fatal("live peers should contain 1 entry")
+ }
+ // remove peer from kademlia
+ k.Off(newTestKadPeer(k, "01000000", false))
+ // peer should be in the address book
+ if k.addrs.Size() != 1 {
+ t.Fatal("known peer addresses should contain 1 entry")
+ }
+ // peer should not be among live connections
+ if k.conns.Size() != 0 {
+ t.Fatal("live peers should contain 0 entry")
+ }
+}
+
+// a light node should not be in the address book
+func TestOffEffectingAddressBookLightNode(t *testing.T) {
+ k := newTestKademlia("00000000")
+ // light node peer added to kademlia
+ k.On(newTestKadPeer(k, "01000000", true))
+ // peer should not be in the address book
+ if k.addrs.Size() != 0 {
+ t.Fatal("known peer addresses should contain 0 entry")
+ }
+ // peer should be among live connections
+ if k.conns.Size() != 1 {
+ t.Fatal("live peers should contain 1 entry")
+ }
+ // remove peer from kademlia
+ k.Off(newTestKadPeer(k, "01000000", true))
+ // peer should not be in the address book
+ if k.addrs.Size() != 0 {
+ t.Fatal("known peer addresses should contain 0 entry")
+ }
+ // peer should not be among live connections
+ if k.conns.Size() != 0 {
+ t.Fatal("live peers should contain 0 entry")
+ }
+}
+
func TestSuggestPeerRetries(t *testing.T) {
k := newTestKademlia("00000000")
k.RetryInterval = int64(300 * time.Millisecond) // cycle
diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go
index cdf370f359..f0d2666288 100644
--- a/swarm/network/protocol_test.go
+++ b/swarm/network/protocol_test.go
@@ -153,17 +153,7 @@ func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *
// should test handshakes in one exchange? parallelisation
func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptest.Disconnect) error {
- var peers []enode.ID
- id := rhs.Addr.ID()
- if len(disconnects) > 0 {
- for _, d := range disconnects {
- peers = append(peers, d.Peer)
- }
- } else {
- peers = []enode.ID{id}
- }
-
- if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, id)...); err != nil {
+ if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, rhs.Addr.ID())...); err != nil {
return err
}
diff --git a/swarm/network/simulation/bucket_test.go b/swarm/network/simulation/bucket_test.go
index 461d99825c..69df19bfe4 100644
--- a/swarm/network/simulation/bucket_test.go
+++ b/swarm/network/simulation/bucket_test.go
@@ -94,7 +94,7 @@ func TestServiceBucket(t *testing.T) {
t.Fatalf("expected %q, got %q", customValue, s)
}
- v, ok = sim.NodeItem(id2, customKey)
+ _, ok = sim.NodeItem(id2, customKey)
if ok {
t.Fatal("bucket item should not be found")
}
@@ -119,7 +119,7 @@ func TestServiceBucket(t *testing.T) {
t.Fatalf("expected %q, got %q", testValue+id1.String(), s)
}
- v, ok = items[id2]
+ _, ok = items[id2]
if ok {
t.Errorf("node 2 item should not be found")
}
diff --git a/swarm/network/simulation/events.go b/swarm/network/simulation/events.go
index 594d36225c..d73c3af4ee 100644
--- a/swarm/network/simulation/events.go
+++ b/swarm/network/simulation/events.go
@@ -20,16 +20,18 @@ import (
"context"
"sync"
- "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
+ "github.com/ethereum/go-ethereum/p2p/simulations"
)
// PeerEvent is the type of the channel returned by Simulation.PeerEvents.
type PeerEvent struct {
// NodeID is the ID of node that the event is caught on.
NodeID enode.ID
+ // PeerID is the ID of the peer node that the event is caught on.
+ PeerID enode.ID
// Event is the event that is caught.
- Event *p2p.PeerEvent
+ Event *simulations.Event
// Error is the error that may have happened during event watching.
Error error
}
@@ -37,9 +39,13 @@ type PeerEvent struct {
// PeerEventsFilter defines a filter on PeerEvents to exclude messages with
// defined properties. Use PeerEventsFilter methods to set required options.
type PeerEventsFilter struct {
- t *p2p.PeerEventType
- protocol *string
- msgCode *uint64
+ eventType simulations.EventType
+
+ connUp *bool
+
+ msgReceive *bool
+ protocol *string
+ msgCode *uint64
}
// NewPeerEventsFilter returns a new PeerEventsFilter instance.
@@ -47,20 +53,48 @@ func NewPeerEventsFilter() *PeerEventsFilter {
return &PeerEventsFilter{}
}
-// Type sets the filter to only one peer event type.
-func (f *PeerEventsFilter) Type(t p2p.PeerEventType) *PeerEventsFilter {
- f.t = &t
+// Connect sets the filter to events when two nodes connect.
+func (f *PeerEventsFilter) Connect() *PeerEventsFilter {
+ f.eventType = simulations.EventTypeConn
+ b := true
+ f.connUp = &b
+ return f
+}
+
+// Drop sets the filter to events when two nodes disconnect.
+func (f *PeerEventsFilter) Drop() *PeerEventsFilter {
+ f.eventType = simulations.EventTypeConn
+ b := false
+ f.connUp = &b
+ return f
+}
+
+// ReceivedMessages sets the filter to only messages that are received.
+func (f *PeerEventsFilter) ReceivedMessages() *PeerEventsFilter {
+ f.eventType = simulations.EventTypeMsg
+ b := true
+ f.msgReceive = &b
+ return f
+}
+
+// SentMessages sets the filter to only messages that are sent.
+func (f *PeerEventsFilter) SentMessages() *PeerEventsFilter {
+ f.eventType = simulations.EventTypeMsg
+ b := false
+ f.msgReceive = &b
return f
}
// Protocol sets the filter to only one message protocol.
func (f *PeerEventsFilter) Protocol(p string) *PeerEventsFilter {
+ f.eventType = simulations.EventTypeMsg
f.protocol = &p
return f
}
// MsgCode sets the filter to only one msg code.
func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter {
+ f.eventType = simulations.EventTypeMsg
f.msgCode = &c
return f
}
@@ -80,19 +114,8 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ...
go func(id enode.ID) {
defer s.shutdownWG.Done()
- client, err := s.Net.GetNode(id).Client()
- if err != nil {
- subsWG.Done()
- eventC <- PeerEvent{NodeID: id, Error: err}
- return
- }
- events := make(chan *p2p.PeerEvent)
- sub, err := client.Subscribe(ctx, "admin", events, "peerEvents")
- if err != nil {
- subsWG.Done()
- eventC <- PeerEvent{NodeID: id, Error: err}
- return
- }
+ events := make(chan *simulations.Event)
+ sub := s.Net.Events().Subscribe(events)
defer sub.Unsubscribe()
subsWG.Done()
@@ -110,28 +133,55 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ...
case <-s.Done():
return
case e := <-events:
+ // ignore control events
+ if e.Control {
+ continue
+ }
match := len(filters) == 0 // if there are no filters match all events
for _, f := range filters {
- if f.t != nil && *f.t != e.Type {
- continue
+ if f.eventType == simulations.EventTypeConn && e.Conn != nil {
+ if *f.connUp != e.Conn.Up {
+ continue
+ }
+ // all connection filter parameters matched, break the loop
+ match = true
+ break
}
- if f.protocol != nil && *f.protocol != e.Protocol {
- continue
+ if f.eventType == simulations.EventTypeMsg && e.Msg != nil {
+ if f.msgReceive != nil && *f.msgReceive != e.Msg.Received {
+ continue
+ }
+ if f.protocol != nil && *f.protocol != e.Msg.Protocol {
+ continue
+ }
+ if f.msgCode != nil && *f.msgCode != e.Msg.Code {
+ continue
+ }
+ // all message filter parameters matched, break the loop
+ match = true
+ break
}
- if f.msgCode != nil && e.MsgCode != nil && *f.msgCode != *e.MsgCode {
- continue
+ }
+ var peerID enode.ID
+ switch e.Type {
+ case simulations.EventTypeConn:
+ peerID = e.Conn.One
+ if peerID == id {
+ peerID = e.Conn.Other
+ }
+ case simulations.EventTypeMsg:
+ peerID = e.Msg.One
+ if peerID == id {
+ peerID = e.Msg.Other
}
- // all filter parameters matched, break the loop
- match = true
- break
}
if match {
select {
- case eventC <- PeerEvent{NodeID: id, Event: e}:
+ case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Event: e}:
case <-ctx.Done():
if err := ctx.Err(); err != nil {
select {
- case eventC <- PeerEvent{NodeID: id, Error: err}:
+ case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Error: err}:
case <-s.Done():
}
}
diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go
index 84b0634b45..bacc64d53a 100644
--- a/swarm/network/simulation/example_test.go
+++ b/swarm/network/simulation/example_test.go
@@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
- "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/network/simulation"
@@ -87,7 +86,7 @@ func ExampleSimulation_PeerEvents() {
log.Error("peer event", "err", e.Error)
continue
}
- log.Info("peer event", "node", e.NodeID, "peer", e.Event.Peer, "msgcode", e.Event.MsgCode)
+ log.Info("peer event", "node", e.NodeID, "peer", e.PeerID, "type", e.Event.Type)
}
}()
}
@@ -100,7 +99,7 @@ func ExampleSimulation_PeerEvents_disconnections() {
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
go func() {
@@ -109,7 +108,7 @@ func ExampleSimulation_PeerEvents_disconnections() {
log.Error("peer drop", "err", d.Error)
continue
}
- log.Warn("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Warn("peer drop", "node", d.NodeID, "peer", d.PeerID)
}
}()
}
@@ -124,8 +123,8 @@ func ExampleSimulation_PeerEvents_multipleFilters() {
context.Background(),
sim.NodeIDs(),
// Watch when bzz messages 1 and 4 are received.
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(1),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(4),
+ simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(1),
+ simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(4),
)
go func() {
@@ -134,7 +133,7 @@ func ExampleSimulation_PeerEvents_multipleFilters() {
log.Error("bzz message", "err", m.Error)
continue
}
- log.Info("bzz message", "node", m.NodeID, "peer", m.Event.Peer)
+ log.Info("bzz message", "node", m.NodeID, "peer", m.PeerID)
}
}()
}
diff --git a/swarm/network/simulation/node_test.go b/swarm/network/simulation/node_test.go
index 086ab606f6..01346ef14d 100644
--- a/swarm/network/simulation/node_test.go
+++ b/swarm/network/simulation/node_test.go
@@ -160,6 +160,41 @@ func TestAddNodeWithService(t *testing.T) {
}
}
+func TestAddNodeMultipleServices(t *testing.T) {
+ sim := New(map[string]ServiceFunc{
+ "noop1": noopServiceFunc,
+ "noop2": noopService2Func,
+ })
+ defer sim.Close()
+
+ id, err := sim.AddNode()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ n := sim.Net.GetNode(id).Node.(*adapters.SimNode)
+ if n.Service("noop1") == nil {
+ t.Error("service noop1 not found on node")
+ }
+ if n.Service("noop2") == nil {
+ t.Error("service noop2 not found on node")
+ }
+}
+
+func TestAddNodeDuplicateServiceError(t *testing.T) {
+ sim := New(map[string]ServiceFunc{
+ "noop1": noopServiceFunc,
+ "noop2": noopServiceFunc,
+ })
+ defer sim.Close()
+
+ wantErr := "duplicate service: *simulation.noopService"
+ _, err := sim.AddNode()
+ if err.Error() != wantErr {
+ t.Errorf("got error %q, want %q", err, wantErr)
+ }
+}
+
func TestAddNodes(t *testing.T) {
sim := New(noopServiceFuncMap)
defer sim.Close()
diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go
index f6d3ce2290..e5435b9f0e 100644
--- a/swarm/network/simulation/simulation.go
+++ b/swarm/network/simulation/simulation.go
@@ -68,6 +68,10 @@ type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Se
// New creates a new Simulation instance with new
// simulations.Network initialized with provided services.
+// Services map must have unique keys as service names and
+// every ServiceFunc must return a node.Service of the unique type.
+// This restriction is required by node.Node.Start() function
+// which is used to start node.Service returned by ServiceFunc.
func New(services map[string]ServiceFunc) (s *Simulation) {
s = &Simulation{
buckets: make(map[enode.ID]*sync.Map),
@@ -76,6 +80,9 @@ func New(services map[string]ServiceFunc) (s *Simulation) {
adapterServices := make(map[string]adapters.ServiceFunc, len(services))
for name, serviceFunc := range services {
+ // Scope this variables correctly
+ // as they will be in the adapterServices[name] function accessed later.
+ name, serviceFunc := name, serviceFunc
s.serviceNames = append(s.serviceNames, name)
adapterServices[name] = func(ctx *adapters.ServiceContext) (node.Service, error) {
b := new(sync.Map)
diff --git a/swarm/network/simulation/simulation_test.go b/swarm/network/simulation/simulation_test.go
index eed09bf508..ca8599d7c0 100644
--- a/swarm/network/simulation/simulation_test.go
+++ b/swarm/network/simulation/simulation_test.go
@@ -205,3 +205,16 @@ func (t *noopService) Start(server *p2p.Server) error {
func (t *noopService) Stop() error {
return nil
}
+
+// a helper function for most basic noop service
+// of a different type then noopService to test
+// multiple services on one node.
+func noopService2Func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
+ return new(noopService2), nil, nil
+}
+
+// noopService2 is the service that does not do anything
+// but implements node.Service interface.
+type noopService2 struct {
+ noopService
+}
diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go
index 3c3affe58c..cd5456b73e 100644
--- a/swarm/network/simulations/discovery/discovery_test.go
+++ b/swarm/network/simulations/discovery/discovery_test.go
@@ -85,11 +85,12 @@ func getDbStore(nodeID string) (*state.DBStore, error) {
}
var (
- nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)")
- initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)")
- snapshotFile = flag.String("snapshot", "", "create snapshot")
- loglevel = flag.Int("loglevel", 3, "verbosity of logs")
- rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs")
+ nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)")
+ initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)")
+ snapshotFile = flag.String("snapshot", "", "path to create snapshot file in")
+ loglevel = flag.Int("loglevel", 3, "verbosity of logs")
+ rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs")
+ serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery")
)
func init() {
@@ -306,7 +307,25 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
}
if *snapshotFile != "" {
- snap, err := net.Snapshot()
+ var err error
+ var snap *simulations.Snapshot
+ if len(*serviceOverride) > 0 {
+ var addServices []string
+ var removeServices []string
+ for _, osvc := range strings.Split(*serviceOverride, ",") {
+ if strings.Index(osvc, "+") == 0 {
+ addServices = append(addServices, osvc[1:])
+ } else if strings.Index(osvc, "-") == 0 {
+ removeServices = append(removeServices, osvc[1:])
+ } else {
+ panic("stick to the rules, you know what they are")
+ }
+ }
+ snap, err = net.SnapshotWithServices(addServices, removeServices)
+ } else {
+ snap, err = net.Snapshot()
+ }
+
if err != nil {
return nil, errors.New("no shapshot dude")
}
diff --git a/swarm/network/simulations/overlay.go b/swarm/network/simulations/overlay.go
index caf7ff1f2d..284ae6398a 100644
--- a/swarm/network/simulations/overlay.go
+++ b/swarm/network/simulations/overlay.go
@@ -64,12 +64,12 @@ func init() {
type Simulation struct {
mtx sync.Mutex
- stores map[enode.ID]*state.InmemoryStore
+ stores map[enode.ID]state.Store
}
func NewSimulation() *Simulation {
return &Simulation{
- stores: make(map[enode.ID]*state.InmemoryStore),
+ stores: make(map[enode.ID]state.Store),
}
}
diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go
index 72fdb2bd96..e0a7f7e120 100644
--- a/swarm/network/stream/common_test.go
+++ b/swarm/network/stream/common_test.go
@@ -18,7 +18,6 @@ package stream
import (
"context"
- crand "crypto/rand"
"errors"
"flag"
"fmt"
@@ -39,7 +38,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/pot"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
- mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
colorable "github.com/mattn/go-colorable"
)
@@ -69,21 +68,6 @@ func init() {
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
-func createGlobalStore() (string, *mockdb.GlobalStore, error) {
- var globalStore *mockdb.GlobalStore
- globalStoreDir, err := ioutil.TempDir("", "global.store")
- if err != nil {
- log.Error("Error initiating global store temp directory!", "err", err)
- return "", nil, err
- }
- globalStore, err = mockdb.NewGlobalStore(globalStoreDir)
- if err != nil {
- log.Error("Error initiating global store!", "err", err)
- return "", nil, err
- }
- return globalStoreDir, globalStore, nil
-}
-
func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
// setup
addr := network.RandomAddr() // tested peers peer address
@@ -114,7 +98,7 @@ func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest
delivery := NewDelivery(to, netStore)
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
- streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions)
+ streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil)
teardown := func() {
streamer.Close()
removeDataDir()
@@ -230,12 +214,7 @@ func generateRandomFile() (string, error) {
//generate a random file size between minFileSize and maxFileSize
fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
- b := make([]byte, fileSize*1024)
- _, err := crand.Read(b)
- if err != nil {
- log.Error("Error generating random file.", "err", err)
- return "", err
- }
+ b := testutil.RandomBytes(1, fileSize*1024)
return string(b), nil
}
diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go
index 9092ffe3ed..a5c9fa7e86 100644
--- a/swarm/network/stream/delivery.go
+++ b/swarm/network/stream/delivery.go
@@ -169,7 +169,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req *
go func() {
chunk, err := d.chunkStore.Get(ctx, req.Addr)
if err != nil {
- log.Warn("ChunkStore.Get can not retrieve chunk", "err", err)
+ log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err)
return
}
if req.SkipCheck {
@@ -245,14 +245,17 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) (
} else {
d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int, nn bool) bool {
id := p.ID()
- // TODO: skip light nodes that do not accept retrieve requests
+ if p.LightNode {
+ // skip light nodes
+ return true
+ }
if req.SkipPeer(id.String()) {
log.Trace("Delivery.RequestFromPeers: skip peer", "peer id", id)
return true
}
sp = d.getPeer(id)
if sp == nil {
- log.Warn("Delivery.RequestFromPeers: peer not found", "id", id)
+ //log.Warn("Delivery.RequestFromPeers: peer not found", "id", id)
return true
}
spID = &id
diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go
index 949645558b..a6173a3892 100644
--- a/swarm/network/stream/delivery_test.go
+++ b/swarm/network/stream/delivery_test.go
@@ -19,9 +19,7 @@ package stream
import (
"bytes"
"context"
- crand "crypto/rand"
"fmt"
- "io"
"os"
"sync"
"testing"
@@ -29,17 +27,26 @@ import (
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
+ "github.com/ethereum/go-ethereum/p2p/enode"
+ "github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/network"
+ pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue"
"github.com/ethereum/go-ethereum/swarm/network/simulation"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
+//Tests initializing a retrieve request
func TestStreamerRetrieveRequest(t *testing.T) {
- tester, streamer, _, teardown, err := newStreamerTester(t, nil)
+ regOpts := &RegistryOptions{
+ Retrieval: RetrievalClientOnly,
+ Syncing: SyncingDisabled,
+ }
+ tester, streamer, _, teardown, err := newStreamerTester(t, regOpts)
defer teardown()
if err != nil {
t.Fatal(err)
@@ -55,10 +62,21 @@ func TestStreamerRetrieveRequest(t *testing.T) {
)
streamer.delivery.RequestFromPeers(ctx, req)
+ stream := NewStream(swarmChunkServerStreamName, "", true)
+
err = tester.TestExchanges(p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Expects: []p2ptest.Expect{
- {
+ { //start expecting a subscription for RETRIEVE_REQUEST due to `RetrievalClientOnly`
+ Code: 4,
+ Msg: &SubscribeMsg{
+ Stream: stream,
+ History: nil,
+ Priority: Top,
+ },
+ Peer: node.ID(),
+ },
+ { //expect a retrieve request message for the given hash
Code: 5,
Msg: &RetrieveRequestMsg{
Addr: hash0[:],
@@ -74,9 +92,12 @@ func TestStreamerRetrieveRequest(t *testing.T) {
}
}
+//Test requesting a chunk from a peer then issuing a "empty" OfferedHashesMsg (no hashes available yet)
+//Should time out as the peer does not have the chunk (no syncing happened previously)
func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{
- DoServeRetrieve: true,
+ Retrieval: RetrievalEnabled,
+ Syncing: SyncingDisabled, //do no syncing
})
defer teardown()
if err != nil {
@@ -89,16 +110,31 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
peer := streamer.getPeer(node.ID())
+ stream := NewStream(swarmChunkServerStreamName, "", true)
+ //simulate pre-subscription to RETRIEVE_REQUEST stream on peer
peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{
- Stream: NewStream(swarmChunkServerStreamName, "", true),
+ Stream: stream,
History: nil,
Priority: Top,
})
+ //test the exchange
err = tester.TestExchanges(p2ptest.Exchange{
+ Expects: []p2ptest.Expect{
+ { //first expect a subscription to the RETRIEVE_REQUEST stream
+ Code: 4,
+ Msg: &SubscribeMsg{
+ Stream: stream,
+ History: nil,
+ Priority: Top,
+ },
+ Peer: node.ID(),
+ },
+ },
+ }, p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Triggers: []p2ptest.Trigger{
- {
+ { //then the actual RETRIEVE_REQUEST....
Code: 5,
Msg: &RetrieveRequestMsg{
Addr: chunk.Address()[:],
@@ -107,7 +143,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
},
},
Expects: []p2ptest.Expect{
- {
+ { //to which the peer responds with offered hashes
Code: 1,
Msg: &OfferedHashesMsg{
HandoverProof: nil,
@@ -120,7 +156,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
},
})
- expectedError := `exchange #0 "RetrieveRequestMsg": timed out`
+ //should fail with a timeout as the peer we are requesting
+ //the chunk from does not have the chunk
+ expectedError := `exchange #1 "RetrieveRequestMsg": timed out`
if err == nil || err.Error() != expectedError {
t.Fatalf("Expected error %v, got %v", expectedError, err)
}
@@ -130,7 +168,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
// offered hashes or delivery if skipHash is set to true
func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{
- DoServeRetrieve: true,
+ Retrieval: RetrievalEnabled,
+ Syncing: SyncingDisabled,
})
defer teardown()
if err != nil {
@@ -138,6 +177,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
}
node := tester.Nodes[0]
+
peer := streamer.getPeer(node.ID())
stream := NewStream(swarmChunkServerStreamName, "", true)
@@ -156,6 +196,18 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
}
err = tester.TestExchanges(p2ptest.Exchange{
+ Expects: []p2ptest.Expect{
+ {
+ Code: 4,
+ Msg: &SubscribeMsg{
+ Stream: stream,
+ History: nil,
+ Priority: Top,
+ },
+ Peer: node.ID(),
+ },
+ },
+ }, p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Triggers: []p2ptest.Trigger{
{
@@ -224,9 +276,90 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
}
}
+// if there is one peer in the Kademlia, RequestFromPeers should return it
+func TestRequestFromPeers(t *testing.T) {
+ dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8")
+
+ addr := network.RandomAddr()
+ to := network.NewKademlia(addr.OAddr, network.NewKadParams())
+ delivery := NewDelivery(to, nil)
+ protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil)
+ peer := network.NewPeer(&network.BzzPeer{
+ BzzAddr: network.RandomAddr(),
+ LightNode: false,
+ Peer: protocolsPeer,
+ }, to)
+ to.On(peer)
+ r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil)
+
+ // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished
+ sp := &Peer{
+ Peer: protocolsPeer,
+ pq: pq.New(int(PriorityQueue), PriorityQueueCap),
+ streamer: r,
+ }
+ r.setPeer(sp)
+ req := network.NewRequest(
+ storage.Address(hash0[:]),
+ true,
+ &sync.Map{},
+ )
+ ctx := context.Background()
+ id, _, err := delivery.RequestFromPeers(ctx, req)
+
+ if err != nil {
+ t.Fatal(err)
+ }
+ if *id != dummyPeerID {
+ t.Fatalf("Expected an id, got %v", id)
+ }
+}
+
+// RequestFromPeers should not return light nodes
+func TestRequestFromPeersWithLightNode(t *testing.T) {
+ dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8")
+
+ addr := network.RandomAddr()
+ to := network.NewKademlia(addr.OAddr, network.NewKadParams())
+ delivery := NewDelivery(to, nil)
+
+ protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil)
+ // setting up a lightnode
+ peer := network.NewPeer(&network.BzzPeer{
+ BzzAddr: network.RandomAddr(),
+ LightNode: true,
+ Peer: protocolsPeer,
+ }, to)
+ to.On(peer)
+ r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil)
+ // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished
+ sp := &Peer{
+ Peer: protocolsPeer,
+ pq: pq.New(int(PriorityQueue), PriorityQueueCap),
+ streamer: r,
+ }
+ r.setPeer(sp)
+
+ req := network.NewRequest(
+ storage.Address(hash0[:]),
+ true,
+ &sync.Map{},
+ )
+
+ ctx := context.Background()
+ // making a request which should return with "no peer found"
+ _, _, err := delivery.RequestFromPeers(ctx, req)
+
+ expectedError := "no peer found"
+ if err.Error() != expectedError {
+ t.Fatalf("expected '%v', got %v", expectedError, err)
+ }
+}
+
func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{
- DoServeRetrieve: true,
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingDisabled,
})
defer teardown()
if err != nil {
@@ -241,6 +374,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
node := tester.Nodes[0]
+ //subscribe to custom stream
stream := NewStream("foo", "", true)
err = streamer.Subscribe(node.ID(), stream, NewRange(5, 8), Top)
if err != nil {
@@ -253,7 +387,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Subscribe message",
Expects: []p2ptest.Expect{
- {
+ { //first expect subscription to the custom stream...
Code: 4,
Msg: &SubscribeMsg{
Stream: stream,
@@ -267,7 +401,8 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
p2ptest.Exchange{
Label: "ChunkDelivery message",
Triggers: []p2ptest.Trigger{
- {
+ { //...then trigger a chunk delivery for the given chunk from peer in order for
+ //local node to get the chunk delivered
Code: 6,
Msg: &ChunkDeliveryMsg{
Addr: chunkKey,
@@ -342,9 +477,10 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
- SkipCheck: skipCheck,
- DoServeRetrieve: true,
- })
+ SkipCheck: skipCheck,
+ Syncing: SyncingDisabled,
+ Retrieval: RetrievalEnabled,
+ }, nil)
bucket.Store(bucketKeyRegistry, r)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
@@ -393,7 +529,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
//now we can actually upload a (random) file to the round-robin store
size := chunkCount * chunkSize
log.Debug("Storing data to file store")
- fileHash, wait, err := roundRobinFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+ fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
// wait until all chunks stored
if err != nil {
return err
@@ -408,20 +544,6 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
return err
}
- //each of the nodes (except pivot node) subscribes to the stream of the next node
- for j, node := range nodeIDs[0 : nodes-1] {
- sid := nodeIDs[j+1]
- item, ok := sim.NodeItem(node, bucketKeyRegistry)
- if !ok {
- return fmt.Errorf("No registry")
- }
- registry := item.(*Registry)
- err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), nil, Top)
- if err != nil {
- return err
- }
- }
-
//get the pivot node's filestore
item, ok := sim.NodeItem(*sim.PivotNodeID(), bucketKeyFileStore)
if !ok {
@@ -443,13 +565,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
if d.Error != nil {
- log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal(d.Error)
}
}
@@ -530,9 +652,10 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
SkipCheck: skipCheck,
- DoSync: true,
+ Syncing: SyncingDisabled,
+ Retrieval: RetrievalDisabled,
SyncUpdateDelay: 0,
- })
+ }, nil)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
bucket.Store(bucketKeyFileStore, fileStore)
@@ -574,13 +697,13 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
if d.Error != nil {
- log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
b.Fatal(d.Error)
}
}
@@ -595,7 +718,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
for i := 0; i < chunkCount; i++ {
// create actual size real chunks
ctx := context.TODO()
- hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false)
+ hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false)
if err != nil {
b.Fatalf("expected no error. got %v", err)
}
diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go
index 3164193b34..defb6df50a 100644
--- a/swarm/network/stream/intervals_test.go
+++ b/swarm/network/stream/intervals_test.go
@@ -18,10 +18,8 @@ package stream
import (
"context"
- crand "crypto/rand"
"encoding/binary"
"fmt"
- "io"
"os"
"sync"
"testing"
@@ -29,13 +27,13 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
- "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/network/simulation"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestIntervalsLive(t *testing.T) {
@@ -83,8 +81,10 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingRegisterOnly,
SkipCheck: skipCheck,
- })
+ }, nil)
bucket.Store(bucketKeyRegistry, r)
r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) {
@@ -128,7 +128,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
fileStore := item.(*storage.FileStore)
size := chunkCount * chunkSize
- _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+
+ _, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
if err != nil {
log.Error("Store error: %v", "err", err)
t.Fatal(err)
@@ -152,7 +153,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
@@ -163,7 +164,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
go func() {
for d := range disconnections {
if d.Error != nil {
- log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal(d.Error)
}
}
diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go
index 0d3bc7f544..65cde2411c 100644
--- a/swarm/network/stream/lightnode_test.go
+++ b/swarm/network/stream/lightnode_test.go
@@ -25,7 +25,8 @@ import (
// when it is serving Retrieve requests.
func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) {
registryOptions := &RegistryOptions{
- DoServeRetrieve: true,
+ Retrieval: RetrievalClientOnly,
+ Syncing: SyncingDisabled,
}
tester, _, _, teardown, err := newStreamerTester(t, registryOptions)
defer teardown()
@@ -63,7 +64,8 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) {
// requests are disabled
func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) {
registryOptions := &RegistryOptions{
- DoServeRetrieve: false,
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingDisabled,
}
tester, _, _, teardown, err := newStreamerTester(t, registryOptions)
defer teardown()
@@ -106,7 +108,8 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) {
// when syncing is enabled.
func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) {
registryOptions := &RegistryOptions{
- DoSync: true,
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingRegisterOnly,
}
tester, _, _, teardown, err := newStreamerTester(t, registryOptions)
defer teardown()
@@ -150,7 +153,8 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) {
// when syncing is disabled.
func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) {
registryOptions := &RegistryOptions{
- DoSync: false,
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingDisabled,
}
tester, _, _, teardown, err := newStreamerTester(t, registryOptions)
defer teardown()
diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go
index b81cfc0ca0..5ea0b1511b 100644
--- a/swarm/network/stream/snapshot_retrieval_test.go
+++ b/swarm/network/stream/snapshot_retrieval_test.go
@@ -127,11 +127,10 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
- DoSync: true,
+ Retrieval: RetrievalEnabled,
+ Syncing: SyncingAutoSubscribe,
SyncUpdateDelay: 3 * time.Second,
- DoRetrieve: true,
- DoServeRetrieve: true,
- })
+ }, nil)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
bucket.Store(bucketKeyFileStore, fileStore)
diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go
index 8d89f28cba..4e56f71b54 100644
--- a/swarm/network/stream/snapshot_sync_test.go
+++ b/swarm/network/stream/snapshot_sync_test.go
@@ -17,9 +17,7 @@ package stream
import (
"context"
- crand "crypto/rand"
"fmt"
- "io"
"os"
"runtime"
"sync"
@@ -29,7 +27,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
- "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
@@ -38,7 +35,9 @@ import (
"github.com/ethereum/go-ethereum/swarm/pot"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
- mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+ "github.com/ethereum/go-ethereum/swarm/storage/mock"
+ mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
const MaxTimeout = 600
@@ -165,10 +164,10 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic
netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
- DoSync: true,
- DoServeRetrieve: true,
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingAutoSubscribe,
SyncUpdateDelay: 3 * time.Second,
- })
+ }, nil)
bucket.Store(bucketKeyRegistry, r)
@@ -211,12 +210,12 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
- log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal("unexpected disconnect")
cancelSimRun()
}
@@ -270,20 +269,9 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio
// File retrieval check is repeated until all uploaded files are retrieved from all nodes
// or until the timeout is reached.
- var gDir string
- var globalStore *mockdb.GlobalStore
+ var globalStore mock.GlobalStorer
if *useMockStore {
- gDir, globalStore, err = createGlobalStore()
- if err != nil {
- return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
- }
- defer func() {
- os.RemoveAll(gDir)
- err := globalStore.Close()
- if err != nil {
- log.Error("Error closing global store! %v", "err", err)
- }
- }()
+ globalStore = mockmem.NewGlobalStore()
}
REPEAT:
for {
@@ -360,9 +348,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
- DoServeRetrieve: true,
- DoSync: true,
- })
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingRegisterOnly,
+ }, nil)
bucket.Store(bucketKeyRegistry, r)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
@@ -403,12 +391,12 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
- log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal("unexpected disconnect")
cancelSimRun()
}
@@ -429,7 +417,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
var subscriptionCount int
- filter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(4)
+ filter := simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("stream").MsgCode(4)
eventC := sim.PeerEvents(ctx, nodeIDs, filter)
for j, node := range nodeIDs {
@@ -478,14 +466,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
return err
}
- var gDir string
- var globalStore *mockdb.GlobalStore
+ var globalStore mock.GlobalStorer
if *useMockStore {
- gDir, globalStore, err = createGlobalStore()
- if err != nil {
- return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
- }
- defer os.RemoveAll(gDir)
+ globalStore = mockmem.NewGlobalStore()
}
// File retrieval check is repeated until all uploaded files are retrieved from all nodes
// or until the timeout is reached.
@@ -603,7 +586,7 @@ func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, lstore *storage.Lo
size := chunkSize
var rootAddrs []storage.Address
for i := 0; i < chunkCount; i++ {
- rk, wait, err := fileStore.Store(context.TODO(), io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+ rk, wait, err := fileStore.Store(context.TODO(), testutil.RandomReader(i, size), int64(size), false)
if err != nil {
return nil, err
}
diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go
index 0ac374def1..32e107823b 100644
--- a/swarm/network/stream/stream.go
+++ b/swarm/network/stream/stream.go
@@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"math"
+ "reflect"
"sync"
"time"
@@ -47,6 +48,31 @@ const (
HashSize = 32
)
+//Enumerate options for syncing and retrieval
+type SyncingOption int
+type RetrievalOption int
+
+//Syncing options
+const (
+ //Syncing disabled
+ SyncingDisabled SyncingOption = iota
+ //Register the client and the server but not subscribe
+ SyncingRegisterOnly
+ //Both client and server funcs are registered, subscribe sent automatically
+ SyncingAutoSubscribe
+)
+
+const (
+ //Retrieval disabled. Used mostly for tests to isolate syncing features (i.e. syncing only)
+ RetrievalDisabled RetrievalOption = iota
+ //Only the client side of the retrieve request is registered.
+ //(light nodes do not serve retrieve requests)
+ //once the client is registered, subscription to retrieve request stream is always sent
+ RetrievalClientOnly
+ //Both client and server funcs are registered, subscribe sent automatically
+ RetrievalEnabled
+)
+
// Registry registry for outgoing and incoming streamer constructors
type Registry struct {
addr enode.ID
@@ -60,28 +86,33 @@ type Registry struct {
peers map[enode.ID]*Peer
delivery *Delivery
intervalsStore state.Store
- doRetrieve bool
+ autoRetrieval bool //automatically subscribe to retrieve request stream
maxPeerServers int
+ spec *protocols.Spec //this protocol's spec
+ balance protocols.Balance //implements protocols.Balance, for accounting
+ prices protocols.Prices //implements protocols.Prices, provides prices to accounting
}
// RegistryOptions holds optional values for NewRegistry constructor.
type RegistryOptions struct {
SkipCheck bool
- DoSync bool // Sets if the server syncs with peers. Default is true, set to false by lightnode or nosync flags.
- DoRetrieve bool // Sets if the server issues Retrieve requests. Default is true.
- DoServeRetrieve bool // Sets if the server serves Retrieve requests. Default is true, set to false by lightnode flag.
+ Syncing SyncingOption //Defines syncing behavior
+ Retrieval RetrievalOption //Defines retrieval behavior
SyncUpdateDelay time.Duration
MaxPeerServers int // The limit of servers for each peer in registry
}
// NewRegistry is Streamer constructor
-func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry {
+func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions, balance protocols.Balance) *Registry {
if options == nil {
options = &RegistryOptions{}
}
if options.SyncUpdateDelay <= 0 {
options.SyncUpdateDelay = 15 * time.Second
}
+ //check if retriaval has been disabled
+ retrieval := options.Retrieval != RetrievalDisabled
+
streamer := &Registry{
addr: localID,
skipCheck: options.SkipCheck,
@@ -90,13 +121,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
peers: make(map[enode.ID]*Peer),
delivery: delivery,
intervalsStore: intervalsStore,
- doRetrieve: options.DoRetrieve,
+ autoRetrieval: retrieval,
maxPeerServers: options.MaxPeerServers,
+ balance: balance,
}
+ streamer.setupSpec()
+
streamer.api = NewAPI(streamer)
delivery.getPeer = streamer.getPeer
- if options.DoServeRetrieve {
+ //if retrieval is enabled, register the server func, so that retrieve requests will be served (non-light nodes only)
+ if options.Retrieval == RetrievalEnabled {
streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) {
if !live {
return nil, errors.New("only live retrieval requests supported")
@@ -105,16 +140,21 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
})
}
- streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) {
- return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live))
- })
+ //if retrieval is not disabled, register the client func (both light nodes and normal nodes can issue retrieve requests)
+ if options.Retrieval != RetrievalDisabled {
+ streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) {
+ return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live))
+ })
+ }
- if options.DoSync {
+ //If syncing is not disabled, the syncing functions are registered (both client and server)
+ if options.Syncing != SyncingDisabled {
RegisterSwarmSyncerServer(streamer, syncChunkStore)
RegisterSwarmSyncerClient(streamer, syncChunkStore)
}
- if options.DoSync {
+ //if syncing is set to automatically subscribe to the syncing stream, start the subscription process
+ if options.Syncing == SyncingAutoSubscribe {
// latestIntC function ensures that
// - receiving from the in chan is not blocked by processing inside the for loop
// - the latest int value is delivered to the loop after the processing is done
@@ -195,6 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
return streamer
}
+//we need to construct a spec instance per node instance
+func (r *Registry) setupSpec() {
+ //first create the "bare" spec
+ r.createSpec()
+ //if balance is nil, this node has been started without swap support (swapEnabled flag is false)
+ if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() {
+ //swap is enabled, so setup the hook
+ r.spec.Hook = protocols.NewAccounting(r.balance, r.prices)
+ }
+}
+
// RegisterClient registers an incoming streamer constructor
func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) {
r.clientMu.Lock()
@@ -385,7 +436,7 @@ func (r *Registry) Run(p *network.BzzPeer) error {
defer close(sp.quit)
defer sp.close()
- if r.doRetrieve {
+ if r.autoRetrieval && !p.LightNode {
err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", true), nil, Top)
if err != nil {
return err
@@ -459,7 +510,7 @@ func (r *Registry) updateSyncing() {
}
func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error {
- peer := protocols.NewPeer(p, rw, Spec)
+ peer := protocols.NewPeer(p, rw, r.spec)
bp := network.NewBzzPeer(peer)
np := network.NewPeer(bp, r.delivery.kad)
r.delivery.kad.On(np)
@@ -683,35 +734,43 @@ func (c *clientParams) clientCreated() {
close(c.clientCreatedC)
}
-// Spec is the spec of the streamer protocol
-var Spec = &protocols.Spec{
- Name: "stream",
- Version: 8,
- MaxMsgSize: 10 * 1024 * 1024,
- Messages: []interface{}{
- UnsubscribeMsg{},
- OfferedHashesMsg{},
- WantedHashesMsg{},
- TakeoverProofMsg{},
- SubscribeMsg{},
- RetrieveRequestMsg{},
- ChunkDeliveryMsgRetrieval{},
- SubscribeErrorMsg{},
- RequestSubscriptionMsg{},
- QuitMsg{},
- ChunkDeliveryMsgSyncing{},
- },
+//GetSpec returns the streamer spec to callers
+//This used to be a global variable but for simulations with
+//multiple nodes its fields (notably the Hook) would be overwritten
+func (r *Registry) GetSpec() *protocols.Spec {
+ return r.spec
+}
+
+func (r *Registry) createSpec() {
+ // Spec is the spec of the streamer protocol
+ var spec = &protocols.Spec{
+ Name: "stream",
+ Version: 8,
+ MaxMsgSize: 10 * 1024 * 1024,
+ Messages: []interface{}{
+ UnsubscribeMsg{},
+ OfferedHashesMsg{},
+ WantedHashesMsg{},
+ TakeoverProofMsg{},
+ SubscribeMsg{},
+ RetrieveRequestMsg{},
+ ChunkDeliveryMsgRetrieval{},
+ SubscribeErrorMsg{},
+ RequestSubscriptionMsg{},
+ QuitMsg{},
+ ChunkDeliveryMsgSyncing{},
+ },
+ }
+ r.spec = spec
}
func (r *Registry) Protocols() []p2p.Protocol {
return []p2p.Protocol{
{
- Name: Spec.Name,
- Version: Spec.Version,
- Length: Spec.Length(),
+ Name: r.spec.Name,
+ Version: r.spec.Version,
+ Length: r.spec.Length(),
Run: r.runProtocol,
- // NodeInfo: ,
- // PeerInfo: ,
},
}
}
diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go
index e7f79e7a12..16c74d3b34 100644
--- a/swarm/network/stream/streamer_test.go
+++ b/swarm/network/stream/streamer_test.go
@@ -765,6 +765,8 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) {
func TestMaxPeerServersWithUnsubscribe(t *testing.T) {
var maxPeerServers = 6
tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingDisabled,
MaxPeerServers: maxPeerServers,
})
defer teardown()
diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go
index f2be3bef99..5764efc92d 100644
--- a/swarm/network/stream/syncer_test.go
+++ b/swarm/network/stream/syncer_test.go
@@ -18,9 +18,7 @@ package stream
import (
"context"
- crand "crypto/rand"
"fmt"
- "io"
"io/ioutil"
"math"
"os"
@@ -30,7 +28,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/node"
- "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/swarm/log"
@@ -38,7 +35,9 @@ import (
"github.com/ethereum/go-ethereum/swarm/network/simulation"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
- mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+ "github.com/ethereum/go-ethereum/swarm/storage/mock"
+ mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
const dataChunkCount = 200
@@ -50,7 +49,7 @@ func TestSyncerSimulation(t *testing.T) {
testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1)
}
-func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
+func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
address := common.BytesToAddress(id.Bytes())
mockStore := globalStore.NewNodeStore(address)
params := storage.NewDefaultLocalStoreParams()
@@ -62,6 +61,9 @@ func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network
params.Init(datadir)
params.BaseKey = addr.Over()
lstore, err = storage.NewLocalStore(params, mockStore)
+ if err != nil {
+ return nil, "", err
+ }
return lstore, datadir, nil
}
@@ -69,8 +71,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
var store storage.ChunkStore
- var globalStore *mockdb.GlobalStore
- var gDir, datadir string
+ var datadir string
node := ctx.Config.Node()
addr := network.NewAddr(node)
@@ -78,11 +79,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
addr.OAddr[0] = byte(0)
if *useMockStore {
- gDir, globalStore, err = createGlobalStore()
- if err != nil {
- return nil, nil, fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
- }
- store, datadir, err = createMockStore(globalStore, node.ID(), addr)
+ store, datadir, err = createMockStore(mockmem.NewGlobalStore(), node.ID(), addr)
} else {
store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
}
@@ -93,13 +90,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
cleanup = func() {
store.Close()
os.RemoveAll(datadir)
- if *useMockStore {
- err := globalStore.Close()
- if err != nil {
- log.Error("Error closing global store! %v", "err", err)
- }
- os.RemoveAll(gDir)
- }
}
localStore := store.(*storage.LocalStore)
netStore, err := storage.NewNetStore(localStore, nil)
@@ -114,8 +104,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
bucket.Store(bucketKeyDelivery, delivery)
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
+ Retrieval: RetrievalDisabled,
+ Syncing: SyncingAutoSubscribe,
SkipCheck: skipCheck,
- })
+ }, nil)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
bucket.Store(bucketKeyFileStore, fileStore)
@@ -147,13 +139,13 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
- simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+ simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
if d.Error != nil {
- log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+ log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal(d.Error)
}
}
@@ -178,7 +170,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
}
fileStore := item.(*storage.FileStore)
size := chunkCount * chunkSize
- _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+ _, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false)
if err != nil {
t.Fatal(err.Error())
}
diff --git a/swarm/network_test.go b/swarm/network_test.go
index aab1c05099..d84f28147f 100644
--- a/swarm/network_test.go
+++ b/swarm/network_test.go
@@ -335,7 +335,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
nodeIDs := sim.UpNodeIDs()
- shuffle(len(nodeIDs), func(i, j int) {
+ rand.Shuffle(len(nodeIDs), func(i, j int) {
nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i]
})
for _, id := range nodeIDs {
@@ -404,7 +404,7 @@ func retrieve(
nodeStatusM *sync.Map,
totalFoundCount *uint64,
) (missing uint64) {
- shuffle(len(files), func(i, j int) {
+ rand.Shuffle(len(files), func(i, j int) {
files[i], files[j] = files[j], files[i]
})
@@ -499,32 +499,3 @@ func retrieve(
return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount)
}
-
-// Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333
-//
-// Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped.
-//
-// shuffle pseudo-randomizes the order of elements.
-// n is the number of elements. Shuffle panics if n < 0.
-// swap swaps the elements with indexes i and j.
-func shuffle(n int, swap func(i, j int)) {
- if n < 0 {
- panic("invalid argument to Shuffle")
- }
-
- // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
- // Shuffle really ought not be called with n that doesn't fit in 32 bits.
- // Not only will it take a very long time, but with 2³¹! possible permutations,
- // there's no way that any PRNG can have a big enough internal state to
- // generate even a minuscule percentage of the possible permutations.
- // Nevertheless, the right API signature accepts an int n, so handle it as best we can.
- i := n - 1
- for ; i > 1<<31-1-1; i-- {
- j := int(rand.Int63n(int64(i + 1)))
- swap(i, j)
- }
- for ; i > 0; i-- {
- j := int(rand.Int31n(int32(i + 1)))
- swap(i, j)
- }
-}
diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go
index cfef3c7947..8f2f0e8059 100644
--- a/swarm/pss/client/client_test.go
+++ b/swarm/pss/client/client_test.go
@@ -252,7 +252,13 @@ func newServices() adapters.Services {
ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
keys, err := wapi.NewKeyPair(ctxlocal)
+ if err != nil {
+ return nil, err
+ }
privkey, err := w.GetPrivateKey(keys)
+ if err != nil {
+ return nil, err
+ }
psparams := pss.NewPssParams().WithPrivateKey(privkey)
pskad := kademlia(ctx.Config.ID)
ps, err := pss.NewPss(pskad, psparams)
diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go
index 675b41ada2..d4d383a6ba 100644
--- a/swarm/pss/notify/notify_test.go
+++ b/swarm/pss/notify/notify_test.go
@@ -223,7 +223,13 @@ func newServices(allowRaw bool) adapters.Services {
ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
keys, err := wapi.NewKeyPair(ctxlocal)
+ if err != nil {
+ return nil, err
+ }
privkey, err := w.GetPrivateKey(keys)
+ if err != nil {
+ return nil, err
+ }
pssp := pss.NewPssParams().WithPrivateKey(privkey)
pssp.MsgTTL = time.Second * 30
pssp.AllowRaw = allowRaw
diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go
index f4209fea59..4ef3e90a04 100644
--- a/swarm/pss/protocol_test.go
+++ b/swarm/pss/protocol_test.go
@@ -93,11 +93,17 @@ func testProtocol(t *testing.T) {
lctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic)
+ if err != nil {
+ t.Fatal(err)
+ }
defer lsub.Unsubscribe()
rmsgC := make(chan APIMsg)
rctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic)
+ if err != nil {
+ t.Fatal(err)
+ }
defer rsub.Unsubscribe()
// set reciprocal public keys
diff --git a/swarm/state/dbstore.go b/swarm/state/dbstore.go
index 5e5c172b25..fc5dd8f7ca 100644
--- a/swarm/state/dbstore.go
+++ b/swarm/state/dbstore.go
@@ -22,6 +22,7 @@ import (
"errors"
"github.com/syndtr/goleveldb/leveldb"
+ "github.com/syndtr/goleveldb/leveldb/storage"
)
// ErrNotFound is returned when no results are returned from the database
@@ -30,6 +31,15 @@ var ErrNotFound = errors.New("ErrorNotFound")
// ErrInvalidArgument is returned when the argument type does not match the expected type
var ErrInvalidArgument = errors.New("ErrorInvalidArgument")
+// Store defines methods required to get, set, delete values for different keys
+// and close the underlying resources.
+type Store interface {
+ Get(key string, i interface{}) (err error)
+ Put(key string, i interface{}) (err error)
+ Delete(key string) (err error)
+ Close() error
+}
+
// DBStore uses LevelDB to store values.
type DBStore struct {
db *leveldb.DB
@@ -46,6 +56,17 @@ func NewDBStore(path string) (s *DBStore, err error) {
}, nil
}
+// NewInmemoryStore returns a new instance of DBStore. To be used only in tests and simulations.
+func NewInmemoryStore() *DBStore {
+ db, err := leveldb.Open(storage.NewMemStorage(), nil)
+ if err != nil {
+ panic(err)
+ }
+ return &DBStore{
+ db: db,
+ }
+}
+
// Get retrieves a persisted value for a specific key. If there is no results
// ErrNotFound is returned. The provided parameter should be either a byte slice or
// a struct that implements the encoding.BinaryUnmarshaler interface
@@ -69,7 +90,7 @@ func (s *DBStore) Get(key string, i interface{}) (err error) {
// Put stores an object that implements Binary for a specific key.
func (s *DBStore) Put(key string, i interface{}) (err error) {
- bytes := []byte{}
+ var bytes []byte
marshaler, ok := i.(encoding.BinaryMarshaler)
if !ok {
diff --git a/swarm/state/dbstore_test.go b/swarm/state/dbstore_test.go
index 6683e788f7..f7098956d0 100644
--- a/swarm/state/dbstore_test.go
+++ b/swarm/state/dbstore_test.go
@@ -112,6 +112,9 @@ func testPersistedStore(t *testing.T, store Store) {
as := []string{}
err = store.Get("key2", &as)
+ if err != nil {
+ t.Fatal(err)
+ }
if len(as) != 3 {
t.Fatalf("serialized array did not match expectation")
diff --git a/swarm/state/inmemorystore.go b/swarm/state/inmemorystore.go
deleted file mode 100644
index 1ca25404a1..0000000000
--- a/swarm/state/inmemorystore.go
+++ /dev/null
@@ -1,94 +0,0 @@
-// Copyright 2018 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package state
-
-import (
- "encoding"
- "encoding/json"
- "sync"
-)
-
-// InmemoryStore is the reference implementation of Store interface that is supposed
-// to be used in tests.
-type InmemoryStore struct {
- db map[string][]byte
- mu sync.RWMutex
-}
-
-// NewInmemoryStore returns a new instance of InmemoryStore.
-func NewInmemoryStore() *InmemoryStore {
- return &InmemoryStore{
- db: make(map[string][]byte),
- }
-}
-
-// Get retrieves a value stored for a specific key. If there is no value found,
-// ErrNotFound is returned.
-func (s *InmemoryStore) Get(key string, i interface{}) (err error) {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- bytes, ok := s.db[key]
- if !ok {
- return ErrNotFound
- }
-
- unmarshaler, ok := i.(encoding.BinaryUnmarshaler)
- if !ok {
- return json.Unmarshal(bytes, i)
- }
-
- return unmarshaler.UnmarshalBinary(bytes)
-}
-
-// Put stores a value for a specific key.
-func (s *InmemoryStore) Put(key string, i interface{}) (err error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- bytes := []byte{}
-
- marshaler, ok := i.(encoding.BinaryMarshaler)
- if !ok {
- if bytes, err = json.Marshal(i); err != nil {
- return err
- }
- } else {
- if bytes, err = marshaler.MarshalBinary(); err != nil {
- return err
- }
- }
-
- s.db[key] = bytes
- return nil
-}
-
-// Delete removes value stored under a specific key.
-func (s *InmemoryStore) Delete(key string) (err error) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if _, ok := s.db[key]; !ok {
- return ErrNotFound
- }
- delete(s.db, key)
- return nil
-}
-
-// Close does not do anything.
-func (s *InmemoryStore) Close() error {
- return nil
-}
diff --git a/swarm/state/store.go b/swarm/state/store.go
deleted file mode 100644
index fb7fe258f1..0000000000
--- a/swarm/state/store.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2018 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package state
-
-// Store defines methods required to get, set, delete values for different keys
-// and close the underlying resources.
-type Store interface {
- Get(key string, i interface{}) (err error)
- Put(key string, i interface{}) (err error)
- Delete(key string) (err error)
- Close() error
-}
diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go
index 6172d8a092..1f847edcb9 100644
--- a/swarm/storage/chunker_test.go
+++ b/swarm/storage/chunker_test.go
@@ -19,13 +19,13 @@ package storage
import (
"bytes"
"context"
- "crypto/rand"
"encoding/binary"
"fmt"
"io"
"testing"
"github.com/ethereum/go-ethereum/crypto/sha3"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
/*
@@ -47,7 +47,7 @@ func newTestHasherStore(store ChunkStore, hash string) *hasherStore {
}
func testRandomBrokenData(n int, tester *chunkerTester) {
- data := io.LimitReader(rand.Reader, int64(n))
+ data := testutil.RandomReader(1, n)
brokendata := brokenLimitReader(data, n, n/2)
buf := make([]byte, n)
@@ -56,7 +56,7 @@ func testRandomBrokenData(n int, tester *chunkerTester) {
tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err)
}
- data = io.LimitReader(rand.Reader, int64(n))
+ data = testutil.RandomReader(2, n)
brokendata = brokenLimitReader(data, n, n/2)
putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash)
@@ -77,7 +77,8 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester)
input, found := tester.inputs[uint64(n)]
var data io.Reader
if !found {
- data, input = GenerateRandomData(n)
+ input = testutil.RandomBytes(1, n)
+ data = bytes.NewReader(input)
tester.inputs[uint64(n)] = input
} else {
data = io.LimitReader(bytes.NewReader(input), int64(n))
@@ -118,14 +119,13 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester)
// testing partial read
for i := 1; i < n; i += 10000 {
readableLength := n - i
- output := make([]byte, readableLength)
r, err := reader.ReadAt(output, int64(i))
if r != readableLength || err != io.EOF {
tester.t.Fatalf("readAt error with offset %v read: %v n = %v err = %v\n", i, r, readableLength, err)
}
if input != nil {
- if !bytes.Equal(output, input[i:]) {
- tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output)
+ if !bytes.Equal(output[:readableLength], input[i:]) {
+ tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output[:readableLength])
}
}
}
@@ -177,7 +177,8 @@ func TestDataAppend(t *testing.T) {
input, found := tester.inputs[uint64(n)]
var data io.Reader
if !found {
- data, input = GenerateRandomData(n)
+ input = testutil.RandomBytes(i, n)
+ data = bytes.NewReader(input)
tester.inputs[uint64(n)] = input
} else {
data = io.LimitReader(bytes.NewReader(input), int64(n))
@@ -199,7 +200,8 @@ func TestDataAppend(t *testing.T) {
appendInput, found := tester.inputs[uint64(m)]
var appendData io.Reader
if !found {
- appendData, appendInput = GenerateRandomData(m)
+ appendInput = testutil.RandomBytes(i, m)
+ appendData = bytes.NewReader(appendInput)
tester.inputs[uint64(m)] = appendInput
} else {
appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m))
@@ -272,7 +274,7 @@ func benchReadAll(reader LazySectionReader) {
func benchmarkSplitJoin(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
- data := testDataReader(n)
+ data := testutil.RandomReader(i, n)
putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash)
ctx := context.TODO()
@@ -292,7 +294,7 @@ func benchmarkSplitJoin(n int, t *testing.B) {
func benchmarkSplitTreeSHA3(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
- data := testDataReader(n)
+ data := testutil.RandomReader(i, n)
putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash)
ctx := context.Background()
@@ -311,7 +313,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) {
func benchmarkSplitTreeBMT(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
- data := testDataReader(n)
+ data := testutil.RandomReader(i, n)
putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash)
ctx := context.Background()
@@ -329,7 +331,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) {
func benchmarkSplitPyramidBMT(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
- data := testDataReader(n)
+ data := testutil.RandomReader(i, n)
putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash)
ctx := context.Background()
@@ -347,7 +349,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) {
func benchmarkSplitPyramidSHA3(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
- data := testDataReader(n)
+ data := testutil.RandomReader(i, n)
putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash)
ctx := context.Background()
@@ -365,8 +367,8 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) {
func benchmarkSplitAppendPyramid(n, m int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
- data := testDataReader(n)
- data1 := testDataReader(m)
+ data := testutil.RandomReader(i, n)
+ data1 := testutil.RandomReader(t.N+i, m)
store := NewMapChunkStore()
putGetter := newTestHasherStore(store, SHA3Hash)
diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go
index 600be164a9..af104a5aeb 100644
--- a/swarm/storage/common_test.go
+++ b/swarm/storage/common_test.go
@@ -19,7 +19,6 @@ package storage
import (
"bytes"
"context"
- "crypto/rand"
"flag"
"fmt"
"io"
@@ -31,7 +30,7 @@ import (
"github.com/ethereum/go-ethereum/log"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
- colorable "github.com/mattn/go-colorable"
+ "github.com/mattn/go-colorable"
)
var (
@@ -151,10 +150,6 @@ func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error)
return err
}
-func testDataReader(l int) (r io.Reader) {
- return io.LimitReader(rand.Reader, int64(l))
-}
-
func (r *brokenLimitedReader) Read(buf []byte) (int, error) {
if r.off+len(buf) > r.errAt {
return 0, fmt.Errorf("Broken reader")
diff --git a/swarm/storage/encryption/encryption_test.go b/swarm/storage/encryption/encryption_test.go
index c3abefdcec..0c0d0508c3 100644
--- a/swarm/storage/encryption/encryption_test.go
+++ b/swarm/storage/encryption/encryption_test.go
@@ -18,12 +18,12 @@ package encryption
import (
"bytes"
- "crypto/rand"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto/sha3"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
var expectedTransformedHex = "352187af3a843decc63ceca6cb01ea39dbcf77caf0a8f705f5c30d557044ceec9392b94a79376f1e5c10cd0c0f2a98e5353bf22b3ea4fdac6677ee553dec192e3db64e179d0474e96088fb4abd2babd67de123fb398bdf84d818f7bda2c1ab60b3ea0e0569ae54aa969658eb4844e6960d2ff44d7c087ee3aaffa1c0ee5df7e50b615f7ad90190f022934ad5300c7d1809bfe71a11cc04cece5274eb97a5f20350630522c1dbb7cebaf4f97f84e03f5cfd88f2b48880b25d12f4d5e75c150f704ef6b46c72e07db2b705ac3644569dccd22fd8f964f6ef787fda63c46759af334e6f665f70eac775a7017acea49f3c7696151cb1b9434fa4ac27fb803921ffb5ec58dafa168098d7d5b97e384be3384cf5bc235c3d887fef89fe76c0065f9b8d6ad837b442340d9e797b46ef5709ea3358bc415df11e4830de986ef0f1c418ffdcc80e9a3cda9bea0ab5676c0d4240465c43ba527e3b4ea50b4f6255b510e5d25774a75449b0bd71e56c537ade4fcf0f4d63c99ae1dbb5a844971e2c19941b8facfcfc8ee3056e7cb3c7114c5357e845b52f7103cb6e00d2308c37b12baa5b769e1cc7b00fc06f2d16e70cc27a82cb9c1a4e40cb0d43907f73df2c9db44f1b51a6b0bc6d09f77ac3be14041fae3f9df2da42df43ae110904f9ecee278030185254d7c6e918a5512024d047f77a992088cb3190a6587aa54d0c7231c1cd2e455e0d4c07f74bece68e29cd8ba0190c0bcfb26d24634af5d91a81ef5d4dd3d614836ce942ddbf7bb1399317f4c03faa675f325f18324bf9433844bfe5c4cc04130c8d5c329562b7cd66e72f7355de8f5375a72202971613c32bd7f3fcdcd51080758cd1d0a46dbe8f0374381dbc359f5864250c63dde8131cbd7c98ae2b0147d6ea4bf65d1443d511b18e6d608bbb46ac036353b4c51df306a10a6f6939c38629a5c18aaf89cac04bd3ad5156e6b92011c88341cb08551bab0a89e6a46538f5af33b86121dba17e3a434c273f385cd2e8cb90bdd32747d8425d929ccbd9b0815c73325988855549a8489dfd047daf777aaa3099e54cf997175a5d9e1edfe363e3b68c70e02f6bf4fcde6a0f3f7d0e7e98bde1a72ae8b6cd27b32990680cc4a04fc467f41c5adcaddabfc71928a3f6872c360c1d765260690dd28b269864c8e380d9c92ef6b89b0094c8f9bb22608b4156381b19b920e9583c9616ce5693b4d2a6c689f02e6a91584a8e501e107403d2689dd0045269dd9946c0e969fb656a3b39d84a798831f5f9290f163eb2f97d3ae25071324e95e2256d9c1e56eb83c26397855323edc202d56ad05894333b7f0ed3c1e4734782eb8bd5477242fd80d7a89b12866f85cfae476322f032465d6b1253993033fccd4723530630ab97a1566460af9c90c9da843c229406e65f3fa578bd6bf04dee9b6153807ddadb8ceefc5c601a8ab26023c67b1ab1e8e0f29ce94c78c308005a781853e7a2e0e51738939a657c987b5e611f32f47b5ff461c52e63e0ea390515a8e1f5393dae54ea526934b5f310b76e3fa050e40718cb4c8a20e58946d6ee1879f08c52764422fe542b3240e75eccb7aa75b1f8a651e37a3bc56b0932cdae0e985948468db1f98eb4b77b82081ea25d8a762db00f7898864984bd80e2f3f35f236bf57291dec28f550769943bcfb6f884b7687589b673642ef7fe5d7d5a87d3eca5017f83ccb9a3310520474479464cb3f433440e7e2f1e28c0aef700a45848573409e7ab66e0cfd4fe5d2147ace81bc65fd8891f6245cd69246bbf5c27830e5ab882dd1d02aba34ff6ca9af88df00fd602892f02fedbdc65dedec203faf3f8ff4a97314e0ddb58b9ab756a61a562597f4088b445fcc3b28a708ca7b1485dcd791b779fbf2b3ef1ec5c6205f595fbe45a02105034147e5a146089c200a49dae33ae051a08ea5f974a21540aaeffa7f9d9e3d35478016fb27b871036eb27217a5b834b461f535752fb5f1c8dded3ae14ce3a2ef6639e2fe41939e3509e46e347a95d50b2080f1ba42c804b290ddc912c952d1cec3f2661369f738feacc0dbf1ea27429c644e45f9e26f30c341acd34c7519b2a1663e334621691e810767e9918c2c547b2e23cce915f97d26aac8d0d2fcd3edb7986ad4e2b8a852edebad534cb6c0e9f0797d3563e5409d7e068e48356c67ce519246cd9c560e881453df97cbba562018811e6cf8c327f399d1d1253ab47a19f4a0ccc7c6d86a9603e0551da310ea595d71305c4aad96819120a92cdbaf1f77ec8df9cc7c838c0d4de1e8692dd81da38268d1d71324bcffdafbe5122e4b81828e021e936d83ae8021eac592aa52cd296b5ce392c7173d622f8e07d18f59bb1b08ba15211af6703463b09b593af3c37735296816d9f2e7a369354a5374ea3955e14ca8ac56d5bfe4aef7a21bd825d6ae85530bee5d2aaaa4914981b3dfdb2e92ec2a27c83d74b59e84ff5c056f7d8945745f2efc3dcf28f288c6cd8383700fb2312f7001f24dd40015e436ae23e052fe9070ea9535b9c989898a9bda3d5382cf10e432fae6ccf0c825b3e6436edd3a9f8846e5606f8563931b5f29ba407c5236e5730225dda211a8504ec1817bc935e1fd9a532b648c502df302ed2063aed008fd5676131ac9e95998e9447b02bd29d77e38fcfd2959f2de929b31970335eb2a74348cc6918bc35b9bf749eab0fe304c946cd9e1ca284e6853c42646e60b6b39e0d3fb3c260abfc5c1b4ca3c3770f344118ca7c7f5c1ad1f123f8f369cd60afc3cdb3e9e81968c5c9fa7c8b014ffe0508dd4f0a2a976d5d1ca8fc9ad7a237d92cfe7b41413d934d6e142824b252699397e48e4bac4e91ebc10602720684bd0863773c548f9a2f9724245e47b129ecf65afd7252aac48c8a8d6fd3d888af592a01fb02dc71ed7538a700d3d16243e4621e0fcf9f8ed2b4e11c9fa9a95338bb1dac74a7d9bc4eb8cbf900b634a2a56469c00f5994e4f0934bdb947640e6d67e47d0b621aacd632bfd3c800bd7d93bd329f494a90e06ed51535831bd6e07ac1b4b11434ef3918fa9511813a002913f33f836454798b8d1787fea9a4c4743ba091ed192ed92f4d33e43a226bf9503e1a83a16dd340b3cbbf38af6db0d99201da8de529b4225f3d2fa2aad6621afc6c79ef3537720591edfc681ae6d00ede53ed724fc71b23b90d2e9b7158aaee98d626a4fe029107df2cb5f90147e07ebe423b1519d848af18af365c71bfd0665db46be493bbe99b79a188de0cf3594aef2299f0324075bdce9eb0b87bc29d62401ba4fd6ae48b1ba33261b5b845279becf38ee03e3dc5c45303321c5fac96fd02a3ad8c9e3b02127b320501333c9e6360440d1ad5e64a6239501502dde1a49c9abe33b66098458eee3d611bb06ffcd234a1b9aef4af5021cd61f0de6789f822ee116b5078aae8c129e8391d8987500d322b58edd1595dc570b57341f2df221b94a96ab7fbcf32a8ca9684196455694024623d7ed49f7d66e8dd453c0bae50e0d8b34377b22d0ece059e2c385dfc70b9089fcd27577c51f4d870b5738ee2b68c361a67809c105c7848b68860a829f29930857a9f9d40b14fd2384ac43bafdf43c0661103794c4bd07d1cfdd4681b6aeaefad53d4c1473359bcc5a83b09189352e5bb9a7498dd0effb89c35aad26954551f8b0621374b449bf515630bd3974dca982279733470fdd059aa9c3df403d8f22b38c4709c82d8f12b888e22990350490e16179caf406293cc9e65f116bafcbe96af132f679877061107a2f690a82a8cb46eea57a90abd23798c5937c6fe6b17be3f9bfa01ce117d2c268181b9095bf49f395fea07ca03838de0588c5e2db633e836d64488c1421e653ea52d810d096048c092d0da6e02fa6613890219f51a76148c8588c2487b171a28f17b7a299204874af0131725d793481333be5f08e86ca837a226850b0c1060891603bfecf9e55cddd22c0dbb28d495342d9cc3de8409f72e52a0115141cffe755c74f061c1a770428ccb0ae59536ee6fc074fbfc6cacb51a549d327527e20f8407477e60355863f1153f9ce95641198663c968874e7fdb29407bd771d94fdda8180cbb0358f5874738db705924b8cbe0cd5e1484aeb64542fe8f38667b7c34baf818c63b1e18440e9fba575254d063fd49f24ef26432f4eb323f3836972dca87473e3e9bb26dc3be236c3aae6bc8a6da567442309da0e8450e242fc9db836e2964f2c76a3b80a2c677979882dda7d7ebf62c93664018bcf4ec431fe6b403d49b3b36618b9c07c2d0d4569cb8d52223903debc72ec113955b206c34f1ae5300990ccfc0180f47d91afdb542b6312d12aeff7e19c645dc0b9fe6e3288e9539f6d5870f99882df187bfa6d24d179dfd1dac22212c8b5339f7171a3efc15b760fed8f68538bc5cbd845c2d1ab41f3a6c692820653eaef7930c02fbe6061d93805d73decdbb945572a7c44ed0241982a6e4d2d730898f82b3d9877cb7bca41cc6dcee67aa0c3d6db76f0b0a708ace0031113e48429de5d886c10e9200f68f32263a2fbf44a5992c2459fda7b8796ba796e3a0804fc25992ed2c9a5fe0580a6b809200ecde6caa0364b58be11564dcb9a616766dd7906db5636ee708b0204f38d309466d8d4a162965dd727e29f5a6c133e9b4ed5bafe803e479f9b2a7640c942c4a40b14ac7dc9828546052761a070f6404008f1ec3605836339c3da95a00b4fd81b2cabf88b51d2087d5b83e8c5b69bf96d8c72cbd278dad3bbb42b404b436f84ad688a22948adf60a81090f1e904291503c16e9f54b05fc76c881a5f95f0e732949e95d3f1bae2d3652a14fe0dda2d68879604657171856ef72637def2a96ac47d7b3fe86eb3198f5e0e626f06be86232305f2ae79ffcd2725e48208f9d8d63523f81915acc957563ab627cd6bc68c2a37d59fb0ed77a90aa9d085d6914a8ebada22a2c2d471b5163aeddd799d90fbb10ed6851ace2c4af504b7d572686700a59d6db46d5e42bb83f8e0c0ffe1dfa6582cc0b34c921ff6e85e83188d24906d5c08bb90069639e713051b3102b53e6f703e8210017878add5df68e6f2b108de279c5490e9eef5590185c4a1c744d4e00d244e1245a8805bd30407b1bc488db44870ccfd75a8af104df78efa2fb7ba31f048a263efdb3b63271fff4922bece9a71187108f65744a24f4947dc556b7440cb4fa45d296bb7f724588d1f245125b21ea063500029bd49650237f53899daf1312809552c81c5827341263cc807a29fe84746170cdfa1ff3838399a5645319bcaff674bb70efccdd88b3d3bb2f2d98111413585dc5d5bd5168f43b3f55e58972a5b2b9b3733febf02f931bd436648cb617c3794841aab961fe41277ab07812e1d3bc4ff6f4350a3e615bfba08c3b9480ef57904d3a16f7e916345202e3f93d11f7a7305170cb8c4eb9ac88ace8bbd1f377bdd5855d3162d6723d4435e84ce529b8f276a8927915ac759a0d04e5ca4a9d3da6291f0333b475df527e99fe38f7a4082662e8125936640c26dd1d17cf284ce6e2b17777a05aa0574f7793a6a062cc6f7263f7ab126b4528a17becfdec49ac0f7d8705aa1704af97fb861faa8a466161b2b5c08a5bacc79fe8500b913d65c8d3c52d1fd52d2ab2c9f52196e712455619c1cd3e0f391b274487944240e2ed8858dd0823c801094310024ae3fe4dd1cf5a2b6487b42cc5937bbafb193ee331d87e378258963d49b9da90899bbb4b88e79f78e866b0213f4719f67da7bcc2fce073c01e87c62ea3cdbcd589cfc41281f2f4c757c742d6d1e"
@@ -125,8 +125,7 @@ func testEncryptDecryptIsIdentity(t *testing.T, padding int, initCtr uint32, dat
key := GenerateRandomKey(keyLength)
enc := New(key, padding, initCtr, hashFunc)
- data := make([]byte, dataLength)
- rand.Read(data)
+ data := testutil.RandomBytes(1, dataLength)
encrypted, err := enc.Encrypt(data)
if err != nil {
diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go
index 9fe58f60c8..fb0f761a4a 100644
--- a/swarm/storage/filestore_test.go
+++ b/swarm/storage/filestore_test.go
@@ -23,6 +23,8 @@ import (
"io/ioutil"
"os"
"testing"
+
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
const testDataSize = 0x0001000
@@ -49,9 +51,9 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) {
fileStore := NewFileStore(localStore, NewFileStoreParams())
defer os.RemoveAll("/tmp/bzz")
- reader, slice := GenerateRandomData(testDataSize)
+ slice := testutil.RandomBytes(1, testDataSize)
ctx := context.TODO()
- key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt)
+ key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt)
if err != nil {
t.Fatalf("Store error: %v", err)
}
@@ -63,13 +65,13 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) {
if isEncrypted != toEncrypt {
t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
}
- resultSlice := make([]byte, len(slice))
+ resultSlice := make([]byte, testDataSize)
n, err := resultReader.ReadAt(resultSlice, 0)
if err != io.EOF {
t.Fatalf("Retrieve error: %v", err)
}
- if n != len(slice) {
- t.Fatalf("Slice size error got %d, expected %d.", n, len(slice))
+ if n != testDataSize {
+ t.Fatalf("Slice size error got %d, expected %d.", n, testDataSize)
}
if !bytes.Equal(slice, resultSlice) {
t.Fatalf("Comparison error.")
@@ -114,9 +116,9 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) {
DbStore: db,
}
fileStore := NewFileStore(localStore, NewFileStoreParams())
- reader, slice := GenerateRandomData(testDataSize)
+ slice := testutil.RandomBytes(1, testDataSize)
ctx := context.TODO()
- key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt)
+ key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt)
if err != nil {
t.Errorf("Store error: %v", err)
}
diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go
index 9feb687413..bd4f6b9162 100644
--- a/swarm/storage/ldbstore.go
+++ b/swarm/storage/ldbstore.go
@@ -57,7 +57,6 @@ var (
var (
keyIndex = byte(0)
- keyOldData = byte(1)
keyAccessCnt = []byte{2}
keyEntryCnt = []byte{3}
keyDataIdx = []byte{4}
@@ -186,6 +185,20 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) {
return s, nil
}
+// MarkAccessed increments the access counter as a best effort for a chunk, so
+// the chunk won't get garbage collected.
+func (s *LDBStore) MarkAccessed(addr Address) {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ if s.closed {
+ return
+ }
+
+ proximity := s.po(addr)
+ s.tryAccessIdx(addr, proximity)
+}
+
// initialize and set values for processing of gc round
func (s *LDBStore) startGC(c int) {
@@ -271,6 +284,10 @@ func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte {
return val
}
+func parseIdxKey(key []byte) (byte, []byte) {
+ return key[0], key[1:]
+}
+
func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) {
index = &dpaDBIndex{
Idx: binary.BigEndian.Uint64(val[1:]),
@@ -349,6 +366,7 @@ func (s *LDBStore) collectGarbage() error {
s.delete(s.gc.batch.Batch, index, keyIdx, po)
singleIterationCount++
s.gc.count++
+ log.Trace("garbage collect enqueued chunk for deletion", "key", hash)
// break if target is not on max garbage batch boundary
if s.gc.count >= s.gc.target {
@@ -489,7 +507,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
}
}
-//Cleanup iterates over the database and deletes chunks if they pass the `f` condition
+// Cleanup iterates over the database and deletes chunks if they pass the `f` condition
func (s *LDBStore) Cleanup(f func(*chunk) bool) {
var errorsFound, removed, total int
@@ -554,47 +572,157 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) {
log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed))
}
-func (s *LDBStore) ReIndex() {
- //Iterates over the database and checks that there are no faulty chunks
+// CleanGCIndex rebuilds the garbage collector index from scratch, while
+// removing inconsistent elements, e.g., indices with missing data chunks.
+// WARN: it's a pretty heavy, long running function.
+func (s *LDBStore) CleanGCIndex() error {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ batch := leveldb.Batch{}
+
+ var okEntryCount uint64
+ var totalEntryCount uint64
+
+ // throw out all gc indices, we will rebuild from cleaned index
it := s.db.NewIterator()
- startPosition := []byte{keyOldData}
- it.Seek(startPosition)
- var key []byte
- var errorsFound, total int
+ it.Seek([]byte{keyGCIdx})
+ var gcDeletes int
for it.Valid() {
- key = it.Key()
- if (key == nil) || (key[0] != keyOldData) {
+ rowType, _ := parseIdxKey(it.Key())
+ if rowType != keyGCIdx {
break
}
- data := it.Value()
- hasher := s.hashfunc()
- hasher.Write(data)
- hash := hasher.Sum(nil)
-
- newKey := make([]byte, 10)
- oldCntKey := make([]byte, 2)
- newCntKey := make([]byte, 2)
- oldCntKey[0] = keyDistanceCnt
- newCntKey[0] = keyDistanceCnt
- key[0] = keyData
- key[1] = s.po(Address(key[1:]))
- oldCntKey[1] = key[1]
- newCntKey[1] = s.po(Address(newKey[1:]))
- copy(newKey[2:], key[1:])
- newValue := append(hash, data...)
-
- batch := new(leveldb.Batch)
- batch.Delete(key)
- s.bucketCnt[oldCntKey[1]]--
- batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]]))
- batch.Put(newKey, newValue)
- s.bucketCnt[newCntKey[1]]++
- batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]]))
- s.db.Write(batch)
+ batch.Delete(it.Key())
+ gcDeletes++
it.Next()
}
+ log.Debug("gc", "deletes", gcDeletes)
+ if err := s.db.Write(&batch); err != nil {
+ return err
+ }
+ batch.Reset()
+
it.Release()
- log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total))
+
+ // corrected po index pointer values
+ var poPtrs [256]uint64
+
+ // set to true if chunk count not on 4096 iteration boundary
+ var doneIterating bool
+
+ // last key index in previous iteration
+ lastIdxKey := []byte{keyIndex}
+
+ // counter for debug output
+ var cleanBatchCount int
+
+ // go through all key index entries
+ for !doneIterating {
+ cleanBatchCount++
+ var idxs []dpaDBIndex
+ var chunkHashes [][]byte
+ var pos []uint8
+ it := s.db.NewIterator()
+
+ it.Seek(lastIdxKey)
+
+ // 4096 is just a nice number, don't look for any hidden meaning here...
+ var i int
+ for i = 0; i < 4096; i++ {
+
+ // this really shouldn't happen unless database is empty
+ // but let's keep it to be safe
+ if !it.Valid() {
+ doneIterating = true
+ break
+ }
+
+ // if it's not keyindex anymore we're done iterating
+ rowType, chunkHash := parseIdxKey(it.Key())
+ if rowType != keyIndex {
+ doneIterating = true
+ break
+ }
+
+ // decode the retrieved index
+ var idx dpaDBIndex
+ err := decodeIndex(it.Value(), &idx)
+ if err != nil {
+ return fmt.Errorf("corrupt index: %v", err)
+ }
+ po := s.po(chunkHash)
+ lastIdxKey = it.Key()
+
+ // if we don't find the data key, remove the entry
+ // if we find it, add to the array of new gc indices to create
+ dataKey := getDataKey(idx.Idx, po)
+ _, err = s.db.Get(dataKey)
+ if err != nil {
+ log.Warn("deleting inconsistent index (missing data)", "key", chunkHash)
+ batch.Delete(it.Key())
+ } else {
+ idxs = append(idxs, idx)
+ chunkHashes = append(chunkHashes, chunkHash)
+ pos = append(pos, po)
+ okEntryCount++
+ if idx.Idx > poPtrs[po] {
+ poPtrs[po] = idx.Idx
+ }
+ }
+ totalEntryCount++
+ it.Next()
+ }
+ it.Release()
+
+ // flush the key index corrections
+ err := s.db.Write(&batch)
+ if err != nil {
+ return err
+ }
+ batch.Reset()
+
+ // add correct gc indices
+ for i, okIdx := range idxs {
+ gcIdxKey := getGCIdxKey(&okIdx)
+ gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i])
+ batch.Put(gcIdxKey, gcIdxData)
+ log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData)
+ }
+
+ // flush them
+ err = s.db.Write(&batch)
+ if err != nil {
+ return err
+ }
+ batch.Reset()
+
+ log.Debug("clean gc index pass", "batch", cleanBatchCount, "checked", i, "kept", len(idxs))
+ }
+
+ log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len())
+
+ // lastly add updated entry count
+ var entryCount [8]byte
+ binary.BigEndian.PutUint64(entryCount[:], okEntryCount)
+ batch.Put(keyEntryCnt, entryCount[:])
+
+ // and add the new po index pointers
+ var poKey [2]byte
+ poKey[0] = keyDistanceCnt
+ for i, poPtr := range poPtrs {
+ poKey[1] = uint8(i)
+ if poPtr == 0 {
+ batch.Delete(poKey[:])
+ } else {
+ var idxCount [8]byte
+ binary.BigEndian.PutUint64(idxCount[:], poPtr)
+ batch.Put(poKey[:], idxCount[:])
+ }
+ }
+
+ // if you made it this far your harddisk has survived. Congratulations
+ return s.db.Write(&batch)
}
// Delete is removes a chunk and updates indices.
@@ -685,12 +813,7 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error {
idata, err := s.db.Get(ikey)
if err != nil {
s.doPut(chunk, &index, po)
- } else {
- log.Debug("ldbstore.put: chunk already exists, only update access", "key", chunk.Address(), "po", po)
- decodeIndex(idata, &index)
}
- index.Access = s.accessCnt
- s.accessCnt++
idata = encodeIndex(&index)
s.batch.Put(ikey, idata)
@@ -723,7 +846,8 @@ func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) {
s.entryCnt++
dbEntryCount.Inc(1)
s.dataIdx++
-
+ index.Access = s.accessCnt
+ s.accessCnt++
cntKey := make([]byte, 2)
cntKey[0] = keyDistanceCnt
cntKey[1] = po
@@ -796,28 +920,33 @@ func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte {
}
}
-// try to find index; if found, update access cnt and return true
-func (s *LDBStore) tryAccessIdx(ikey []byte, po uint8, index *dpaDBIndex) bool {
+// tryAccessIdx tries to find index entry. If found then increments the access
+// count for garbage collection and returns the index entry and true for found,
+// otherwise returns nil and false.
+func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) {
+ ikey := getIndexKey(addr)
idata, err := s.db.Get(ikey)
if err != nil {
- return false
+ return nil, false
}
+
+ index := new(dpaDBIndex)
decodeIndex(idata, index)
oldGCIdxKey := getGCIdxKey(index)
s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
- s.accessCnt++
index.Access = s.accessCnt
idata = encodeIndex(index)
+ s.accessCnt++
s.batch.Put(ikey, idata)
newGCIdxKey := getGCIdxKey(index)
- newGCIdxData := getGCIdxValue(index, po, ikey)
+ newGCIdxData := getGCIdxValue(index, po, ikey[1:])
s.batch.Delete(oldGCIdxKey)
s.batch.Put(newGCIdxKey, newGCIdxData)
select {
case s.batchesC <- struct{}{}:
default:
}
- return true
+ return index, true
}
// GetSchema is returning the current named schema of the datastore as read from LevelDB
@@ -828,7 +957,7 @@ func (s *LDBStore) GetSchema() (string, error) {
data, err := s.db.Get(keySchema)
if err != nil {
if err == leveldb.ErrNotFound {
- return "", nil
+ return DbSchemaNone, nil
}
return "", err
}
@@ -858,12 +987,12 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error)
// TODO: To conform with other private methods of this object indices should not be updated
func (s *LDBStore) get(addr Address) (chunk *chunk, err error) {
- var indx dpaDBIndex
if s.closed {
return nil, ErrDBClosed
}
proximity := s.po(addr)
- if s.tryAccessIdx(getIndexKey(addr), proximity, &indx) {
+ index, found := s.tryAccessIdx(addr, proximity)
+ if found {
var data []byte
if s.getDataFunc != nil {
// if getDataFunc is defined, use it to retrieve the chunk data
@@ -874,12 +1003,12 @@ func (s *LDBStore) get(addr Address) (chunk *chunk, err error) {
}
} else {
// default DbStore functionality to retrieve chunk data
- datakey := getDataKey(indx.Idx, proximity)
+ datakey := getDataKey(index.Idx, proximity)
data, err = s.db.Get(datakey)
- log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", indx.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity)
+ log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", index.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity)
if err != nil {
log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err)
- s.deleteNow(&indx, getIndexKey(addr), s.po(addr))
+ s.deleteNow(index, getIndexKey(addr), s.po(addr))
return
}
}
diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go
index 48af8c57c0..e8b9ae39bc 100644
--- a/swarm/storage/ldbstore_test.go
+++ b/swarm/storage/ldbstore_test.go
@@ -19,6 +19,7 @@ package storage
import (
"bytes"
"context"
+ "encoding/binary"
"fmt"
"io/ioutil"
"os"
@@ -31,7 +32,6 @@ import (
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
-
ldberrors "github.com/syndtr/goleveldb/leveldb/errors"
)
@@ -105,6 +105,46 @@ func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) {
testStoreCorrect(db, n, chunksize, t)
}
+func TestMarkAccessed(t *testing.T) {
+ db, cleanup, err := newTestDbStore(false, true)
+ defer cleanup()
+ if err != nil {
+ t.Fatalf("init dbStore failed: %v", err)
+ }
+
+ h := GenerateRandomChunk(ch.DefaultSize)
+
+ db.Put(context.Background(), h)
+
+ var index dpaDBIndex
+ addr := h.Address()
+ idxk := getIndexKey(addr)
+
+ idata, err := db.db.Get(idxk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ decodeIndex(idata, &index)
+
+ if index.Access != 0 {
+ t.Fatalf("Expected the access index to be %d, but it is %d", 0, index.Access)
+ }
+
+ db.MarkAccessed(addr)
+ db.writeCurrentBatch()
+
+ idata, err = db.db.Get(idxk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ decodeIndex(idata, &index)
+
+ if index.Access != 1 {
+ t.Fatalf("Expected the access index to be %d, but it is %d", 1, index.Access)
+ }
+
+}
+
func TestDbStoreRandom_1(t *testing.T) {
testDbStoreRandom(1, 0, false, t)
}
@@ -304,17 +344,18 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) {
func TestLDBStoreCollectGarbage(t *testing.T) {
// below max ronud
- cap := defaultMaxGCRound / 2
+ initialCap := defaultMaxGCRound / 100
+ cap := initialCap / 2
t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage)
t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage)
// at max round
- cap = defaultMaxGCRound
+ cap = initialCap
t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage)
t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage)
// more than max around, not on threshold
- cap = defaultMaxGCRound * 1.1
+ cap = initialCap + 500
t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage)
t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage)
@@ -538,7 +579,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) {
// TestLDBStoreCollectGarbageAccessUnlikeIndex tests garbage collection where accesscount differs from indexcount
func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) {
- capacity := defaultMaxGCRound * 2
+ capacity := defaultMaxGCRound / 100 * 2
n := capacity - 1
ldb, cleanup := newLDBStore(t)
@@ -584,6 +625,177 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) {
log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt)
}
+func TestCleanIndex(t *testing.T) {
+ capacity := 5000
+ n := 3
+
+ ldb, cleanup := newLDBStore(t)
+ ldb.setCapacity(uint64(capacity))
+ defer cleanup()
+
+ chunks, err := mputRandomChunks(ldb, n, 4096)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // remove the data of the first chunk
+ po := ldb.po(chunks[0].Address()[:])
+ dataKey := make([]byte, 10)
+ dataKey[0] = keyData
+ dataKey[1] = byte(po)
+ // dataKey[2:10] = first chunk has storageIdx 0 on [2:10]
+ if _, err := ldb.db.Get(dataKey); err != nil {
+ t.Fatal(err)
+ }
+ if err := ldb.db.Delete(dataKey); err != nil {
+ t.Fatal(err)
+ }
+
+ // remove the gc index row for the first chunk
+ gcFirstCorrectKey := make([]byte, 9)
+ gcFirstCorrectKey[0] = keyGCIdx
+ if err := ldb.db.Delete(gcFirstCorrectKey); err != nil {
+ t.Fatal(err)
+ }
+
+ // warp the gc data of the second chunk
+ // this data should be correct again after the clean
+ gcSecondCorrectKey := make([]byte, 9)
+ gcSecondCorrectKey[0] = keyGCIdx
+ binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(1))
+ gcSecondCorrectVal, err := ldb.db.Get(gcSecondCorrectKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ warpedGCVal := make([]byte, len(gcSecondCorrectVal)+1)
+ copy(warpedGCVal[1:], gcSecondCorrectVal)
+ if err := ldb.db.Delete(gcSecondCorrectKey); err != nil {
+ t.Fatal(err)
+ }
+ if err := ldb.db.Put(gcSecondCorrectKey, warpedGCVal); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ldb.CleanGCIndex(); err != nil {
+ t.Fatal(err)
+ }
+
+ // the index without corresponding data should have been deleted
+ idxKey := make([]byte, 33)
+ idxKey[0] = keyIndex
+ copy(idxKey[1:], chunks[0].Address())
+ if _, err := ldb.db.Get(idxKey); err == nil {
+ t.Fatalf("expected chunk 0 idx to be pruned: %v", idxKey)
+ }
+
+ // the two other indices should be present
+ copy(idxKey[1:], chunks[1].Address())
+ if _, err := ldb.db.Get(idxKey); err != nil {
+ t.Fatalf("expected chunk 1 idx to be present: %v", idxKey)
+ }
+
+ copy(idxKey[1:], chunks[2].Address())
+ if _, err := ldb.db.Get(idxKey); err != nil {
+ t.Fatalf("expected chunk 2 idx to be present: %v", idxKey)
+ }
+
+ // first gc index should still be gone
+ if _, err := ldb.db.Get(gcFirstCorrectKey); err == nil {
+ t.Fatalf("expected gc 0 idx to be pruned: %v", idxKey)
+ }
+
+ // second gc index should still be fixed
+ if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil {
+ t.Fatalf("expected gc 1 idx to be present: %v", idxKey)
+ }
+
+ // third gc index should be unchanged
+ binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(2))
+ if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil {
+ t.Fatalf("expected gc 2 idx to be present: %v", idxKey)
+ }
+
+ c, err := ldb.db.Get(keyEntryCnt)
+ if err != nil {
+ t.Fatalf("expected gc 2 idx to be present: %v", idxKey)
+ }
+
+ // entrycount should now be one less
+ entryCount := binary.BigEndian.Uint64(c)
+ if entryCount != 2 {
+ t.Fatalf("expected entrycnt to be 2, was %d", c)
+ }
+
+ // the chunks might accidentally be in the same bin
+ // if so that bin counter will now be 2 - the highest added index.
+ // if not, the total of them will be 3
+ poBins := []uint8{ldb.po(chunks[1].Address()), ldb.po(chunks[2].Address())}
+ if poBins[0] == poBins[1] {
+ poBins = poBins[:1]
+ }
+
+ var binTotal uint64
+ var currentBin [2]byte
+ currentBin[0] = keyDistanceCnt
+ if len(poBins) == 1 {
+ currentBin[1] = poBins[0]
+ c, err := ldb.db.Get(currentBin[:])
+ if err != nil {
+ t.Fatalf("expected gc 2 idx to be present: %v", idxKey)
+ }
+ binCount := binary.BigEndian.Uint64(c)
+ if binCount != 2 {
+ t.Fatalf("expected entrycnt to be 2, was %d", binCount)
+ }
+ } else {
+ for _, bin := range poBins {
+ currentBin[1] = bin
+ c, err := ldb.db.Get(currentBin[:])
+ if err != nil {
+ t.Fatalf("expected gc 2 idx to be present: %v", idxKey)
+ }
+ binCount := binary.BigEndian.Uint64(c)
+ binTotal += binCount
+
+ }
+ if binTotal != 3 {
+ t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal)
+ }
+ }
+
+ // check that the iterator quits properly
+ chunks, err = mputRandomChunks(ldb, 4100, 4096)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ po = ldb.po(chunks[4099].Address()[:])
+ dataKey = make([]byte, 10)
+ dataKey[0] = keyData
+ dataKey[1] = byte(po)
+ binary.BigEndian.PutUint64(dataKey[2:], 4099+3)
+ if _, err := ldb.db.Get(dataKey); err != nil {
+ t.Fatal(err)
+ }
+ if err := ldb.db.Delete(dataKey); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ldb.CleanGCIndex(); err != nil {
+ t.Fatal(err)
+ }
+
+ // entrycount should now be one less of added chunks
+ c, err = ldb.db.Get(keyEntryCnt)
+ if err != nil {
+ t.Fatalf("expected gc 2 idx to be present: %v", idxKey)
+ }
+ entryCount = binary.BigEndian.Uint64(c)
+ if entryCount != 4099+2 {
+ t.Fatalf("expected entrycnt to be 2, was %d", c)
+ }
+}
+
func waitGc(ctx context.Context, ldb *LDBStore) {
<-ldb.gc.runC
ldb.gc.runC <- struct{}{}
diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go
index 4fa6fb2f60..111821ff65 100644
--- a/swarm/storage/localstore.go
+++ b/swarm/storage/localstore.go
@@ -153,6 +153,7 @@ func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err e
if err == nil {
metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1)
+ go ls.DbStore.MarkAccessed(addr)
return chunk, nil
}
@@ -193,33 +194,51 @@ func (ls *LocalStore) Close() {
ls.DbStore.Close()
}
-// Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match
+// Migrate checks the datastore schema vs the runtime schema and runs
+// migrations if they don't match
func (ls *LocalStore) Migrate() error {
- schema, err := ls.DbStore.GetSchema()
+ actualDbSchema, err := ls.DbStore.GetSchema()
if err != nil {
log.Error(err.Error())
return err
}
- log.Debug("found schema", "schema", schema, "runtime-schema", CurrentDbSchema)
- if schema != CurrentDbSchema {
- // run migrations
-
- if schema == "" {
- log.Debug("running migrations for", "schema", schema, "runtime-schema", CurrentDbSchema)
-
- // delete chunks that are not valid, i.e. chunks that do not pass any of the ls.Validators
- ls.DbStore.Cleanup(func(c *chunk) bool {
- return !ls.isValid(c)
- })
-
- err := ls.DbStore.PutSchema(DbSchemaPurity)
- if err != nil {
- log.Error(err.Error())
- return err
- }
- }
+ if actualDbSchema == CurrentDbSchema {
+ return nil
}
+ log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema)
+
+ if actualDbSchema == DbSchemaNone {
+ ls.migrateFromNoneToPurity()
+ actualDbSchema = DbSchemaPurity
+ }
+
+ if err := ls.DbStore.PutSchema(actualDbSchema); err != nil {
+ return err
+ }
+
+ if actualDbSchema == DbSchemaPurity {
+ if err := ls.migrateFromPurityToHalloween(); err != nil {
+ return err
+ }
+ actualDbSchema = DbSchemaHalloween
+ }
+
+ if err := ls.DbStore.PutSchema(actualDbSchema); err != nil {
+ return err
+ }
return nil
}
+
+func (ls *LocalStore) migrateFromNoneToPurity() {
+ // delete chunks that are not valid, i.e. chunks that do not pass
+ // any of the ls.Validators
+ ls.DbStore.Cleanup(func(c *chunk) bool {
+ return !ls.isValid(c)
+ })
+}
+
+func (ls *LocalStore) migrateFromPurityToHalloween() error {
+ return ls.DbStore.CleanGCIndex()
+}
diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go
index 10f43f30f7..7a07726d1c 100644
--- a/swarm/storage/localstore_test.go
+++ b/swarm/storage/localstore_test.go
@@ -21,6 +21,7 @@ import (
"io/ioutil"
"os"
"testing"
+ "time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
)
@@ -144,3 +145,67 @@ func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs []
}
return hs, errs
}
+
+// TestGetFrequentlyAccessedChunkWontGetGarbageCollected tests that the most
+// frequently accessed chunk is not garbage collected from LDBStore, i.e.,
+// from disk when we are at the capacity and garbage collector runs. For that
+// we start putting random chunks into the DB while continuously accessing the
+// chunk we care about then check if we can still retrieve it from disk.
+func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) {
+ ldbCap := defaultGCRatio
+ store, cleanup := setupLocalStore(t, ldbCap)
+ defer cleanup()
+
+ var chunks []Chunk
+ for i := 0; i < ldbCap; i++ {
+ chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize))
+ }
+
+ mostAccessed := chunks[0].Address()
+ for _, chunk := range chunks {
+ if err := store.Put(context.Background(), chunk); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := store.Get(context.Background(), mostAccessed); err != nil {
+ t.Fatal(err)
+ }
+ // Add time for MarkAccessed() to be able to finish in a separate Goroutine
+ time.Sleep(1 * time.Millisecond)
+ }
+
+ store.DbStore.collectGarbage()
+ if _, err := store.DbStore.Get(context.Background(), mostAccessed); err != nil {
+ t.Logf("most frequntly accessed chunk not found on disk (key: %v)", mostAccessed)
+ t.Fatal(err)
+ }
+
+}
+
+func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) {
+ t.Helper()
+
+ var err error
+ datadir, err := ioutil.TempDir("", "storage")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ params := &LocalStoreParams{
+ StoreParams: NewStoreParams(uint64(ldbCap), uint(ldbCap), nil, nil),
+ }
+ params.Init(datadir)
+
+ store, err := NewLocalStore(params, nil)
+ if err != nil {
+ _ = os.RemoveAll(datadir)
+ t.Fatal(err)
+ }
+
+ cleanup = func() {
+ store.Close()
+ _ = os.RemoveAll(datadir)
+ }
+
+ return store, cleanup
+}
diff --git a/swarm/storage/mock/rpc/rpc_test.go b/swarm/storage/mock/rpc/rpc_test.go
index 52b634a445..f62340edec 100644
--- a/swarm/storage/mock/rpc/rpc_test.go
+++ b/swarm/storage/mock/rpc/rpc_test.go
@@ -37,5 +37,5 @@ func TestRPCStore(t *testing.T) {
store := NewGlobalStore(rpc.DialInProc(server))
defer store.Close()
- test.MockStore(t, store, 100)
+ test.MockStore(t, store, 30)
}
diff --git a/swarm/storage/schema.go b/swarm/storage/schema.go
index fb8498a293..91847ca0f9 100644
--- a/swarm/storage/schema.go
+++ b/swarm/storage/schema.go
@@ -1,6 +1,17 @@
package storage
+// The DB schema we want to use. The actual/current DB schema might differ
+// until migrations are run.
+const CurrentDbSchema = DbSchemaHalloween
+
+// There was a time when we had no schema at all.
+const DbSchemaNone = ""
+
// "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5
const DbSchemaPurity = "purity"
-const CurrentDbSchema = DbSchemaPurity
+// "halloween" is here because we had a screw in the garbage collector index.
+// Because of that we had to rebuild the GC index to get rid of erroneous
+// entries and that takes a long time. This schema is used for bookkeeping,
+// so rebuild index will run just once.
+const DbSchemaHalloween = "halloween"
diff --git a/swarm/storage/types.go b/swarm/storage/types.go
index ada86831fc..42557766ef 100644
--- a/swarm/storage/types.go
+++ b/swarm/storage/types.go
@@ -25,7 +25,6 @@ import (
"fmt"
"hash"
"io"
- "io/ioutil"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/sha3"
@@ -244,26 +243,13 @@ func GenerateRandomChunk(dataSize int64) Chunk {
}
func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) {
- if dataSize > ch.DefaultSize {
- dataSize = ch.DefaultSize
- }
for i := 0; i < count; i++ {
- ch := GenerateRandomChunk(ch.DefaultSize)
+ ch := GenerateRandomChunk(dataSize)
chunks = append(chunks, ch)
}
return chunks
}
-func GenerateRandomData(l int) (r io.Reader, slice []byte) {
- slice, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(l)))
- if err != nil {
- panic("rand error")
- }
- // log.Warn("generate random data", "len", len(slice), "data", common.Bytes2Hex(slice))
- r = io.LimitReader(bytes.NewReader(slice), int64(l))
- return r, slice
-}
-
// Size, Seek, Read, ReadAt
type LazySectionReader interface {
Context() context.Context
diff --git a/swarm/swap/swap.go b/swarm/swap/swap.go
new file mode 100644
index 0000000000..137eb141d9
--- /dev/null
+++ b/swarm/swap/swap.go
@@ -0,0 +1,93 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package swap
+
+import (
+ "errors"
+ "fmt"
+ "strconv"
+ "sync"
+
+ "github.com/ethereum/go-ethereum/p2p/enode"
+ "github.com/ethereum/go-ethereum/p2p/protocols"
+ "github.com/ethereum/go-ethereum/swarm/log"
+ "github.com/ethereum/go-ethereum/swarm/state"
+)
+
+// SwAP Swarm Accounting Protocol
+// a peer to peer micropayment system
+// A node maintains an individual balance with every peer
+// Only messages which have a price will be accounted for
+type Swap struct {
+ stateStore state.Store //stateStore is needed in order to keep balances across sessions
+ lock sync.RWMutex //lock the balances
+ balances map[enode.ID]int64 //map of balances for each peer
+}
+
+// New - swap constructor
+func New(stateStore state.Store) (swap *Swap) {
+ swap = &Swap{
+ stateStore: stateStore,
+ balances: make(map[enode.ID]int64),
+ }
+ return
+}
+
+//Swap implements the protocols.Balance interface
+//Add is the (sole) accounting function
+func (s *Swap) Add(amount int64, peer *protocols.Peer) (err error) {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ //load existing balances from the state store
+ err = s.loadState(peer)
+ if err != nil && err != state.ErrNotFound {
+ return
+ }
+ //adjust the balance
+ //if amount is negative, it will decrease, otherwise increase
+ s.balances[peer.ID()] += amount
+ //save the new balance to the state store
+ peerBalance := s.balances[peer.ID()]
+ err = s.stateStore.Put(peer.ID().String(), &peerBalance)
+
+ log.Debug(fmt.Sprintf("balance for peer %s: %s", peer.ID().String(), strconv.FormatInt(peerBalance, 10)))
+ return err
+}
+
+//GetPeerBalance returns the balance for a given peer
+func (swap *Swap) GetPeerBalance(peer enode.ID) (int64, error) {
+ swap.lock.RLock()
+ defer swap.lock.RUnlock()
+ if p, ok := swap.balances[peer]; ok {
+ return p, nil
+ }
+ return 0, errors.New("Peer not found")
+}
+
+//load balances from the state store (persisted)
+func (s *Swap) loadState(peer *protocols.Peer) (err error) {
+ var peerBalance int64
+ peerID := peer.ID()
+ //only load if the current instance doesn't already have this peer's
+ //balance in memory
+ if _, ok := s.balances[peerID]; !ok {
+ err = s.stateStore.Get(peerID.String(), &peerBalance)
+ s.balances[peerID] = peerBalance
+ }
+ return
+}
diff --git a/swarm/swap/swap_test.go b/swarm/swap/swap_test.go
new file mode 100644
index 0000000000..f2e3ba168a
--- /dev/null
+++ b/swarm/swap/swap_test.go
@@ -0,0 +1,184 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package swap
+
+import (
+ "flag"
+ "fmt"
+ "io/ioutil"
+ mrand "math/rand"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/p2p"
+ "github.com/ethereum/go-ethereum/p2p/protocols"
+ "github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+ "github.com/ethereum/go-ethereum/swarm/state"
+ colorable "github.com/mattn/go-colorable"
+)
+
+var (
+ loglevel = flag.Int("loglevel", 2, "verbosity of logs")
+)
+
+func init() {
+ flag.Parse()
+ mrand.Seed(time.Now().UnixNano())
+
+ log.PrintOrigins(true)
+ log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+}
+
+//Test getting a peer's balance
+func TestGetPeerBalance(t *testing.T) {
+ //create a test swap account
+ swap, testDir := createTestSwap(t)
+ defer os.RemoveAll(testDir)
+
+ //test for correct value
+ testPeer := newDummyPeer()
+ swap.balances[testPeer.ID()] = 888
+ b, err := swap.GetPeerBalance(testPeer.ID())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if b != 888 {
+ t.Fatalf("Expected peer's balance to be %d, but is %d", 888, b)
+ }
+
+ //test for inexistent node
+ id := adapters.RandomNodeConfig().ID
+ _, err = swap.GetPeerBalance(id)
+ if err == nil {
+ t.Fatal("Expected call to fail, but it didn't!")
+ }
+ if err.Error() != "Peer not found" {
+ t.Fatalf("Expected test to fail with %s, but is %s", "Peer not found", err.Error())
+ }
+}
+
+//Test that repeated bookings do correct accounting
+func TestRepeatedBookings(t *testing.T) {
+ //create a test swap account
+ swap, testDir := createTestSwap(t)
+ defer os.RemoveAll(testDir)
+
+ testPeer := newDummyPeer()
+ amount := mrand.Intn(100)
+ cnt := 1 + mrand.Intn(10)
+ for i := 0; i < cnt; i++ {
+ swap.Add(int64(amount), testPeer.Peer)
+ }
+ expectedBalance := int64(cnt * amount)
+ realBalance := swap.balances[testPeer.ID()]
+ if expectedBalance != realBalance {
+ t.Fatal(fmt.Sprintf("After %d credits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance))
+ }
+
+ testPeer2 := newDummyPeer()
+ amount = mrand.Intn(100)
+ cnt = 1 + mrand.Intn(10)
+ for i := 0; i < cnt; i++ {
+ swap.Add(0-int64(amount), testPeer2.Peer)
+ }
+ expectedBalance = int64(0 - (cnt * amount))
+ realBalance = swap.balances[testPeer2.ID()]
+ if expectedBalance != realBalance {
+ t.Fatal(fmt.Sprintf("After %d debits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance))
+ }
+
+ //mixed debits and credits
+ amount1 := mrand.Intn(100)
+ amount2 := mrand.Intn(55)
+ amount3 := mrand.Intn(999)
+ swap.Add(int64(amount1), testPeer2.Peer)
+ swap.Add(int64(0-amount2), testPeer2.Peer)
+ swap.Add(int64(0-amount3), testPeer2.Peer)
+
+ expectedBalance = expectedBalance + int64(amount1-amount2-amount3)
+ realBalance = swap.balances[testPeer2.ID()]
+
+ if expectedBalance != realBalance {
+ t.Fatal(fmt.Sprintf("After mixed debits and credits, expected balance to be: %d, but is: %d", expectedBalance, realBalance))
+ }
+}
+
+//try restoring a balance from state store
+//this is simulated by creating a node,
+//assigning it an arbitrary balance,
+//then closing the state store.
+//Then we re-open the state store and check that
+//the balance is still the same
+func TestRestoreBalanceFromStateStore(t *testing.T) {
+ //create a test swap account
+ swap, testDir := createTestSwap(t)
+ defer os.RemoveAll(testDir)
+
+ testPeer := newDummyPeer()
+ swap.balances[testPeer.ID()] = -8888
+
+ tmpBalance := swap.balances[testPeer.ID()]
+ swap.stateStore.Put(testPeer.ID().String(), &tmpBalance)
+
+ swap.stateStore.Close()
+ swap.stateStore = nil
+
+ stateStore, err := state.NewDBStore(testDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var newBalance int64
+ stateStore.Get(testPeer.ID().String(), &newBalance)
+
+ //compare the balances
+ if tmpBalance != newBalance {
+ t.Fatal(fmt.Sprintf("Unexpected balance value after sending cheap message test. Expected balance: %d, balance is: %d",
+ tmpBalance, newBalance))
+ }
+}
+
+//create a test swap account
+//creates a stateStore for persistence and a Swap account
+func createTestSwap(t *testing.T) (*Swap, string) {
+ dir, err := ioutil.TempDir("", "swap_test_store")
+ if err != nil {
+ t.Fatal(err)
+ }
+ stateStore, err2 := state.NewDBStore(dir)
+ if err2 != nil {
+ t.Fatal(err2)
+ }
+ swap := New(stateStore)
+ return swap, dir
+}
+
+type dummyPeer struct {
+ *protocols.Peer
+}
+
+//creates a dummy protocols.Peer with dummy MsgReadWriter
+func newDummyPeer() *dummyPeer {
+ id := adapters.RandomNodeConfig().ID
+ protoPeer := protocols.NewPeer(p2p.NewPeer(id, "testPeer", nil), nil, nil)
+ dummy := &dummyPeer{
+ Peer: protoPeer,
+ }
+ return dummy
+}
diff --git a/swarm/swarm.go b/swarm/swarm.go
index 7214abbda4..dc3756d3af 100644
--- a/swarm/swarm.go
+++ b/swarm/swarm.go
@@ -51,6 +51,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
+ "github.com/ethereum/go-ethereum/swarm/swap"
"github.com/ethereum/go-ethereum/swarm/tracing"
)
@@ -78,6 +79,7 @@ type Swarm struct {
netStore *storage.NetStore
sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit
ps *pss.Pss
+ swap *swap.Swap
tracerClose io.Closer
}
@@ -171,23 +173,37 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
delivery := stream.NewDelivery(to, self.netStore)
self.netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, config.DeliverySkipCheck).New
+ if config.SwapEnabled {
+ balancesStore, err := state.NewDBStore(filepath.Join(config.Path, "balances.db"))
+ if err != nil {
+ return nil, err
+ }
+ self.swap = swap.New(balancesStore)
+ }
+
var nodeID enode.ID
if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil {
return nil, err
}
+
+ syncing := stream.SyncingAutoSubscribe
+ if !config.SyncEnabled || config.LightNodeEnabled {
+ syncing = stream.SyncingDisabled
+ }
+
+ retrieval := stream.RetrievalEnabled
+ if config.LightNodeEnabled {
+ retrieval = stream.RetrievalClientOnly
+ }
+
registryOptions := &stream.RegistryOptions{
SkipCheck: config.DeliverySkipCheck,
- DoSync: config.SyncEnabled,
- DoRetrieve: true,
- DoServeRetrieve: true,
+ Syncing: syncing,
+ Retrieval: retrieval,
SyncUpdateDelay: config.SyncUpdateDelay,
MaxPeerServers: config.MaxStreamPeerServers,
}
- if config.LightNodeEnabled {
- registryOptions.DoSync = false
- registryOptions.DoRetrieve = false
- }
- self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions)
+ self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions, self.swap)
// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams)
@@ -210,7 +226,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
log.Debug("Setup local storage")
- self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run)
+ self.bzz = network.NewBzz(bzzconfig, to, stateStore, self.streamer.GetSpec(), self.streamer.Run)
// Pss = postal service over swarm (devp2p over bzz)
self.ps, err = pss.NewPss(to, config.Pss)
@@ -347,7 +363,9 @@ func (self *Swarm) Start(srv *p2p.Server) error {
newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String()))
log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr))
// set chequebook
- if self.config.SwapEnabled {
+ //TODO: Currently if swap is enabled and no chequebook (or inexistent) contract is provided, the node would crash.
+ //Once we integrate back the contracts, this check MUST be revisited
+ if self.config.SwapEnabled && self.config.SwapAPI != "" {
ctx := context.Background() // The initial setup has no deadline.
err := self.SetChequebook(ctx)
if err != nil {
diff --git a/swarm/testutil/file.go b/swarm/testutil/file.go
index ecb0d971ed..70732aa92e 100644
--- a/swarm/testutil/file.go
+++ b/swarm/testutil/file.go
@@ -17,8 +17,10 @@
package testutil
import (
+ "bytes"
"io"
"io/ioutil"
+ "math/rand"
"os"
"strings"
"testing"
@@ -42,3 +44,22 @@ func TempFileWithContent(t *testing.T, content string) string {
}
return tempFile.Name()
}
+
+// RandomBytes returns pseudo-random deterministic result
+// because test fails must be reproducible
+func RandomBytes(seed, length int) []byte {
+ b := make([]byte, length)
+ reader := rand.New(rand.NewSource(int64(seed)))
+ for n := 0; n < length; {
+ read, err := reader.Read(b[n:])
+ if err != nil {
+ panic(err)
+ }
+ n += read
+ }
+ return b
+}
+
+func RandomReader(seed, length int) *bytes.Reader {
+ return bytes.NewReader(RandomBytes(seed, length))
+}
diff --git a/swarm/version/version.go b/swarm/version/version.go
index c70c30943d..17ef34f5f4 100644
--- a/swarm/version/version.go
+++ b/swarm/version/version.go
@@ -23,7 +23,7 @@ import (
const (
VersionMajor = 0 // Major version component of the current release
VersionMinor = 3 // Minor version component of the current release
- VersionPatch = 6 // Patch version component of the current release
+ VersionPatch = 7 // Patch version component of the current release
VersionMeta = "unstable" // Version metadata to append to the version string
)
diff --git a/tests/block_test_util.go b/tests/block_test_util.go
index 12cba3244f..9fa69bf4ed 100644
--- a/tests/block_test_util.go
+++ b/tests/block_test_util.go
@@ -118,7 +118,7 @@ func (t *BlockTest) Run() error {
} else {
engine = ethash.NewShared()
}
- chain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
+ chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieCleanLimit: 0}, config, engine, vm.Config{}, nil)
if err != nil {
return err
}
diff --git a/trie/database.go b/trie/database.go
index d0691b637e..739a98addb 100644
--- a/trie/database.go
+++ b/trie/database.go
@@ -22,6 +22,7 @@ import (
"sync"
"time"
+ "github.com/allegro/bigcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
@@ -30,6 +31,11 @@ import (
)
var (
+ memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil)
+ memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil)
+ memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil)
+ memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil)
+
memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil)
memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil)
memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil)
@@ -64,9 +70,10 @@ type DatabaseReader interface {
type Database struct {
diskdb ethdb.Database // Persistent storage for matured trie nodes
- nodes map[common.Hash]*cachedNode // Data and references relationships of a node
- oldest common.Hash // Oldest tracked node, flush-list head
- newest common.Hash // Newest tracked node, flush-list tail
+ cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs
+ dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes
+ oldest common.Hash // Oldest tracked node, flush-list head
+ newest common.Hash // Newest tracked node, flush-list tail
preimages map[common.Hash][]byte // Preimages of nodes from the secure trie
seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys
@@ -79,7 +86,7 @@ type Database struct {
flushnodes uint64 // Nodes flushed since last commit
flushsize common.StorageSize // Data storage flushed since last commit
- nodesSize common.StorageSize // Storage size of the nodes cache (exc. flushlist)
+ dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. flushlist)
preimagesSize common.StorageSize // Storage size of the preimages cache
lock sync.RWMutex
@@ -134,7 +141,7 @@ type cachedNode struct {
node node // Cached collapsed trie node, or raw rlp data
size uint16 // Byte size of the useful cached data
- parents uint16 // Number of live nodes referencing this one
+ parents uint32 // Number of live nodes referencing this one
children map[common.Hash]uint16 // External children referenced by this node
flushPrev common.Hash // Previous node in the flush-list
@@ -262,11 +269,30 @@ func expandNode(hash hashNode, n node, cachegen uint16) node {
}
// NewDatabase creates a new trie database to store ephemeral trie content before
-// its written out to disk or garbage collected.
+// its written out to disk or garbage collected. No read cache is created, so all
+// data retrievals will hit the underlying disk database.
func NewDatabase(diskdb ethdb.Database) *Database {
+ return NewDatabaseWithCache(diskdb, 0)
+}
+
+// NewDatabaseWithCache creates a new trie database to store ephemeral trie content
+// before its written out to disk or garbage collected. It also acts as a read cache
+// for nodes loaded from disk.
+func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database {
+ var cleans *bigcache.BigCache
+ if cache > 0 {
+ cleans, _ = bigcache.NewBigCache(bigcache.Config{
+ Shards: 1024,
+ LifeWindow: time.Hour,
+ MaxEntriesInWindow: cache * 1024,
+ MaxEntrySize: 512,
+ HardMaxCacheSize: cache,
+ })
+ }
return &Database{
diskdb: diskdb,
- nodes: map[common.Hash]*cachedNode{{}: {}},
+ cleans: cleans,
+ dirties: map[common.Hash]*cachedNode{{}: {}},
preimages: make(map[common.Hash][]byte),
}
}
@@ -293,7 +319,7 @@ func (db *Database) InsertBlob(hash common.Hash, blob []byte) {
// size tracking.
func (db *Database) insert(hash common.Hash, blob []byte, node node) {
// If the node's already cached, skip
- if _, ok := db.nodes[hash]; ok {
+ if _, ok := db.dirties[hash]; ok {
return
}
// Create the cached entry for this node
@@ -303,19 +329,19 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) {
flushPrev: db.newest,
}
for _, child := range entry.childs() {
- if c := db.nodes[child]; c != nil {
+ if c := db.dirties[child]; c != nil {
c.parents++
}
}
- db.nodes[hash] = entry
+ db.dirties[hash] = entry
// Update the flush-list endpoints
if db.oldest == (common.Hash{}) {
db.oldest, db.newest = hash, hash
} else {
- db.nodes[db.newest].flushNext, db.newest = hash, hash
+ db.dirties[db.newest].flushNext, db.newest = hash, hash
}
- db.nodesSize += common.StorageSize(common.HashLength + entry.size)
+ db.dirtiesSize += common.StorageSize(common.HashLength + entry.size)
}
// insertPreimage writes a new trie node pre-image to the memory database if it's
@@ -333,35 +359,64 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
// node retrieves a cached trie node from memory, or returns nil if none can be
// found in the memory cache.
func (db *Database) node(hash common.Hash, cachegen uint16) node {
- // Retrieve the node from cache if available
+ // Retrieve the node from the clean cache if available
+ if db.cleans != nil {
+ if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil {
+ memcacheCleanHitMeter.Mark(1)
+ memcacheCleanReadMeter.Mark(int64(len(enc)))
+ return mustDecodeNode(hash[:], enc, cachegen)
+ }
+ }
+ // Retrieve the node from the dirty cache if available
db.lock.RLock()
- node := db.nodes[hash]
+ dirty := db.dirties[hash]
db.lock.RUnlock()
- if node != nil {
- return node.obj(hash, cachegen)
+ if dirty != nil {
+ return dirty.obj(hash, cachegen)
}
// Content unavailable in memory, attempt to retrieve from disk
enc, err := db.diskdb.Get(hash[:])
if err != nil || enc == nil {
return nil
}
+ if db.cleans != nil {
+ db.cleans.Set(string(hash[:]), enc)
+ memcacheCleanMissMeter.Mark(1)
+ memcacheCleanWriteMeter.Mark(int64(len(enc)))
+ }
return mustDecodeNode(hash[:], enc, cachegen)
}
// Node retrieves an encoded cached trie node from memory. If it cannot be found
// cached, the method queries the persistent database for the content.
func (db *Database) Node(hash common.Hash) ([]byte, error) {
- // Retrieve the node from cache if available
+ // Retrieve the node from the clean cache if available
+ if db.cleans != nil {
+ if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil {
+ memcacheCleanHitMeter.Mark(1)
+ memcacheCleanReadMeter.Mark(int64(len(enc)))
+ return enc, nil
+ }
+ }
+ // Retrieve the node from the dirty cache if available
db.lock.RLock()
- node := db.nodes[hash]
+ dirty := db.dirties[hash]
db.lock.RUnlock()
- if node != nil {
- return node.rlp(), nil
+ if dirty != nil {
+ return dirty.rlp(), nil
}
// Content unavailable in memory, attempt to retrieve from disk
- return db.diskdb.Get(hash[:])
+ enc, err := db.diskdb.Get(hash[:])
+ if err == nil && enc != nil {
+ if db.cleans != nil {
+ db.cleans.Set(string(hash[:]), enc)
+ memcacheCleanMissMeter.Mark(1)
+ memcacheCleanWriteMeter.Mark(int64(len(enc)))
+ }
+ }
+ return enc, err
}
// preimage retrieves a cached trie node pre-image from memory. If it cannot be
@@ -395,8 +450,8 @@ func (db *Database) Nodes() []common.Hash {
db.lock.RLock()
defer db.lock.RUnlock()
- var hashes = make([]common.Hash, 0, len(db.nodes))
- for hash := range db.nodes {
+ var hashes = make([]common.Hash, 0, len(db.dirties))
+ for hash := range db.dirties {
if hash != (common.Hash{}) { // Special case for "root" references/nodes
hashes = append(hashes, hash)
}
@@ -415,18 +470,18 @@ func (db *Database) Reference(child common.Hash, parent common.Hash) {
// reference is the private locked version of Reference.
func (db *Database) reference(child common.Hash, parent common.Hash) {
// If the node does not exist, it's a node pulled from disk, skip
- node, ok := db.nodes[child]
+ node, ok := db.dirties[child]
if !ok {
return
}
// If the reference already exists, only duplicate for roots
- if db.nodes[parent].children == nil {
- db.nodes[parent].children = make(map[common.Hash]uint16)
- } else if _, ok = db.nodes[parent].children[child]; ok && parent != (common.Hash{}) {
+ if db.dirties[parent].children == nil {
+ db.dirties[parent].children = make(map[common.Hash]uint16)
+ } else if _, ok = db.dirties[parent].children[child]; ok && parent != (common.Hash{}) {
return
}
node.parents++
- db.nodes[parent].children[child]++
+ db.dirties[parent].children[child]++
}
// Dereference removes an existing reference from a root node.
@@ -439,25 +494,25 @@ func (db *Database) Dereference(root common.Hash) {
db.lock.Lock()
defer db.lock.Unlock()
- nodes, storage, start := len(db.nodes), db.nodesSize, time.Now()
+ nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now()
db.dereference(root, common.Hash{})
- db.gcnodes += uint64(nodes - len(db.nodes))
- db.gcsize += storage - db.nodesSize
+ db.gcnodes += uint64(nodes - len(db.dirties))
+ db.gcsize += storage - db.dirtiesSize
db.gctime += time.Since(start)
memcacheGCTimeTimer.Update(time.Since(start))
- memcacheGCSizeMeter.Mark(int64(storage - db.nodesSize))
- memcacheGCNodesMeter.Mark(int64(nodes - len(db.nodes)))
+ memcacheGCSizeMeter.Mark(int64(storage - db.dirtiesSize))
+ memcacheGCNodesMeter.Mark(int64(nodes - len(db.dirties)))
- log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start),
- "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize)
+ log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start),
+ "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize)
}
// dereference is the private locked version of Dereference.
func (db *Database) dereference(child common.Hash, parent common.Hash) {
// Dereference the parent-child
- node := db.nodes[parent]
+ node := db.dirties[parent]
if node.children != nil && node.children[child] > 0 {
node.children[child]--
@@ -466,7 +521,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) {
}
}
// If the child does not exist, it's a previously committed node.
- node, ok := db.nodes[child]
+ node, ok := db.dirties[child]
if !ok {
return
}
@@ -483,20 +538,20 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) {
switch child {
case db.oldest:
db.oldest = node.flushNext
- db.nodes[node.flushNext].flushPrev = common.Hash{}
+ db.dirties[node.flushNext].flushPrev = common.Hash{}
case db.newest:
db.newest = node.flushPrev
- db.nodes[node.flushPrev].flushNext = common.Hash{}
+ db.dirties[node.flushPrev].flushNext = common.Hash{}
default:
- db.nodes[node.flushPrev].flushNext = node.flushNext
- db.nodes[node.flushNext].flushPrev = node.flushPrev
+ db.dirties[node.flushPrev].flushNext = node.flushNext
+ db.dirties[node.flushNext].flushPrev = node.flushPrev
}
// Dereference all children and delete the node
for _, hash := range node.childs() {
db.dereference(hash, child)
}
- delete(db.nodes, child)
- db.nodesSize -= common.StorageSize(common.HashLength + int(node.size))
+ delete(db.dirties, child)
+ db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size))
}
}
@@ -509,13 +564,13 @@ func (db *Database) Cap(limit common.StorageSize) error {
// by only uncaching existing data when the database write finalizes.
db.lock.RLock()
- nodes, storage, start := len(db.nodes), db.nodesSize, time.Now()
+ nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now()
batch := db.diskdb.NewBatch()
- // db.nodesSize only contains the useful data in the cache, but when reporting
+ // db.dirtiesSize only contains the useful data in the cache, but when reporting
// the total memory consumption, the maintenance metadata is also needed to be
// counted. For every useful node, we track 2 extra hashes as the flushlist.
- size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength)
+ size := db.dirtiesSize + common.StorageSize((len(db.dirties)-1)*2*common.HashLength)
// If the preimage cache got large enough, push to disk. If it's still small
// leave for later to deduplicate writes.
@@ -540,7 +595,7 @@ func (db *Database) Cap(limit common.StorageSize) error {
oldest := db.oldest
for size > limit && oldest != (common.Hash{}) {
// Fetch the oldest referenced node and push into the batch
- node := db.nodes[oldest]
+ node := db.dirties[oldest]
if err := batch.Put(oldest[:], node.rlp()); err != nil {
db.lock.RUnlock()
return err
@@ -578,25 +633,25 @@ func (db *Database) Cap(limit common.StorageSize) error {
db.preimagesSize = 0
}
for db.oldest != oldest {
- node := db.nodes[db.oldest]
- delete(db.nodes, db.oldest)
+ node := db.dirties[db.oldest]
+ delete(db.dirties, db.oldest)
db.oldest = node.flushNext
- db.nodesSize -= common.StorageSize(common.HashLength + int(node.size))
+ db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size))
}
if db.oldest != (common.Hash{}) {
- db.nodes[db.oldest].flushPrev = common.Hash{}
+ db.dirties[db.oldest].flushPrev = common.Hash{}
}
- db.flushnodes += uint64(nodes - len(db.nodes))
- db.flushsize += storage - db.nodesSize
+ db.flushnodes += uint64(nodes - len(db.dirties))
+ db.flushsize += storage - db.dirtiesSize
db.flushtime += time.Since(start)
memcacheFlushTimeTimer.Update(time.Since(start))
- memcacheFlushSizeMeter.Mark(int64(storage - db.nodesSize))
- memcacheFlushNodesMeter.Mark(int64(nodes - len(db.nodes)))
+ memcacheFlushSizeMeter.Mark(int64(storage - db.dirtiesSize))
+ memcacheFlushNodesMeter.Mark(int64(nodes - len(db.dirties)))
- log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start),
- "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.nodes), "livesize", db.nodesSize)
+ log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start),
+ "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize)
return nil
}
@@ -630,7 +685,7 @@ func (db *Database) Commit(node common.Hash, report bool) error {
}
}
// Move the trie itself into the batch, flushing if enough data is accumulated
- nodes, storage := len(db.nodes), db.nodesSize
+ nodes, storage := len(db.dirties), db.dirtiesSize
if err := db.commit(node, batch); err != nil {
log.Error("Failed to commit trie from trie database", "err", err)
db.lock.RUnlock()
@@ -654,15 +709,15 @@ func (db *Database) Commit(node common.Hash, report bool) error {
db.uncache(node)
memcacheCommitTimeTimer.Update(time.Since(start))
- memcacheCommitSizeMeter.Mark(int64(storage - db.nodesSize))
- memcacheCommitNodesMeter.Mark(int64(nodes - len(db.nodes)))
+ memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize))
+ memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties)))
logger := log.Info
if !report {
logger = log.Debug
}
- logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes)+int(db.flushnodes), "size", storage-db.nodesSize+db.flushsize, "time", time.Since(start)+db.flushtime,
- "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize)
+ logger("Persisted trie from memory database", "nodes", nodes-len(db.dirties)+int(db.flushnodes), "size", storage-db.dirtiesSize+db.flushsize, "time", time.Since(start)+db.flushtime,
+ "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize)
// Reset the garbage collection statistics
db.gcnodes, db.gcsize, db.gctime = 0, 0, 0
@@ -674,7 +729,7 @@ func (db *Database) Commit(node common.Hash, report bool) error {
// commit is the private locked version of Commit.
func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error {
// If the node does not exist, it's a previously committed node
- node, ok := db.nodes[hash]
+ node, ok := db.dirties[hash]
if !ok {
return nil
}
@@ -702,7 +757,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error {
// to disk.
func (db *Database) uncache(hash common.Hash) {
// If the node does not exist, we're done on this path
- node, ok := db.nodes[hash]
+ node, ok := db.dirties[hash]
if !ok {
return
}
@@ -710,20 +765,20 @@ func (db *Database) uncache(hash common.Hash) {
switch hash {
case db.oldest:
db.oldest = node.flushNext
- db.nodes[node.flushNext].flushPrev = common.Hash{}
+ db.dirties[node.flushNext].flushPrev = common.Hash{}
case db.newest:
db.newest = node.flushPrev
- db.nodes[node.flushPrev].flushNext = common.Hash{}
+ db.dirties[node.flushPrev].flushNext = common.Hash{}
default:
- db.nodes[node.flushPrev].flushNext = node.flushNext
- db.nodes[node.flushNext].flushPrev = node.flushPrev
+ db.dirties[node.flushPrev].flushNext = node.flushNext
+ db.dirties[node.flushNext].flushPrev = node.flushPrev
}
// Uncache the node's subtries and remove the node itself too
for _, child := range node.childs() {
db.uncache(child)
}
- delete(db.nodes, hash)
- db.nodesSize -= common.StorageSize(common.HashLength + int(node.size))
+ delete(db.dirties, hash)
+ db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size))
}
// Size returns the current storage size of the memory cache in front of the
@@ -732,11 +787,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) {
db.lock.RLock()
defer db.lock.RUnlock()
- // db.nodesSize only contains the useful data in the cache, but when reporting
+ // db.dirtiesSize only contains the useful data in the cache, but when reporting
// the total memory consumption, the maintenance metadata is also needed to be
// counted. For every useful node, we track 2 extra hashes as the flushlist.
- var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength)
- return db.nodesSize + flushlistSize, db.preimagesSize
+ var flushlistSize = common.StorageSize((len(db.dirties) - 1) * 2 * common.HashLength)
+ return db.dirtiesSize + flushlistSize, db.preimagesSize
}
// verifyIntegrity is a debug method to iterate over the entire trie stored in
@@ -749,12 +804,12 @@ func (db *Database) verifyIntegrity() {
// Iterate over all the cached nodes and accumulate them into a set
reachable := map[common.Hash]struct{}{{}: {}}
- for child := range db.nodes[common.Hash{}].children {
+ for child := range db.dirties[common.Hash{}].children {
db.accumulate(child, reachable)
}
// Find any unreachable but cached nodes
unreachable := []string{}
- for hash, node := range db.nodes {
+ for hash, node := range db.dirties {
if _, ok := reachable[hash]; !ok {
unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}",
hash, node.node, node.parents, node.flushPrev, node.flushNext))
@@ -769,7 +824,7 @@ func (db *Database) verifyIntegrity() {
// cached children found in memory.
func (db *Database) accumulate(hash common.Hash, reachable map[common.Hash]struct{}) {
// Mark the node reachable if present in the memory cache
- node, ok := db.nodes[hash]
+ node, ok := db.dirties[hash]
if !ok {
return
}
diff --git a/trie/iterator.go b/trie/iterator.go
index 00b890eb89..77f1681665 100644
--- a/trie/iterator.go
+++ b/trie/iterator.go
@@ -181,6 +181,8 @@ func (it *nodeIterator) LeafProof() [][]byte {
if len(it.stack) > 0 {
if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
hasher := newHasher(0, 0, nil)
+ defer returnHasherToPool(hasher)
+
proofs := make([][]byte, 0, len(it.stack))
for i, item := range it.stack[:len(it.stack)-1] {
diff --git a/trie/iterator_test.go b/trie/iterator_test.go
index 2a510b1c2d..4f633b195f 100644
--- a/trie/iterator_test.go
+++ b/trie/iterator_test.go
@@ -113,7 +113,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
t.Errorf("failed to retrieve reported node %x: %v", hash, err)
}
}
- for hash, obj := range db.nodes {
+ for hash, obj := range db.dirties {
if obj != nil && hash != (common.Hash{}) {
if _, ok := hashes[hash]; !ok {
t.Errorf("state entry not reported %x", hash)
@@ -333,8 +333,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
}
}
if memonly {
- robj = triedb.nodes[rkey]
- delete(triedb.nodes, rkey)
+ robj = triedb.dirties[rkey]
+ delete(triedb.dirties, rkey)
} else {
rval, _ = diskdb.Get(rkey[:])
diskdb.Delete(rkey[:])
@@ -350,7 +350,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
// Add the node back and continue iteration.
if memonly {
- triedb.nodes[rkey] = robj
+ triedb.dirties[rkey] = robj
} else {
diskdb.Put(rkey[:], rval)
}
@@ -393,8 +393,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
barNodeObj *cachedNode
)
if memonly {
- barNodeObj = triedb.nodes[barNodeHash]
- delete(triedb.nodes, barNodeHash)
+ barNodeObj = triedb.dirties[barNodeHash]
+ delete(triedb.dirties, barNodeHash)
} else {
barNodeBlob, _ = diskdb.Get(barNodeHash[:])
diskdb.Delete(barNodeHash[:])
@@ -411,7 +411,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
}
// Reinsert the missing node.
if memonly {
- triedb.nodes[barNodeHash] = barNodeObj
+ triedb.dirties[barNodeHash] = barNodeObj
} else {
diskdb.Put(barNodeHash[:], barNodeBlob)
}
diff --git a/trie/proof.go b/trie/proof.go
index 6cb8f4d5f7..f90ecd7d88 100644
--- a/trie/proof.go
+++ b/trie/proof.go
@@ -66,6 +66,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
}
}
hasher := newHasher(0, 0, nil)
+ defer returnHasherToPool(hasher)
+
for i, n := range nodes {
// Don't bother checking for errors here since hasher panics
// if encoding doesn't work and we're not writing to any database.
diff --git a/trie/trie_test.go b/trie/trie_test.go
index f8e5fd12a1..f9d6029c99 100644
--- a/trie/trie_test.go
+++ b/trie/trie_test.go
@@ -119,7 +119,7 @@ func testMissingNode(t *testing.T, memonly bool) {
hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
if memonly {
- delete(triedb.nodes, hash)
+ delete(triedb.dirties, hash)
} else {
diskdb.Delete(hash[:])
}
@@ -342,15 +342,16 @@ func TestCacheUnload(t *testing.T) {
// Commit the trie repeatedly and access key1.
// The branch containing it is loaded from DB exactly two times:
// in the 0th and 6th iteration.
- db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)}
- trie, _ = New(root, NewDatabase(db))
+ diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)}
+ triedb := NewDatabase(diskdb)
+ trie, _ = New(root, triedb)
trie.SetCacheLimit(5)
for i := 0; i < 12; i++ {
getString(trie, key1)
trie.Commit(nil)
}
// Check that it got loaded two times.
- for dbkey, count := range db.gets {
+ for dbkey, count := range diskdb.gets {
if count != 2 {
t.Errorf("db key %x loaded %d times, want %d times", []byte(dbkey), count, 2)
}
diff --git a/vendor/github.com/allegro/bigcache/LICENSE b/vendor/github.com/allegro/bigcache/LICENSE
new file mode 100644
index 0000000000..8dada3edaf
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/vendor/github.com/allegro/bigcache/README.md b/vendor/github.com/allegro/bigcache/README.md
new file mode 100644
index 0000000000..c23f7f36ce
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/README.md
@@ -0,0 +1,150 @@
+# BigCache [](https://travis-ci.org/allegro/bigcache) [](https://coveralls.io/github/allegro/bigcache?branch=master) [](https://godoc.org/github.com/allegro/bigcache) [](https://goreportcard.com/report/github.com/allegro/bigcache)
+
+Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance.
+BigCache keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place,
+therefore entries (de)serialization in front of the cache will be needed in most use cases.
+
+## Usage
+
+### Simple initialization
+
+```go
+import "github.com/allegro/bigcache"
+
+cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute))
+
+cache.Set("my-unique-key", []byte("value"))
+
+entry, _ := cache.Get("my-unique-key")
+fmt.Println(string(entry))
+```
+
+### Custom initialization
+
+When cache load can be predicted in advance then it is better to use custom initialization because additional memory
+allocation can be avoided in that way.
+
+```go
+import (
+ "log"
+
+ "github.com/allegro/bigcache"
+)
+
+config := bigcache.Config {
+ // number of shards (must be a power of 2)
+ Shards: 1024,
+ // time after which entry can be evicted
+ LifeWindow: 10 * time.Minute,
+ // rps * lifeWindow, used only in initial memory allocation
+ MaxEntriesInWindow: 1000 * 10 * 60,
+ // max entry size in bytes, used only in initial memory allocation
+ MaxEntrySize: 500,
+ // prints information about additional memory allocation
+ Verbose: true,
+ // cache will not allocate more memory than this limit, value in MB
+ // if value is reached then the oldest entries can be overridden for the new ones
+ // 0 value means no size limit
+ HardMaxCacheSize: 8192,
+ // callback fired when the oldest entry is removed because of its expiration time or no space left
+ // for the new entry, or because delete was called. A bitmask representing the reason will be returned.
+ // Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+ OnRemove: nil,
+ // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left
+ // for the new entry, or because delete was called. A constant representing the reason will be passed through.
+ // Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+ // Ignored if OnRemove is specified.
+ OnRemoveWithReason: nil,
+ }
+
+cache, initErr := bigcache.NewBigCache(config)
+if initErr != nil {
+ log.Fatal(initErr)
+}
+
+cache.Set("my-unique-key", []byte("value"))
+
+if entry, err := cache.Get("my-unique-key"); err == nil {
+ fmt.Println(string(entry))
+}
+```
+
+## Benchmarks
+
+Three caches were compared: bigcache, [freecache](https://github.com/coocood/freecache) and map.
+Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10.
+
+### Writes and reads
+
+```bash
+cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m
+
+BenchmarkMapSet-8 3000000 569 ns/op 202 B/op 3 allocs/op
+BenchmarkConcurrentMapSet-8 1000000 1592 ns/op 347 B/op 8 allocs/op
+BenchmarkFreeCacheSet-8 3000000 775 ns/op 355 B/op 2 allocs/op
+BenchmarkBigCacheSet-8 3000000 640 ns/op 303 B/op 2 allocs/op
+BenchmarkMapGet-8 5000000 407 ns/op 24 B/op 1 allocs/op
+BenchmarkConcurrentMapGet-8 3000000 558 ns/op 24 B/op 2 allocs/op
+BenchmarkFreeCacheGet-8 2000000 682 ns/op 136 B/op 2 allocs/op
+BenchmarkBigCacheGet-8 3000000 512 ns/op 152 B/op 4 allocs/op
+BenchmarkBigCacheSetParallel-8 10000000 225 ns/op 313 B/op 3 allocs/op
+BenchmarkFreeCacheSetParallel-8 10000000 218 ns/op 341 B/op 3 allocs/op
+BenchmarkConcurrentMapSetParallel-8 5000000 318 ns/op 200 B/op 6 allocs/op
+BenchmarkBigCacheGetParallel-8 20000000 178 ns/op 152 B/op 4 allocs/op
+BenchmarkFreeCacheGetParallel-8 20000000 295 ns/op 136 B/op 3 allocs/op
+BenchmarkConcurrentMapGetParallel-8 10000000 237 ns/op 24 B/op 2 allocs/op
+```
+
+Writes and reads in bigcache are faster than in freecache.
+Writes to map are the slowest.
+
+### GC pause time
+
+```bash
+cd caches_bench; go run caches_gc_overhead_comparison.go
+
+Number of entries: 20000000
+GC pause for bigcache: 5.8658ms
+GC pause for freecache: 32.4341ms
+GC pause for map: 52.9661ms
+```
+
+Test shows how long are the GC pauses for caches filled with 20mln of entries.
+Bigcache and freecache have very similar GC pause time.
+It is clear that both reduce GC overhead in contrast to map
+which GC pause time took more than 10 seconds.
+
+## How it works
+
+BigCache relies on optimization presented in 1.5 version of Go ([issue-9477](https://github.com/golang/go/issues/9477)).
+This optimization states that if map without pointers in keys and values is used then GC will omit its content.
+Therefore BigCache uses `map[uint64]uint32` where keys are hashed and values are offsets of entries.
+
+Entries are kept in bytes array, to omit GC again.
+Bytes array size can grow to gigabytes without impact on performance
+because GC will only see single pointer to it.
+
+## Bigcache vs Freecache
+
+Both caches provide the same core features but they reduce GC overhead in different ways.
+Bigcache relies on `map[uint64]uint32`, freecache implements its own mapping built on
+slices to reduce number of pointers.
+
+Results from benchmark tests are presented above.
+One of the advantage of bigcache over freecache is that you don’t need to know
+the size of the cache in advance, because when bigcache is full,
+it can allocate additional memory for new entries instead of
+overwriting existing ones as freecache does currently.
+However hard max size in bigcache also can be set, check [HardMaxCacheSize](https://godoc.org/github.com/allegro/bigcache#Config).
+
+## HTTP Server
+
+This package also includes an easily deployable HTTP implementation of BigCache, which can be found in the [server](/server) package.
+
+## More
+
+Bigcache genesis is described in allegro.tech blog post: [writing a very fast cache service in Go](http://allegro.tech/2016/03/writing-fast-cache-service-in-go.html)
+
+## License
+
+BigCache is released under the Apache 2.0 license (see [LICENSE](LICENSE))
diff --git a/vendor/github.com/allegro/bigcache/bigcache.go b/vendor/github.com/allegro/bigcache/bigcache.go
new file mode 100644
index 0000000000..e6aee8663b
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/bigcache.go
@@ -0,0 +1,202 @@
+package bigcache
+
+import (
+ "fmt"
+ "time"
+)
+
+const (
+ minimumEntriesInShard = 10 // Minimum number of entries in single shard
+)
+
+// BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance.
+// It keeps entries on heap but omits GC for them. To achieve that, operations take place on byte arrays,
+// therefore entries (de)serialization in front of the cache will be needed in most use cases.
+type BigCache struct {
+ shards []*cacheShard
+ lifeWindow uint64
+ clock clock
+ hash Hasher
+ config Config
+ shardMask uint64
+ maxShardSize uint32
+ close chan struct{}
+}
+
+// RemoveReason is a value used to signal to the user why a particular key was removed in the OnRemove callback.
+type RemoveReason uint32
+
+const (
+ // Expired means the key is past its LifeWindow.
+ Expired RemoveReason = iota
+ // NoSpace means the key is the oldest and the cache size was at its maximum when Set was called, or the
+ // entry exceeded the maximum shard size.
+ NoSpace
+ // Deleted means Delete was called and this key was removed as a result.
+ Deleted
+)
+
+// NewBigCache initialize new instance of BigCache
+func NewBigCache(config Config) (*BigCache, error) {
+ return newBigCache(config, &systemClock{})
+}
+
+func newBigCache(config Config, clock clock) (*BigCache, error) {
+
+ if !isPowerOfTwo(config.Shards) {
+ return nil, fmt.Errorf("Shards number must be power of two")
+ }
+
+ if config.Hasher == nil {
+ config.Hasher = newDefaultHasher()
+ }
+
+ cache := &BigCache{
+ shards: make([]*cacheShard, config.Shards),
+ lifeWindow: uint64(config.LifeWindow.Seconds()),
+ clock: clock,
+ hash: config.Hasher,
+ config: config,
+ shardMask: uint64(config.Shards - 1),
+ maxShardSize: uint32(config.maximumShardSize()),
+ close: make(chan struct{}),
+ }
+
+ var onRemove func(wrappedEntry []byte, reason RemoveReason)
+ if config.OnRemove != nil {
+ onRemove = cache.providedOnRemove
+ } else if config.OnRemoveWithReason != nil {
+ onRemove = cache.providedOnRemoveWithReason
+ } else {
+ onRemove = cache.notProvidedOnRemove
+ }
+
+ for i := 0; i < config.Shards; i++ {
+ cache.shards[i] = initNewShard(config, onRemove, clock)
+ }
+
+ if config.CleanWindow > 0 {
+ go func() {
+ ticker := time.NewTicker(config.CleanWindow)
+ defer ticker.Stop()
+ for {
+ select {
+ case t := <-ticker.C:
+ cache.cleanUp(uint64(t.Unix()))
+ case <-cache.close:
+ return
+ }
+ }
+ }()
+ }
+
+ return cache, nil
+}
+
+// Close is used to signal a shutdown of the cache when you are done with it.
+// This allows the cleaning goroutines to exit and ensures references are not
+// kept to the cache preventing GC of the entire cache.
+func (c *BigCache) Close() error {
+ close(c.close)
+ return nil
+}
+
+// Get reads entry for the key.
+// It returns an EntryNotFoundError when
+// no entry exists for the given key.
+func (c *BigCache) Get(key string) ([]byte, error) {
+ hashedKey := c.hash.Sum64(key)
+ shard := c.getShard(hashedKey)
+ return shard.get(key, hashedKey)
+}
+
+// Set saves entry under the key
+func (c *BigCache) Set(key string, entry []byte) error {
+ hashedKey := c.hash.Sum64(key)
+ shard := c.getShard(hashedKey)
+ return shard.set(key, hashedKey, entry)
+}
+
+// Delete removes the key
+func (c *BigCache) Delete(key string) error {
+ hashedKey := c.hash.Sum64(key)
+ shard := c.getShard(hashedKey)
+ return shard.del(key, hashedKey)
+}
+
+// Reset empties all cache shards
+func (c *BigCache) Reset() error {
+ for _, shard := range c.shards {
+ shard.reset(c.config)
+ }
+ return nil
+}
+
+// Len computes number of entries in cache
+func (c *BigCache) Len() int {
+ var len int
+ for _, shard := range c.shards {
+ len += shard.len()
+ }
+ return len
+}
+
+// Capacity returns amount of bytes store in the cache.
+func (c *BigCache) Capacity() int {
+ var len int
+ for _, shard := range c.shards {
+ len += shard.capacity()
+ }
+ return len
+}
+
+// Stats returns cache's statistics
+func (c *BigCache) Stats() Stats {
+ var s Stats
+ for _, shard := range c.shards {
+ tmp := shard.getStats()
+ s.Hits += tmp.Hits
+ s.Misses += tmp.Misses
+ s.DelHits += tmp.DelHits
+ s.DelMisses += tmp.DelMisses
+ s.Collisions += tmp.Collisions
+ }
+ return s
+}
+
+// Iterator returns iterator function to iterate over EntryInfo's from whole cache.
+func (c *BigCache) Iterator() *EntryInfoIterator {
+ return newIterator(c)
+}
+
+func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool {
+ oldestTimestamp := readTimestampFromEntry(oldestEntry)
+ if currentTimestamp-oldestTimestamp > c.lifeWindow {
+ evict(Expired)
+ return true
+ }
+ return false
+}
+
+func (c *BigCache) cleanUp(currentTimestamp uint64) {
+ for _, shard := range c.shards {
+ shard.cleanUp(currentTimestamp)
+ }
+}
+
+func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) {
+ return c.shards[hashedKey&c.shardMask]
+}
+
+func (c *BigCache) providedOnRemove(wrappedEntry []byte, reason RemoveReason) {
+ c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry))
+}
+
+func (c *BigCache) providedOnRemoveWithReason(wrappedEntry []byte, reason RemoveReason) {
+ if c.config.onRemoveFilter == 0 || (1< 0 {
+ c.config.OnRemoveWithReason(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry), reason)
+ }
+}
+
+func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte, reason RemoveReason) {
+}
diff --git a/vendor/github.com/allegro/bigcache/bytes.go b/vendor/github.com/allegro/bigcache/bytes.go
new file mode 100644
index 0000000000..3944bfe135
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/bytes.go
@@ -0,0 +1,14 @@
+// +build !appengine
+
+package bigcache
+
+import (
+ "reflect"
+ "unsafe"
+)
+
+func bytesToString(b []byte) string {
+ bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+ strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len}
+ return *(*string)(unsafe.Pointer(&strHeader))
+}
diff --git a/vendor/github.com/allegro/bigcache/bytes_appengine.go b/vendor/github.com/allegro/bigcache/bytes_appengine.go
new file mode 100644
index 0000000000..3892f3b54f
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/bytes_appengine.go
@@ -0,0 +1,7 @@
+// +build appengine
+
+package bigcache
+
+func bytesToString(b []byte) string {
+ return string(b)
+}
diff --git a/vendor/github.com/allegro/bigcache/clock.go b/vendor/github.com/allegro/bigcache/clock.go
new file mode 100644
index 0000000000..f8b535e133
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/clock.go
@@ -0,0 +1,14 @@
+package bigcache
+
+import "time"
+
+type clock interface {
+ epoch() int64
+}
+
+type systemClock struct {
+}
+
+func (c systemClock) epoch() int64 {
+ return time.Now().Unix()
+}
diff --git a/vendor/github.com/allegro/bigcache/config.go b/vendor/github.com/allegro/bigcache/config.go
new file mode 100644
index 0000000000..9654143ab1
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/config.go
@@ -0,0 +1,86 @@
+package bigcache
+
+import "time"
+
+// Config for BigCache
+type Config struct {
+ // Number of cache shards, value must be a power of two
+ Shards int
+ // Time after which entry can be evicted
+ LifeWindow time.Duration
+ // Interval between removing expired entries (clean up).
+ // If set to <= 0 then no action is performed. Setting to < 1 second is counterproductive — bigcache has a one second resolution.
+ CleanWindow time.Duration
+ // Max number of entries in life window. Used only to calculate initial size for cache shards.
+ // When proper value is set then additional memory allocation does not occur.
+ MaxEntriesInWindow int
+ // Max size of entry in bytes. Used only to calculate initial size for cache shards.
+ MaxEntrySize int
+ // Verbose mode prints information about new memory allocation
+ Verbose bool
+ // Hasher used to map between string keys and unsigned 64bit integers, by default fnv64 hashing is used.
+ Hasher Hasher
+ // HardMaxCacheSize is a limit for cache size in MB. Cache will not allocate more memory than this limit.
+ // It can protect application from consuming all available memory on machine, therefore from running OOM Killer.
+ // Default value is 0 which means unlimited size. When the limit is higher than 0 and reached then
+ // the oldest entries are overridden for the new ones.
+ HardMaxCacheSize int
+ // OnRemove is a callback fired when the oldest entry is removed because of its expiration time or no space left
+ // for the new entry, or because delete was called.
+ // Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+ OnRemove func(key string, entry []byte)
+ // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left
+ // for the new entry, or because delete was called. A constant representing the reason will be passed through.
+ // Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
+ // Ignored if OnRemove is specified.
+ OnRemoveWithReason func(key string, entry []byte, reason RemoveReason)
+
+ onRemoveFilter int
+
+ // Logger is a logging interface and used in combination with `Verbose`
+ // Defaults to `DefaultLogger()`
+ Logger Logger
+}
+
+// DefaultConfig initializes config with default values.
+// When load for BigCache can be predicted in advance then it is better to use custom config.
+func DefaultConfig(eviction time.Duration) Config {
+ return Config{
+ Shards: 1024,
+ LifeWindow: eviction,
+ CleanWindow: 0,
+ MaxEntriesInWindow: 1000 * 10 * 60,
+ MaxEntrySize: 500,
+ Verbose: true,
+ Hasher: newDefaultHasher(),
+ HardMaxCacheSize: 0,
+ Logger: DefaultLogger(),
+ }
+}
+
+// initialShardSize computes initial shard size
+func (c Config) initialShardSize() int {
+ return max(c.MaxEntriesInWindow/c.Shards, minimumEntriesInShard)
+}
+
+// maximumShardSize computes maximum shard size
+func (c Config) maximumShardSize() int {
+ maxShardSize := 0
+
+ if c.HardMaxCacheSize > 0 {
+ maxShardSize = convertMBToBytes(c.HardMaxCacheSize) / c.Shards
+ }
+
+ return maxShardSize
+}
+
+// OnRemoveFilterSet sets which remove reasons will trigger a call to OnRemoveWithReason.
+// Filtering out reasons prevents bigcache from unwrapping them, which saves cpu.
+func (c Config) OnRemoveFilterSet(reasons ...RemoveReason) Config {
+ c.onRemoveFilter = 0
+ for i := range reasons {
+ c.onRemoveFilter |= 1 << uint(reasons[i])
+ }
+
+ return c
+}
diff --git a/vendor/github.com/allegro/bigcache/encoding.go b/vendor/github.com/allegro/bigcache/encoding.go
new file mode 100644
index 0000000000..4d434e5dce
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/encoding.go
@@ -0,0 +1,62 @@
+package bigcache
+
+import (
+ "encoding/binary"
+)
+
+const (
+ timestampSizeInBytes = 8 // Number of bytes used for timestamp
+ hashSizeInBytes = 8 // Number of bytes used for hash
+ keySizeInBytes = 2 // Number of bytes used for size of entry key
+ headersSizeInBytes = timestampSizeInBytes + hashSizeInBytes + keySizeInBytes // Number of bytes used for all headers
+)
+
+func wrapEntry(timestamp uint64, hash uint64, key string, entry []byte, buffer *[]byte) []byte {
+ keyLength := len(key)
+ blobLength := len(entry) + headersSizeInBytes + keyLength
+
+ if blobLength > len(*buffer) {
+ *buffer = make([]byte, blobLength)
+ }
+ blob := *buffer
+
+ binary.LittleEndian.PutUint64(blob, timestamp)
+ binary.LittleEndian.PutUint64(blob[timestampSizeInBytes:], hash)
+ binary.LittleEndian.PutUint16(blob[timestampSizeInBytes+hashSizeInBytes:], uint16(keyLength))
+ copy(blob[headersSizeInBytes:], key)
+ copy(blob[headersSizeInBytes+keyLength:], entry)
+
+ return blob[:blobLength]
+}
+
+func readEntry(data []byte) []byte {
+ length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:])
+
+ // copy on read
+ dst := make([]byte, len(data)-int(headersSizeInBytes+length))
+ copy(dst, data[headersSizeInBytes+length:])
+
+ return dst
+}
+
+func readTimestampFromEntry(data []byte) uint64 {
+ return binary.LittleEndian.Uint64(data)
+}
+
+func readKeyFromEntry(data []byte) string {
+ length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:])
+
+ // copy on read
+ dst := make([]byte, length)
+ copy(dst, data[headersSizeInBytes:headersSizeInBytes+length])
+
+ return bytesToString(dst)
+}
+
+func readHashFromEntry(data []byte) uint64 {
+ return binary.LittleEndian.Uint64(data[timestampSizeInBytes:])
+}
+
+func resetKeyFromEntry(data []byte) {
+ binary.LittleEndian.PutUint64(data[timestampSizeInBytes:], 0)
+}
diff --git a/vendor/github.com/allegro/bigcache/entry_not_found_error.go b/vendor/github.com/allegro/bigcache/entry_not_found_error.go
new file mode 100644
index 0000000000..883bdc29eb
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/entry_not_found_error.go
@@ -0,0 +1,17 @@
+package bigcache
+
+import "fmt"
+
+// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key
+type EntryNotFoundError struct {
+ key string
+}
+
+func notFound(key string) error {
+ return &EntryNotFoundError{key}
+}
+
+// Error returned when entry does not exist.
+func (e EntryNotFoundError) Error() string {
+ return fmt.Sprintf("Entry %q not found", e.key)
+}
diff --git a/vendor/github.com/allegro/bigcache/fnv.go b/vendor/github.com/allegro/bigcache/fnv.go
new file mode 100644
index 0000000000..188c9aa6dc
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/fnv.go
@@ -0,0 +1,28 @@
+package bigcache
+
+// newDefaultHasher returns a new 64-bit FNV-1a Hasher which makes no memory allocations.
+// Its Sum64 method will lay the value out in big-endian byte order.
+// See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function
+func newDefaultHasher() Hasher {
+ return fnv64a{}
+}
+
+type fnv64a struct{}
+
+const (
+ // offset64 FNVa offset basis. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash
+ offset64 = 14695981039346656037
+ // prime64 FNVa prime value. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash
+ prime64 = 1099511628211
+)
+
+// Sum64 gets the string and returns its uint64 hash value.
+func (f fnv64a) Sum64(key string) uint64 {
+ var hash uint64 = offset64
+ for i := 0; i < len(key); i++ {
+ hash ^= uint64(key[i])
+ hash *= prime64
+ }
+
+ return hash
+}
diff --git a/vendor/github.com/allegro/bigcache/hash.go b/vendor/github.com/allegro/bigcache/hash.go
new file mode 100644
index 0000000000..5f8ade774d
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/hash.go
@@ -0,0 +1,8 @@
+package bigcache
+
+// Hasher is responsible for generating unsigned, 64 bit hash of provided string. Hasher should minimize collisions
+// (generating same hash for different strings) and while performance is also important fast functions are preferable (i.e.
+// you can use FarmHash family).
+type Hasher interface {
+ Sum64(string) uint64
+}
diff --git a/vendor/github.com/allegro/bigcache/iterator.go b/vendor/github.com/allegro/bigcache/iterator.go
new file mode 100644
index 0000000000..70b98d9004
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/iterator.go
@@ -0,0 +1,122 @@
+package bigcache
+
+import "sync"
+
+type iteratorError string
+
+func (e iteratorError) Error() string {
+ return string(e)
+}
+
+// ErrInvalidIteratorState is reported when iterator is in invalid state
+const ErrInvalidIteratorState = iteratorError("Iterator is in invalid state. Use SetNext() to move to next position")
+
+// ErrCannotRetrieveEntry is reported when entry cannot be retrieved from underlying
+const ErrCannotRetrieveEntry = iteratorError("Could not retrieve entry from cache")
+
+var emptyEntryInfo = EntryInfo{}
+
+// EntryInfo holds informations about entry in the cache
+type EntryInfo struct {
+ timestamp uint64
+ hash uint64
+ key string
+ value []byte
+}
+
+// Key returns entry's underlying key
+func (e EntryInfo) Key() string {
+ return e.key
+}
+
+// Hash returns entry's hash value
+func (e EntryInfo) Hash() uint64 {
+ return e.hash
+}
+
+// Timestamp returns entry's timestamp (time of insertion)
+func (e EntryInfo) Timestamp() uint64 {
+ return e.timestamp
+}
+
+// Value returns entry's underlying value
+func (e EntryInfo) Value() []byte {
+ return e.value
+}
+
+// EntryInfoIterator allows to iterate over entries in the cache
+type EntryInfoIterator struct {
+ mutex sync.Mutex
+ cache *BigCache
+ currentShard int
+ currentIndex int
+ elements []uint32
+ elementsCount int
+ valid bool
+}
+
+// SetNext moves to next element and returns true if it exists.
+func (it *EntryInfoIterator) SetNext() bool {
+ it.mutex.Lock()
+
+ it.valid = false
+ it.currentIndex++
+
+ if it.elementsCount > it.currentIndex {
+ it.valid = true
+ it.mutex.Unlock()
+ return true
+ }
+
+ for i := it.currentShard + 1; i < it.cache.config.Shards; i++ {
+ it.elements, it.elementsCount = it.cache.shards[i].copyKeys()
+
+ // Non empty shard - stick with it
+ if it.elementsCount > 0 {
+ it.currentIndex = 0
+ it.currentShard = i
+ it.valid = true
+ it.mutex.Unlock()
+ return true
+ }
+ }
+ it.mutex.Unlock()
+ return false
+}
+
+func newIterator(cache *BigCache) *EntryInfoIterator {
+ elements, count := cache.shards[0].copyKeys()
+
+ return &EntryInfoIterator{
+ cache: cache,
+ currentShard: 0,
+ currentIndex: -1,
+ elements: elements,
+ elementsCount: count,
+ }
+}
+
+// Value returns current value from the iterator
+func (it *EntryInfoIterator) Value() (EntryInfo, error) {
+ it.mutex.Lock()
+
+ if !it.valid {
+ it.mutex.Unlock()
+ return emptyEntryInfo, ErrInvalidIteratorState
+ }
+
+ entry, err := it.cache.shards[it.currentShard].getEntry(int(it.elements[it.currentIndex]))
+
+ if err != nil {
+ it.mutex.Unlock()
+ return emptyEntryInfo, ErrCannotRetrieveEntry
+ }
+ it.mutex.Unlock()
+
+ return EntryInfo{
+ timestamp: readTimestampFromEntry(entry),
+ hash: readHashFromEntry(entry),
+ key: readKeyFromEntry(entry),
+ value: readEntry(entry),
+ }, nil
+}
diff --git a/vendor/github.com/allegro/bigcache/logger.go b/vendor/github.com/allegro/bigcache/logger.go
new file mode 100644
index 0000000000..50e84abc80
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/logger.go
@@ -0,0 +1,30 @@
+package bigcache
+
+import (
+ "log"
+ "os"
+)
+
+// Logger is invoked when `Config.Verbose=true`
+type Logger interface {
+ Printf(format string, v ...interface{})
+}
+
+// this is a safeguard, breaking on compile time in case
+// `log.Logger` does not adhere to our `Logger` interface.
+// see https://golang.org/doc/faq#guarantee_satisfies_interface
+var _ Logger = &log.Logger{}
+
+// DefaultLogger returns a `Logger` implementation
+// backed by stdlib's log
+func DefaultLogger() *log.Logger {
+ return log.New(os.Stdout, "", log.LstdFlags)
+}
+
+func newLogger(custom Logger) Logger {
+ if custom != nil {
+ return custom
+ }
+
+ return DefaultLogger()
+}
diff --git a/vendor/github.com/allegro/bigcache/queue/bytes_queue.go b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go
new file mode 100644
index 0000000000..0285c72cd5
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go
@@ -0,0 +1,210 @@
+package queue
+
+import (
+ "encoding/binary"
+ "log"
+ "time"
+)
+
+const (
+ // Number of bytes used to keep information about entry size
+ headerEntrySize = 4
+ // Bytes before left margin are not used. Zero index means element does not exist in queue, useful while reading slice from index
+ leftMarginIndex = 1
+ // Minimum empty blob size in bytes. Empty blob fills space between tail and head in additional memory allocation.
+ // It keeps entries indexes unchanged
+ minimumEmptyBlobSize = 32 + headerEntrySize
+)
+
+// BytesQueue is a non-thread safe queue type of fifo based on bytes array.
+// For every push operation index of entry is returned. It can be used to read the entry later
+type BytesQueue struct {
+ array []byte
+ capacity int
+ maxCapacity int
+ head int
+ tail int
+ count int
+ rightMargin int
+ headerBuffer []byte
+ verbose bool
+ initialCapacity int
+}
+
+type queueError struct {
+ message string
+}
+
+// NewBytesQueue initialize new bytes queue.
+// Initial capacity is used in bytes array allocation
+// When verbose flag is set then information about memory allocation are printed
+func NewBytesQueue(initialCapacity int, maxCapacity int, verbose bool) *BytesQueue {
+ return &BytesQueue{
+ array: make([]byte, initialCapacity),
+ capacity: initialCapacity,
+ maxCapacity: maxCapacity,
+ headerBuffer: make([]byte, headerEntrySize),
+ tail: leftMarginIndex,
+ head: leftMarginIndex,
+ rightMargin: leftMarginIndex,
+ verbose: verbose,
+ initialCapacity: initialCapacity,
+ }
+}
+
+// Reset removes all entries from queue
+func (q *BytesQueue) Reset() {
+ // Just reset indexes
+ q.tail = leftMarginIndex
+ q.head = leftMarginIndex
+ q.rightMargin = leftMarginIndex
+ q.count = 0
+}
+
+// Push copies entry at the end of queue and moves tail pointer. Allocates more space if needed.
+// Returns index for pushed data or error if maximum size queue limit is reached.
+func (q *BytesQueue) Push(data []byte) (int, error) {
+ dataLen := len(data)
+
+ if q.availableSpaceAfterTail() < dataLen+headerEntrySize {
+ if q.availableSpaceBeforeHead() >= dataLen+headerEntrySize {
+ q.tail = leftMarginIndex
+ } else if q.capacity+headerEntrySize+dataLen >= q.maxCapacity && q.maxCapacity > 0 {
+ return -1, &queueError{"Full queue. Maximum size limit reached."}
+ } else {
+ q.allocateAdditionalMemory(dataLen + headerEntrySize)
+ }
+ }
+
+ index := q.tail
+
+ q.push(data, dataLen)
+
+ return index, nil
+}
+
+func (q *BytesQueue) allocateAdditionalMemory(minimum int) {
+ start := time.Now()
+ if q.capacity < minimum {
+ q.capacity += minimum
+ }
+ q.capacity = q.capacity * 2
+ if q.capacity > q.maxCapacity && q.maxCapacity > 0 {
+ q.capacity = q.maxCapacity
+ }
+
+ oldArray := q.array
+ q.array = make([]byte, q.capacity)
+
+ if leftMarginIndex != q.rightMargin {
+ copy(q.array, oldArray[:q.rightMargin])
+
+ if q.tail < q.head {
+ emptyBlobLen := q.head - q.tail - headerEntrySize
+ q.push(make([]byte, emptyBlobLen), emptyBlobLen)
+ q.head = leftMarginIndex
+ q.tail = q.rightMargin
+ }
+ }
+
+ if q.verbose {
+ log.Printf("Allocated new queue in %s; Capacity: %d \n", time.Since(start), q.capacity)
+ }
+}
+
+func (q *BytesQueue) push(data []byte, len int) {
+ binary.LittleEndian.PutUint32(q.headerBuffer, uint32(len))
+ q.copy(q.headerBuffer, headerEntrySize)
+
+ q.copy(data, len)
+
+ if q.tail > q.head {
+ q.rightMargin = q.tail
+ }
+
+ q.count++
+}
+
+func (q *BytesQueue) copy(data []byte, len int) {
+ q.tail += copy(q.array[q.tail:], data[:len])
+}
+
+// Pop reads the oldest entry from queue and moves head pointer to the next one
+func (q *BytesQueue) Pop() ([]byte, error) {
+ data, size, err := q.peek(q.head)
+ if err != nil {
+ return nil, err
+ }
+
+ q.head += headerEntrySize + size
+ q.count--
+
+ if q.head == q.rightMargin {
+ q.head = leftMarginIndex
+ if q.tail == q.rightMargin {
+ q.tail = leftMarginIndex
+ }
+ q.rightMargin = q.tail
+ }
+
+ return data, nil
+}
+
+// Peek reads the oldest entry from list without moving head pointer
+func (q *BytesQueue) Peek() ([]byte, error) {
+ data, _, err := q.peek(q.head)
+ return data, err
+}
+
+// Get reads entry from index
+func (q *BytesQueue) Get(index int) ([]byte, error) {
+ data, _, err := q.peek(index)
+ return data, err
+}
+
+// Capacity returns number of allocated bytes for queue
+func (q *BytesQueue) Capacity() int {
+ return q.capacity
+}
+
+// Len returns number of entries kept in queue
+func (q *BytesQueue) Len() int {
+ return q.count
+}
+
+// Error returns error message
+func (e *queueError) Error() string {
+ return e.message
+}
+
+func (q *BytesQueue) peek(index int) ([]byte, int, error) {
+
+ if q.count == 0 {
+ return nil, 0, &queueError{"Empty queue"}
+ }
+
+ if index <= 0 {
+ return nil, 0, &queueError{"Index must be grater than zero. Invalid index."}
+ }
+
+ if index+headerEntrySize >= len(q.array) {
+ return nil, 0, &queueError{"Index out of range"}
+ }
+
+ blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize]))
+ return q.array[index+headerEntrySize : index+headerEntrySize+blockSize], blockSize, nil
+}
+
+func (q *BytesQueue) availableSpaceAfterTail() int {
+ if q.tail >= q.head {
+ return q.capacity - q.tail
+ }
+ return q.head - q.tail - minimumEmptyBlobSize
+}
+
+func (q *BytesQueue) availableSpaceBeforeHead() int {
+ if q.tail >= q.head {
+ return q.head - leftMarginIndex - minimumEmptyBlobSize
+ }
+ return q.head - q.tail - minimumEmptyBlobSize
+}
diff --git a/vendor/github.com/allegro/bigcache/shard.go b/vendor/github.com/allegro/bigcache/shard.go
new file mode 100644
index 0000000000..56a9fb0895
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/shard.go
@@ -0,0 +1,236 @@
+package bigcache
+
+import (
+ "fmt"
+ "sync"
+ "sync/atomic"
+
+ "github.com/allegro/bigcache/queue"
+)
+
+type onRemoveCallback func(wrappedEntry []byte, reason RemoveReason)
+
+type cacheShard struct {
+ hashmap map[uint64]uint32
+ entries queue.BytesQueue
+ lock sync.RWMutex
+ entryBuffer []byte
+ onRemove onRemoveCallback
+
+ isVerbose bool
+ logger Logger
+ clock clock
+ lifeWindow uint64
+
+ stats Stats
+}
+
+func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) {
+ s.lock.RLock()
+ itemIndex := s.hashmap[hashedKey]
+
+ if itemIndex == 0 {
+ s.lock.RUnlock()
+ s.miss()
+ return nil, notFound(key)
+ }
+
+ wrappedEntry, err := s.entries.Get(int(itemIndex))
+ if err != nil {
+ s.lock.RUnlock()
+ s.miss()
+ return nil, err
+ }
+ if entryKey := readKeyFromEntry(wrappedEntry); key != entryKey {
+ if s.isVerbose {
+ s.logger.Printf("Collision detected. Both %q and %q have the same hash %x", key, entryKey, hashedKey)
+ }
+ s.lock.RUnlock()
+ s.collision()
+ return nil, notFound(key)
+ }
+ s.lock.RUnlock()
+ s.hit()
+ return readEntry(wrappedEntry), nil
+}
+
+func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error {
+ currentTimestamp := uint64(s.clock.epoch())
+
+ s.lock.Lock()
+
+ if previousIndex := s.hashmap[hashedKey]; previousIndex != 0 {
+ if previousEntry, err := s.entries.Get(int(previousIndex)); err == nil {
+ resetKeyFromEntry(previousEntry)
+ }
+ }
+
+ if oldestEntry, err := s.entries.Peek(); err == nil {
+ s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
+ }
+
+ w := wrapEntry(currentTimestamp, hashedKey, key, entry, &s.entryBuffer)
+
+ for {
+ if index, err := s.entries.Push(w); err == nil {
+ s.hashmap[hashedKey] = uint32(index)
+ s.lock.Unlock()
+ return nil
+ }
+ if s.removeOldestEntry(NoSpace) != nil {
+ s.lock.Unlock()
+ return fmt.Errorf("entry is bigger than max shard size")
+ }
+ }
+}
+
+func (s *cacheShard) del(key string, hashedKey uint64) error {
+ s.lock.RLock()
+ itemIndex := s.hashmap[hashedKey]
+
+ if itemIndex == 0 {
+ s.lock.RUnlock()
+ s.delmiss()
+ return notFound(key)
+ }
+
+ wrappedEntry, err := s.entries.Get(int(itemIndex))
+ if err != nil {
+ s.lock.RUnlock()
+ s.delmiss()
+ return err
+ }
+ s.lock.RUnlock()
+
+ s.lock.Lock()
+ {
+ delete(s.hashmap, hashedKey)
+ s.onRemove(wrappedEntry, Deleted)
+ resetKeyFromEntry(wrappedEntry)
+ }
+ s.lock.Unlock()
+
+ s.delhit()
+ return nil
+}
+
+func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool {
+ oldestTimestamp := readTimestampFromEntry(oldestEntry)
+ if currentTimestamp-oldestTimestamp > s.lifeWindow {
+ evict(Expired)
+ return true
+ }
+ return false
+}
+
+func (s *cacheShard) cleanUp(currentTimestamp uint64) {
+ s.lock.Lock()
+ for {
+ if oldestEntry, err := s.entries.Peek(); err != nil {
+ break
+ } else if evicted := s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry); !evicted {
+ break
+ }
+ }
+ s.lock.Unlock()
+}
+
+func (s *cacheShard) getOldestEntry() ([]byte, error) {
+ return s.entries.Peek()
+}
+
+func (s *cacheShard) getEntry(index int) ([]byte, error) {
+ return s.entries.Get(index)
+}
+
+func (s *cacheShard) copyKeys() (keys []uint32, next int) {
+ keys = make([]uint32, len(s.hashmap))
+
+ s.lock.RLock()
+
+ for _, index := range s.hashmap {
+ keys[next] = index
+ next++
+ }
+
+ s.lock.RUnlock()
+ return keys, next
+}
+
+func (s *cacheShard) removeOldestEntry(reason RemoveReason) error {
+ oldest, err := s.entries.Pop()
+ if err == nil {
+ hash := readHashFromEntry(oldest)
+ delete(s.hashmap, hash)
+ s.onRemove(oldest, reason)
+ return nil
+ }
+ return err
+}
+
+func (s *cacheShard) reset(config Config) {
+ s.lock.Lock()
+ s.hashmap = make(map[uint64]uint32, config.initialShardSize())
+ s.entryBuffer = make([]byte, config.MaxEntrySize+headersSizeInBytes)
+ s.entries.Reset()
+ s.lock.Unlock()
+}
+
+func (s *cacheShard) len() int {
+ s.lock.RLock()
+ res := len(s.hashmap)
+ s.lock.RUnlock()
+ return res
+}
+
+func (s *cacheShard) capacity() int {
+ s.lock.RLock()
+ res := s.entries.Capacity()
+ s.lock.RUnlock()
+ return res
+}
+
+func (s *cacheShard) getStats() Stats {
+ var stats = Stats{
+ Hits: atomic.LoadInt64(&s.stats.Hits),
+ Misses: atomic.LoadInt64(&s.stats.Misses),
+ DelHits: atomic.LoadInt64(&s.stats.DelHits),
+ DelMisses: atomic.LoadInt64(&s.stats.DelMisses),
+ Collisions: atomic.LoadInt64(&s.stats.Collisions),
+ }
+ return stats
+}
+
+func (s *cacheShard) hit() {
+ atomic.AddInt64(&s.stats.Hits, 1)
+}
+
+func (s *cacheShard) miss() {
+ atomic.AddInt64(&s.stats.Misses, 1)
+}
+
+func (s *cacheShard) delhit() {
+ atomic.AddInt64(&s.stats.DelHits, 1)
+}
+
+func (s *cacheShard) delmiss() {
+ atomic.AddInt64(&s.stats.DelMisses, 1)
+}
+
+func (s *cacheShard) collision() {
+ atomic.AddInt64(&s.stats.Collisions, 1)
+}
+
+func initNewShard(config Config, callback onRemoveCallback, clock clock) *cacheShard {
+ return &cacheShard{
+ hashmap: make(map[uint64]uint32, config.initialShardSize()),
+ entries: *queue.NewBytesQueue(config.initialShardSize()*config.MaxEntrySize, config.maximumShardSize(), config.Verbose),
+ entryBuffer: make([]byte, config.MaxEntrySize+headersSizeInBytes),
+ onRemove: callback,
+
+ isVerbose: config.Verbose,
+ logger: newLogger(config.Logger),
+ clock: clock,
+ lifeWindow: uint64(config.LifeWindow.Seconds()),
+ }
+}
diff --git a/vendor/github.com/allegro/bigcache/stats.go b/vendor/github.com/allegro/bigcache/stats.go
new file mode 100644
index 0000000000..07157132a2
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/stats.go
@@ -0,0 +1,15 @@
+package bigcache
+
+// Stats stores cache statistics
+type Stats struct {
+ // Hits is a number of successfully found keys
+ Hits int64 `json:"hits"`
+ // Misses is a number of not found keys
+ Misses int64 `json:"misses"`
+ // DelHits is a number of successfully deleted keys
+ DelHits int64 `json:"delete_hits"`
+ // DelMisses is a number of not deleted keys
+ DelMisses int64 `json:"delete_misses"`
+ // Collisions is a number of happened key-collisions
+ Collisions int64 `json:"collisions"`
+}
diff --git a/vendor/github.com/allegro/bigcache/utils.go b/vendor/github.com/allegro/bigcache/utils.go
new file mode 100644
index 0000000000..ca1df79b93
--- /dev/null
+++ b/vendor/github.com/allegro/bigcache/utils.go
@@ -0,0 +1,16 @@
+package bigcache
+
+func max(a, b int) int {
+ if a > b {
+ return a
+ }
+ return b
+}
+
+func convertMBToBytes(value int) int {
+ return value * 1024 * 1024
+}
+
+func isPowerOfTwo(number int) bool {
+ return (number & (number - 1)) == 0
+}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 06268535e9..1bfe09da72 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -38,6 +38,18 @@
"revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338",
"revisionTime": "2018-01-16T20:38:02Z"
},
+ {
+ "checksumSHA1": "9Niiu1GNhWUrXnGZrl8AU4EzbVE=",
+ "path": "github.com/allegro/bigcache",
+ "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca",
+ "revisionTime": "2018-10-22T20:06:25Z"
+ },
+ {
+ "checksumSHA1": "zqToN+R6KybEskp1D4G/lAOKXU4=",
+ "path": "github.com/allegro/bigcache/queue",
+ "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca",
+ "revisionTime": "2018-10-22T20:06:25Z"
+ },
{
"checksumSHA1": "USkefO0g1U9mr+8hagv3fpSkrxg=",
"path": "github.com/aristanetworks/goarista/monotime",
diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go
index af9418d9f8..d7af4baae3 100644
--- a/whisper/mailserver/mailserver.go
+++ b/whisper/mailserver/mailserver.go
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see .
+// Package mailserver provides a naive, example mailserver implementation
package mailserver
import (
@@ -26,9 +27,11 @@ import (
"github.com/ethereum/go-ethereum/rlp"
whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
"github.com/syndtr/goleveldb/leveldb"
+ "github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/util"
)
+// WMailServer represents the state data of the mailserver.
type WMailServer struct {
db *leveldb.DB
w *whisper.Whisper
@@ -42,6 +45,8 @@ type DBKey struct {
raw []byte
}
+// NewDbKey is a helper function that creates a levelDB
+// key from a hash and an integer.
func NewDbKey(t uint32, h common.Hash) *DBKey {
const sz = common.HashLength + 4
var k DBKey
@@ -53,6 +58,7 @@ func NewDbKey(t uint32, h common.Hash) *DBKey {
return &k
}
+// Init initializes the mail server.
func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) error {
var err error
if len(path) == 0 {
@@ -63,7 +69,7 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p
return fmt.Errorf("password is not specified")
}
- s.db, err = leveldb.OpenFile(path, nil)
+ s.db, err = leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: 32})
if err != nil {
return fmt.Errorf("open DB file: %s", err)
}
@@ -82,12 +88,14 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p
return nil
}
+// Close cleans up before shutdown.
func (s *WMailServer) Close() {
if s.db != nil {
s.db.Close()
}
}
+// Archive stores the
func (s *WMailServer) Archive(env *whisper.Envelope) {
key := NewDbKey(env.Expiry-env.TTL, env.Hash())
rawEnvelope, err := rlp.EncodeToBytes(env)
@@ -101,6 +109,8 @@ func (s *WMailServer) Archive(env *whisper.Envelope) {
}
}
+// DeliverMail responds with saved messages upon request by the
+// messages' owner.
func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) {
if peer == nil {
log.Error("Whisper peer is nil")
diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go
index 35616aaaf6..2449532070 100644
--- a/whisper/whisperv5/peer_test.go
+++ b/whisper/whisperv5/peer_test.go
@@ -139,7 +139,7 @@ func initialize(t *testing.T) {
err = node.server.Start()
if err != nil {
- t.Fatalf("failed to start server %d.", i)
+ t.Fatalf("failed to start server %d. err: %v", i, err)
}
for j := 0; j < i; j++ {
diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go
index 65e62d96cc..c5b044e1a6 100644
--- a/whisper/whisperv6/peer_test.go
+++ b/whisper/whisperv6/peer_test.go
@@ -232,7 +232,7 @@ func initialize(t *testing.T) {
func startServer(t *testing.T, s *p2p.Server) {
err := s.Start()
if err != nil {
- t.Fatalf("failed to start the fisrt server.")
+ t.Fatalf("failed to start the first server. err: %v", err)
}
atomic.AddInt64(&result.started, 1)