forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 18:32:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' of github.com:ethereum/go-ethereum into ulc/master_2

Browse source
This commit is contained in:
b00ris 2018-12-01 12:44:34 +03:00
commit 4ebea8450a
188 changed files with 7827 additions and 1781 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/CODEOWNERS vendored
View file

@ -9,9 +9,9 @@ les/ @zsfelfoldi
light/ @zsfelfoldi
mobile/ @karalabe
p2p/ @fjl @zsfelfoldi
p2p/simulations @lmars
p2p/protocols @zelig
swarm/api/http @justelad
p2p/simulations @lmars
p2p/protocols @zelig
swarm/api/http @justelad
swarm/bmt @zelig
swarm/dev @lmars
swarm/fuse @jmozah @holisticode

8
.travis.yml
View file

@ -29,6 +29,14 @@ matrix:
- os: osx
go: 1.11.x
script:
- echo "Increase the maximum number of open file descriptors on macOS"
- NOFILE=20480
- sudo sysctl -w kern.maxfiles=$NOFILE
- sudo sysctl -w kern.maxfilesperproc=$NOFILE
- sudo launchctl limit maxfiles $NOFILE $NOFILE
- sudo launchctl limit maxfiles
- ulimit -S -n $NOFILE
- ulimit -n
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES

2
README.md
View file

@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
[Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
on the wiki.
Building geth requires both a Go (version 1.7 or later) and a C compiler.
Building geth requires both a Go (version 1.9 or later) and a C compiler.
You can install them using your favourite package manager.
Once the dependencies are installed, run

5
accounts/keystore/account_cache.go
View file

@ -265,7 +265,10 @@ func (ac *accountCache) scanAccounts() error {
case (addr == common.Address{}):
log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
default:
return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
return &accounts.Account{
Address: addr,
URL: accounts.URL{Scheme: KeyStoreScheme, Path: path},
}
}
return nil
}

8
accounts/keystore/key.go
View file

@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
if err != nil {
return nil, accounts.Account{}, err
}
a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
a := accounts.Account{
Address: key.Address,
URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))},
}
if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
zeroKey(key.PrivateKey)
return nil, a, err
@ -224,5 +227,6 @@ func toISO8601(t time.Time) string {
} else {
tz = fmt.Sprintf("%03d00", offset/3600)
}
return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s",
t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
}

1
accounts/keystore/keystore_passphrase.go → accounts/keystore/passphrase.go
View file

@ -233,6 +233,7 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
PrivateKey: key,
}, nil
}
func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
if cryptoJson.Cipher != "aes-128-ctr" {
return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)

0
accounts/keystore/keystore_passphrase_test.go → accounts/keystore/passphrase_test.go
View file

0
accounts/keystore/keystore_plain.go → accounts/keystore/plain.go
View file

0
accounts/keystore/keystore_plain_test.go → accounts/keystore/plain_test.go
View file

8
accounts/keystore/presale.go
View file

@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou
return accounts.Account{}, nil, err
}
key.Id = uuid.NewRandom()
a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
a := accounts.Account{
Address: key.Address,
URL: accounts.URL{
Scheme: KeyStoreScheme,
Path: keyStore.JoinPath(keyFileName(key.Address)),
},
}
err = keyStore.StoreKey(a.URL.Path, key, password)
return a, key, err
}

0
accounts/keystore/keystore_wallet.go → accounts/keystore/wallet.go
View file

1
cmd/evm/runner.go
View file

@ -206,6 +206,7 @@ func runCmd(ctx *cli.Context) error {
execTime := time.Since(tstart)
if ctx.GlobalBool(DumpFlag.Name) {
statedb.Commit(true)
statedb.IntermediateRoot(true)
fmt.Println(string(statedb.Dump()))
}

1
cmd/geth/main.go
View file

@ -93,6 +93,7 @@ var (
utils.LightKDFFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
utils.ListenPortFlag,

1
cmd/geth/usage.go
View file

@ -132,6 +132,7 @@ var AppHelpFlagGroups = []flagGroup{
Flags: []cli.Flag{
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
},

78
cmd/swarm/access_test.go
View file

@ -14,8 +14,6 @@
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// +build !windows
package main
import (
@ -28,6 +26,7 @@ import (
gorand "math/rand"
"net/http"
"os"
"runtime"
"strings"
"testing"
"time"
@ -37,7 +36,7 @@ import (
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
@ -48,22 +47,41 @@ const (
var DefaultCurve = crypto.S256()
// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
func TestACT(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
initCluster(t)
cases := []struct {
name string
f func(t *testing.T)
}{
{"Password", testPassword},
{"PK", testPK},
{"ACTWithoutBogus", testACTWithoutBogus},
{"ACTWithBogus", testACTWithBogus},
}
for _, tc := range cases {
t.Run(tc.name, tc.f)
}
}
// testPassword tests for the correct creation of an ACT manifest protected by a password.
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
// is then fetched through 2nd node. since the tested code is not key-aware - we can just
// fetch from the 2nd node using HTTP BasicAuth
func TestAccessPassword(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
func testPassword(t *testing.T) {
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
srv.URL, //it doesn't matter through which node we upload content
cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
@ -137,16 +155,17 @@ func TestAccessPassword(t *testing.T) {
if a.Publisher != "" {
t.Fatal("should be empty")
}
client := swarm.NewClient(srv.URL)
client := swarmapi.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
url := srv.URL + "/" + "bzz:/" + hash
httpClient := &http.Client{}
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
@ -188,7 +207,7 @@ func TestAccessPassword(t *testing.T) {
//download file with 'swarm down' with wrong password
up = runSwarm(t,
"--bzzapi",
srv.URL,
cluster.Nodes[0].URL,
"down",
"bzz:/"+hash,
tmp,
@ -202,16 +221,12 @@ func TestAccessPassword(t *testing.T) {
up.ExpectExit()
}
// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
// the test will fail if the proxy's given private key is not granted on the ACT.
func TestAccessPK(t *testing.T) {
// Setup Swarm and upload a test file to it
cluster := newTestCluster(t, 2)
defer cluster.Shutdown()
func testPK(t *testing.T) {
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
@ -317,7 +332,7 @@ func TestAccessPK(t *testing.T) {
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
client := swarm.NewClient(cluster.Nodes[0].URL)
client := swarmapi.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
@ -343,29 +358,24 @@ func TestAccessPK(t *testing.T) {
}
}
// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
func TestAccessACT(t *testing.T) {
testAccessACT(t, 0)
// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
func testACTWithoutBogus(t *testing.T) {
testACT(t, 0)
}
// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
func TestAccessACTScale(t *testing.T) {
testAccessACT(t, 1000)
// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
func testACTWithBogus(t *testing.T) {
testACT(t, 100)
}
// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
// the publisher uploads through one of the nodes then disappears.
func testAccessACT(t *testing.T, bogusEntries int) {
// Setup Swarm and upload a test file to it
const clusterSize = 3
cluster := newTestCluster(t, clusterSize)
defer cluster.Shutdown()
func testACT(t *testing.T, bogusEntries int) {
var uploadThroughNode = cluster.Nodes[0]
client := swarm.NewClient(uploadThroughNode.URL)
client := swarmapi.NewClient(uploadThroughNode.URL)
r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)

24
cmd/swarm/config_test.go
View file

@ -26,14 +26,14 @@ import (
"testing"
"time"
"github.com/docker/docker/pkg/reexec"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/docker/docker/pkg/reexec"
)
func TestDumpConfig(t *testing.T) {
func TestConfigDump(t *testing.T) {
swarm := runSwarm(t, "dumpconfig")
defaultConf := api.NewConfig()
out, err := tomlSettings.Marshal(&defaultConf)
@ -91,8 +91,8 @@ func TestConfigCmdLineOverrides(t *testing.T) {
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name),
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
"--datadir", dir,
"--ipcpath", conf.IPCPath,
fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
@ -189,9 +189,9 @@ func TestConfigFileOverrides(t *testing.T) {
flags := []string{
fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
"--ens-api", "",
"--ipcpath", conf.IPCPath,
"--datadir", dir,
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
@ -407,9 +407,9 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
"--ens-api", "",
"--datadir", dir,
"--ipcpath", conf.IPCPath,
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
@ -466,7 +466,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
node.Shutdown()
}
func TestValidateConfig(t *testing.T) {
func TestConfigValidate(t *testing.T) {
for _, c := range []struct {
cfg *api.Config
err string

38
cmd/swarm/export_test.go
View file

@ -19,9 +19,7 @@ package main
import (
"bytes"
"crypto/md5"
"crypto/rand"
"io"
"io/ioutil"
"net/http"
"os"
"runtime"
@ -29,6 +27,7 @@ import (
"testing"
"github.com/ethereum/go-ethereum/swarm"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
// TestCLISwarmExportImport perform the following test:
@ -44,12 +43,13 @@ func TestCLISwarmExportImport(t *testing.T) {
}
cluster := newTestCluster(t, 1)
// generate random 10mb file
f, cleanup := generateRandomFile(t, 10000000)
defer cleanup()
// generate random 1mb file
content := testutil.RandomBytes(1, 1000000)
fileName := testutil.TempFileWithContent(t, string(content))
defer os.Remove(fileName)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
hash := matches[0]
@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
}
// compare downloaded file with the generated random file
mustEqualFiles(t, f, res.Body)
mustEqualFiles(t, bytes.NewReader(content), res.Body)
}
func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
}
}
func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
// callback for tmp file cleanup
teardown = func() {
tmp.Close()
os.Remove(tmp.Name())
}
// write 10mb random data to file
buf := make([]byte, 10000000)
_, err = rand.Read(buf)
if err != nil {
t.Fatal(err)
}
ioutil.WriteFile(tmp.Name(), buf, 0755)
return tmp, teardown
}

6
cmd/swarm/feeds.go
View file

@ -169,7 +169,6 @@ func feedUpdate(ctx *cli.Context) {
query = new(feed.Query)
query.User = signer.Address()
query.Topic = getTopic(ctx)
}
// Retrieve a feed update request
@ -178,6 +177,11 @@ func feedUpdate(ctx *cli.Context) {
utils.Fatalf("Error retrieving feed status: %s", err.Error())
}
// Check that the provided signer matches the request to sign
if updateRequest.User != signer.Address() {
utils.Fatalf("Signer address does not match the update request")
}
// set the new data
updateRequest.SetData(data)

80
cmd/swarm/feeds_test.go
View file

@ -19,50 +19,35 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestCLIFeedUpdate(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}, nil)
log.Info("starting a test swarm server")
defer srv.Close()
// create a private key file for signing
pkfile, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer pkfile.Close()
defer os.Remove(pkfile.Name())
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
privKey, _ := crypto.HexToECDSA(privkeyHex)
address := crypto.PubkeyToAddress(privKey.PublicKey)
// save the private key to a file
_, err = io.WriteString(pkfile, privkeyHex)
if err != nil {
t.Fatal(err)
}
pkFileName := testutil.TempFileWithContent(t, privkeyHex)
defer os.Remove(pkFileName)
// compose a topic. We'll be doing quotes about Miguel de Cervantes
var topic feed.Topic
@ -76,26 +61,23 @@ func TestCLIFeedUpdate(t *testing.T) {
flags := []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkfile.Name(),
"--bzzaccount", pkFileName,
"feed", "update",
"--topic", topic.Hex(),
"--name", name,
hexData}
// create an update and expect an exit without errors
log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'"))
log.Info("updating a feed with 'swarm feed update'")
cmd := runSwarm(t, flags...)
cmd.ExpectExit()
// now try to get the update using the client
client := swarm.NewClient(srv.URL)
if err != nil {
t.Fatal(err)
}
// build the same topic as before, this time
// we use NewTopic to create a topic automatically.
topic, err = feed.NewTopic(name, subject)
topic, err := feed.NewTopic(name, subject)
if err != nil {
t.Fatal(err)
}
@ -133,7 +115,7 @@ func TestCLIFeedUpdate(t *testing.T) {
"--user", address.Hex(),
}
log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'"))
log.Info("getting feed info with 'swarm feed info'")
cmd = runSwarm(t, flags...)
_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
cmd.ExpectExit()
@ -153,14 +135,14 @@ func TestCLIFeedUpdate(t *testing.T) {
// test publishing a manifest
flags = []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkfile.Name(),
"--bzzaccount", pkFileName,
"feed", "create",
"--topic", topic.Hex(),
}
log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'"))
log.Info("Publishing manifest with 'swarm feed create'")
cmd = runSwarm(t, flags...)
_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
cmd.ExpectExit()
manifestAddress := matches[0] // read the received feed manifest
@ -179,4 +161,36 @@ func TestCLIFeedUpdate(t *testing.T) {
if !bytes.Equal(data, retrieved) {
t.Fatalf("Received %s, expected %s", retrieved, data)
}
// test publishing a manifest for a different user
flags = []string{
"--bzzapi", srv.URL,
"feed", "create",
"--topic", topic.Hex(),
"--user", "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // different user
}
log.Info("Publishing manifest with 'swarm feed create' for a different user")
cmd = runSwarm(t, flags...)
_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
cmd.ExpectExit()
manifestAddress = matches[0] // read the received feed manifest
// now let's try to update that user's manifest which we don't have the private key for
flags = []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkFileName,
"feed", "update",
"--manifest", manifestAddress,
hexData}
// create an update and expect an error given there is a user mismatch
log.Info("updating a feed with 'swarm feed update'")
cmd = runSwarm(t, flags...)
cmd.ExpectRegexp("Fatal:.*") // best way so far to detect a failure.
cmd.ExpectExit()
if cmd.ExitStatus() == 0 {
t.Fatal("Expected nonzero exit code when updating a manifest with the wrong user. Got 0.")
}
}

36
cmd/swarm/fs.go
View file

@ -24,7 +24,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/fuse"
"gopkg.in/urfave/cli.v1"
@ -41,27 +41,24 @@ var fsCommand = cli.Command{
Action: mount,
CustomHelpTemplate: helpTemplate,
Name: "mount",
Flags: []cli.Flag{utils.IPCPathFlag},
Usage: "mount a swarm hash to a mount point",
ArgsUsage: "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
ArgsUsage: "swarm fs mount <manifest hash> <mount point>",
Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
{
Action: unmount,
CustomHelpTemplate: helpTemplate,
Name: "unmount",
Flags: []cli.Flag{utils.IPCPathFlag},
Usage: "unmount a swarmfs mount",
ArgsUsage: "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
ArgsUsage: "swarm fs unmount <mount point>",
Description: "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
{
Action: listMounts,
CustomHelpTemplate: helpTemplate,
Name: "list",
Flags: []cli.Flag{utils.IPCPathFlag},
Usage: "list swarmfs mounts",
ArgsUsage: "swarm fs list --ipcpath <path to bzzd.ipc>",
ArgsUsage: "swarm fs list",
Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
},
@ -70,7 +67,7 @@ var fsCommand = cli.Command{
func mount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 2 {
utils.Fatalf("Usage: swarm fs mount --ipcpath <path to bzzd.ipc> <manifestHash> <file name>")
utils.Fatalf("Usage: swarm fs mount <manifestHash> <file name>")
}
client, err := dialRPC(cliContext)
@ -97,7 +94,7 @@ func unmount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 1 {
utils.Fatalf("Usage: swarm fs unmount --ipcpath <path to bzzd.ipc> <mount path>")
utils.Fatalf("Usage: swarm fs unmount <mount path>")
}
client, err := dialRPC(cliContext)
if err != nil {
@ -145,20 +142,21 @@ func listMounts(cliContext *cli.Context) {
}
func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
var endpoint string
endpoint := getIPCEndpoint(ctx)
log.Info("IPC endpoint", "path", endpoint)
return rpc.Dial(endpoint)
}
if ctx.IsSet(utils.IPCPathFlag.Name) {
endpoint = ctx.String(utils.IPCPathFlag.Name)
} else {
utils.Fatalf("swarm ipc endpoint not specified")
}
func getIPCEndpoint(ctx *cli.Context) string {
cfg := defaultNodeConfig
utils.SetNodeConfig(ctx, &cfg)
if endpoint == "" {
endpoint = node.DefaultIPCEndpoint(clientIdentifier)
} else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
endpoint := cfg.IPCEndpoint()
if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
// Backwards compatibility with geth < 1.5 which required
// these prefixes.
endpoint = endpoint[4:]
}
return rpc.Dial(endpoint)
return endpoint
}

36
cmd/swarm/fs_test.go
View file

@ -20,6 +20,7 @@ package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
@ -28,20 +29,35 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log"
colorable "github.com/mattn/go-colorable"
)
func init() {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
type testFile struct {
filePath string
content string
}
// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list
// can find and correctly connect to a running Swarm node on the default
// IPCPath.
func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
handlingNode := cluster.Nodes[0]
list := runSwarm(t, []string{
"--datadir", handlingNode.Dir,
"fs",
"list",
}...)
list.WaitExit()
if list.Err != nil {
t.Fatal(list.Err)
}
}
// TestCLISwarmFs is a high-level test of swarmfs
//
// This test fails on travis for macOS as this executable exits with code 1
@ -65,9 +81,9 @@ func TestCLISwarmFs(t *testing.T) {
log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
mount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
mhash,
mountPoint,
}...)
@ -107,9 +123,9 @@ func TestCLISwarmFs(t *testing.T) {
log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"unmount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
mountPoint,
}...)
_, matches := unmount.ExpectRegexp(hashRegexp)
@ -142,9 +158,9 @@ func TestCLISwarmFs(t *testing.T) {
//remount, check files
newMount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
hash, // the latest hash
secondMountPoint,
}...)
@ -178,9 +194,9 @@ func TestCLISwarmFs(t *testing.T) {
log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmountSec := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"unmount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
secondMountPoint,
}...)

6
cmd/swarm/manifest_test.go
View file

@ -26,7 +26,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/testutil"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
)
// TestManifestChange tests manifest add, update and remove
@ -58,7 +58,7 @@ func TestManifestChangeEncrypted(t *testing.T) {
// Argument encrypt controls whether to use encryption or not.
func testManifestChange(t *testing.T, encrypt bool) {
t.Parallel()
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
@ -430,7 +430,7 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
t.Parallel()
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")

14
cmd/swarm/run_test.go
View file

@ -42,7 +42,6 @@ import (
"github.com/ethereum/go-ethereum/swarm"
"github.com/ethereum/go-ethereum/swarm/api"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@ -58,7 +57,18 @@ func init() {
})
}
func serverFunc(api *api.API) testutil.TestServer {
const clusterSize = 3
var clusteronce sync.Once
var cluster *testCluster
func initCluster(t *testing.T) {
clusteronce.Do(func() {
cluster = newTestCluster(t, clusterSize)
})
}
func serverFunc(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}
func TestMain(m *testing.M) {

6
cmd/swarm/swarm-smoke/feed_upload_and_sync.go
View file

@ -16,7 +16,6 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
colorable "github.com/mattn/go-colorable"
"github.com/pborman/uuid"
@ -36,7 +35,7 @@ func cliFeedUploadAndSync(c *cli.Context) error {
generateEndpoints(scheme, cluster, from, to)
log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing")
log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing")
// create a random private key to sign updates with and derive the address
pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
@ -218,8 +217,7 @@ func cliFeedUploadAndSync(c *cli.Context) error {
if err != nil {
return err
}
multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes))
multihashHex := hexutil.Encode(hashBytes)
fileHash, err := digest(f)
if err != nil {
return err

4
cmd/swarm/swarm-smoke/main.go
View file

@ -45,8 +45,8 @@ func main() {
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "cluster-endpoint",
Value: "testing",
Usage: "cluster to point to (local, open or testing)",
Value: "prod",
Usage: "cluster to point to (prod or a given namespace)",
Destination: &cluster,
},
cli.IntFlag{

23
cmd/swarm/swarm-smoke/upload_and_sync.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"crypto/md5"
crand "crypto/rand"
"crypto/tls"
"errors"
"fmt"
"io"
@ -32,6 +33,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
colorable "github.com/mattn/go-colorable"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
@ -39,18 +41,13 @@ import (
func generateEndpoints(scheme string, cluster string, from int, to int) {
if cluster == "prod" {
cluster = ""
} else if cluster == "local" {
for port := from; port <= to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
}
return
} else {
cluster = cluster + "."
}
for port := from; port <= to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
for port := from; port <= to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster))
}
}
if includeLocalhost {
@ -59,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
}
func cliUploadAndSync(c *cli.Context) error {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
generateEndpoints(scheme, cluster, from, to)
@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
time.Sleep(3 * time.Second)
log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
res, err := client.Get(endpoint + "/bzz:/" + hash + "/")
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err

120
cmd/swarm/upload_test.go
View file

@ -31,7 +31,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/mattn/go-colorable"
)
@ -41,69 +41,66 @@ func init() {
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
func TestSwarmUp(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
initCluster(t)
cases := []struct {
name string
f func(t *testing.T)
}{
{"NoEncryption", testNoEncryption},
{"Encrypted", testEncrypted},
{"RecursiveNoEncryption", testRecursiveNoEncryption},
{"RecursiveEncrypted", testRecursiveEncrypted},
{"DefaultPathAll", testDefaultPathAll},
}
for _, tc := range cases {
t.Run(tc.name, tc.f)
}
}
// testNoEncryption tests that running 'swarm up' makes the resulting file
// available from all nodes via the HTTP API
func TestCLISwarmUp(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testCLISwarmUp(false, t)
}
func TestCLISwarmUpRecursive(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testCLISwarmUpRecursive(false, t)
func testNoEncryption(t *testing.T) {
testDefault(false, t)
}
// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
// available from all nodes via the HTTP API
func TestCLISwarmUpEncrypted(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testCLISwarmUp(true, t)
}
func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testCLISwarmUpRecursive(true, t)
func testEncrypted(t *testing.T) {
testDefault(true, t)
}
func testCLISwarmUp(toEncrypt bool, t *testing.T) {
log.Info("starting 3 node cluster")
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
func testRecursiveNoEncryption(t *testing.T) {
testRecursive(false, t)
}
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer tmp.Close()
defer os.Remove(tmp.Name())
func testRecursiveEncrypted(t *testing.T) {
testRecursive(true, t)
}
func testDefault(toEncrypt bool, t *testing.T) {
tmpFileName := testutil.TempFileWithContent(t, data)
defer os.Remove(tmpFileName)
// write data to file
data := "notsorandomdata"
_, err = io.WriteString(tmp, data)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
tmp.Name()}
tmpFileName}
if toEncrypt {
hashRegexp = `[a-f\d]{128}`
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
"--encrypt",
tmp.Name()}
tmpFileName}
}
// upload the file with 'swarm up' and expect a hash
log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@ -192,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
}
}
func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
fmt.Println("starting 3 node cluster")
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
func testRecursive(toEncrypt bool, t *testing.T) {
tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpUploadDir)
// create tmp files
data := "notsorandomdata"
for _, path := range []string{"tmp1", "tmp2"} {
if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
t.Fatal(err)
@ -264,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
switch mode := fi.Mode(); {
case mode.IsRegular():
if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
} else {
ff := make([]byte, len(data))
@ -285,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
}
}
// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
// testDefaultPathAll tests swarm recursive upload with relative and absolute
// default paths and with encryption.
func TestCLISwarmUpDefaultPath(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testCLISwarmUpDefaultPath(false, false, t)
testCLISwarmUpDefaultPath(false, true, t)
testCLISwarmUpDefaultPath(true, false, t)
testCLISwarmUpDefaultPath(true, true, t)
func testDefaultPathAll(t *testing.T) {
testDefaultPath(false, false, t)
testDefaultPath(false, true, t)
testDefaultPath(true, false, t)
testDefaultPath(true, true, t)
}
func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
if err != nil {
t.Fatal(err)
@ -323,7 +309,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
args := []string{
"--bzzapi",
srv.URL,
cluster.Nodes[0].URL,
"--recursive",
"--defaultpath",
defaultPath,
@ -340,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
up.ExpectExit()
hash := matches[0]
client := swarm.NewClient(srv.URL)
client := swarmapi.NewClient(cluster.Nodes[0].URL)
m, isEncrypted, err := client.DownloadManifest(hash)
if err != nil {

48
cmd/utils/flags.go
View file

@ -313,7 +313,12 @@ var (
CacheDatabaseFlag = cli.IntFlag{
Name: "cache.database",
Usage: "Percentage of cache memory allowance to use for database io",
Value: 75,
Value: 50,
}
CacheTrieFlag = cli.IntFlag{
Name: "cache.trie",
Usage: "Percentage of cache memory allowance to use for trie caching",
Value: 25,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
@ -1025,16 +1030,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
setWS(ctx, cfg)
setNodeUserIdent(ctx, cfg)
switch {
case ctx.GlobalIsSet(DataDirFlag.Name):
cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
case ctx.GlobalBool(DeveloperFlag.Name):
cfg.DataDir = "" // unless explicitly requested, use memory databases
case ctx.GlobalBool(TestnetFlag.Name):
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
case ctx.GlobalBool(RinkebyFlag.Name):
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
}
setDataDir(ctx, cfg)
if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
@ -1047,6 +1043,19 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
}
}
func setDataDir(ctx *cli.Context, cfg *node.Config) {
switch {
case ctx.GlobalIsSet(DataDirFlag.Name):
cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
case ctx.GlobalBool(DeveloperFlag.Name):
cfg.DataDir = "" // unless explicitly requested, use memory databases
case ctx.GlobalBool(TestnetFlag.Name):
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
case ctx.GlobalBool(RinkebyFlag.Name):
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
}
}
func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
if ctx.GlobalIsSet(GpoBlocksFlag.Name) {
cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name)
@ -1212,8 +1221,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
}
cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
@ -1448,12 +1460,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cache := &core.CacheConfig{
Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
TrieNodeLimit: eth.DefaultConfig.TrieCache,
TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
TrieCleanLimit: eth.DefaultConfig.TrieCleanCache,
TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)

2
consensus/clique/clique.go
View file

@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash {
return sigHash(header)
}
// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
func (c *Clique) Close() error {
return nil
}

437
core/blockchain.go
View file

@ -47,7 +47,10 @@ import (
)
var (
blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil)
blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil)
ErrNoGenesis = errors.New("Genesis not found in chain")
)
@ -68,9 +71,10 @@ const (
// CacheConfig contains the configuration values for the trie caching/pruning
// that's resident in a blockchain.
type CacheConfig struct {
Disabled bool // Whether to disable trie write caching (archive node)
TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
Disabled bool // Whether to disable trie write caching (archive node)
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
}
// BlockChain represents the canonical chain given a database with a genesis
@ -140,8 +144,9 @@ type BlockChain struct {
func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
if cacheConfig == nil {
cacheConfig = &CacheConfig{
TrieNodeLimit: 256 * 1024 * 1024,
TrieTimeLimit: 5 * time.Minute,
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
}
}
bodyCache, _ := lru.New(bodyCacheLimit)
@ -156,7 +161,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
cacheConfig: cacheConfig,
db: db,
triegc: prque.New(nil),
stateCache: state.NewDatabase(db),
stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
quit: make(chan struct{}),
shouldPreserve: shouldPreserve,
bodyCache: bodyCache,
@ -393,6 +398,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
return state.New(root, bc.stateCache)
}
// StateCache returns the caching database underpinning the blockchain instance.
func (bc *BlockChain) StateCache() state.Database {
return bc.stateCache
}
// Reset purges the entire blockchain, restoring it to its genesis state.
func (bc *BlockChain) Reset() error {
return bc.ResetWithGenesisBlock(bc.genesisBlock)
@ -438,7 +448,11 @@ func (bc *BlockChain) repair(head **types.Block) error {
return nil
}
// Otherwise rewind one block and recheck state availability there
(*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
if block == nil {
return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
}
(*head) = block
}
}
@ -554,6 +568,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
return rawdb.HasBody(bc.db, hash, number)
}
// HasFastBlock checks if a fast block is fully present in the database or not.
func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
if !bc.HasBlock(hash, number) {
return false
}
if bc.receiptsCache.Contains(hash) {
return true
}
return rawdb.HasReceipts(bc.db, hash, number)
}
// HasState checks if state trie is fully present in the database or not.
func (bc *BlockChain) HasState(hash common.Hash) bool {
_, err := bc.stateCache.OpenTrie(hash)
@ -611,12 +636,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
if receipts, ok := bc.receiptsCache.Get(hash); ok {
return receipts.(types.Receipts)
}
number := rawdb.ReadHeaderNumber(bc.db, hash)
if number == nil {
return nil
}
receipts := rawdb.ReadReceipts(bc.db, hash, *number)
bc.receiptsCache.Add(hash, receipts)
return receipts
@ -938,7 +961,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
// If we exceeded our memory allowance, flush matured singleton nodes to disk
var (
nodes, imgs = triedb.Size()
limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
)
if nodes > limit || imgs > 4*1024*1024 {
triedb.Cap(limit - ethdb.IdealBatchSize)
@ -1020,6 +1043,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
return status, nil
}
// addFutureBlock checks if the block is within the max allowed window to get
// accepted for future processing, and returns an error if the block is too far
// ahead and was not added.
func (bc *BlockChain) addFutureBlock(block *types.Block) error {
max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
if block.Time().Cmp(max) > 0 {
return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
}
bc.futureBlocks.Add(block.Hash(), block)
return nil
}
// InsertChain attempts to insert the given batch of blocks in to the canonical
// chain or, otherwise, create a fork. If an error is returned it will return
// the index number of the failing block as well an error describing what went
@ -1027,18 +1062,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
//
// After insertion is done, all accumulated events will be fired.
func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
n, events, logs, err := bc.insertChain(chain)
bc.PostChainEvents(events, logs)
return n, err
}
// insertChain will execute the actual chain insertion and event aggregation. The
// only reason this method exists as a separate one is to make locking cleaner
// with deferred statements.
func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) {
// Sanity check that we have something meaningful to import
if len(chain) == 0 {
return 0, nil, nil, nil
return 0, nil
}
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(chain); i++ {
@ -1047,16 +1073,36 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
}
}
// Pre-checks passed, start the full block imports
bc.wg.Add(1)
defer bc.wg.Done()
bc.chainmu.Lock()
defer bc.chainmu.Unlock()
n, events, logs, err := bc.insertChain(chain, true)
bc.chainmu.Unlock()
bc.wg.Done()
bc.PostChainEvents(events, logs)
return n, err
}
// insertChain is the internal implementation of insertChain, which assumes that
// 1) chains are contiguous, and 2) The chain mutex is held.
//
// This method is split out so that import batches that require re-injecting
// historical blocks can do so without releasing the lock, which could lead to
// racey behaviour. If a sidechain import is in progress, and the historic state
// is imported, but then new canon-head is added before the actual sidechain
// completes, then the historic state could be pruned again
func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) {
// If the chain is terminating, don't even bother starting u
if atomic.LoadInt32(&bc.procInterrupt) == 1 {
return 0, nil, nil, nil
}
// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
// A queued approach to delivering events. This is generally
// faster than direct delivery and requires much less mutex
@ -1073,16 +1119,56 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
for i, block := range chain {
headers[i] = block.Header()
seals[i] = true
seals[i] = verifySeals
}
abort, results := bc.engine.VerifyHeaders(bc, headers, seals)
defer close(abort)
// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
// Peek the error for the first block to decide the directing import logic
it := newInsertIterator(chain, results, bc.Validator())
// Iterate over the blocks and insert when the verifier permits
for i, block := range chain {
block, err := it.next()
switch {
// First block is pruned, insert as sidechain and reorg only if TD grows enough
case err == consensus.ErrPrunedAncestor:
return bc.insertSidechain(it)
// First block is future, shove it (and all children) to the future queue (unknown ancestor)
case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) {
if err := bc.addFutureBlock(block); err != nil {
return it.index, events, coalescedLogs, err
}
block, err = it.next()
}
stats.queued += it.processed()
stats.ignored += it.remaining()
// If there are any still remaining, mark as ignored
return it.index, events, coalescedLogs, err
// First block (and state) is known
// 1. We did a roll-back, and should now do a re-import
// 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
// from the canonical chain, which has not been verified.
case err == ErrKnownBlock:
// Skip all known blocks that behind us
current := bc.CurrentBlock().NumberU64()
for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
stats.ignored++
block, err = it.next()
}
// Falls through to the block import
// Some other error occurred, abort
case err != nil:
stats.ignored += len(it.chain)
bc.reportBlock(block, nil, err)
return it.index, events, coalescedLogs, err
}
// No validation errors for the first block (or chain prefix skipped)
for ; block != nil && err == nil; block, err = it.next() {
// If the chain is terminating, stop processing blocks
if atomic.LoadInt32(&bc.procInterrupt) == 1 {
log.Debug("Premature abort during blocks processing")
@ -1091,115 +1177,53 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
// If the header is a banned one, straight out abort
if BadHashes[block.Hash()] {
bc.reportBlock(block, nil, ErrBlacklistedHash)
return i, events, coalescedLogs, ErrBlacklistedHash
return it.index, events, coalescedLogs, ErrBlacklistedHash
}
// Wait for the block's verification to complete
bstart := time.Now()
// Retrieve the parent block and it's state to execute on top
start := time.Now()
err := <-results
if err == nil {
err = bc.Validator().ValidateBody(block)
}
switch {
case err == ErrKnownBlock:
// Block and state both already known. However if the current block is below
// this number we did a rollback and we should reimport it nonetheless.
if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
stats.ignored++
continue
}
case err == consensus.ErrFutureBlock:
// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
// the chain is discarded and processed at a later time if given.
max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
if block.Time().Cmp(max) > 0 {
return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
}
bc.futureBlocks.Add(block.Hash(), block)
stats.queued++
continue
case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
bc.futureBlocks.Add(block.Hash(), block)
stats.queued++
continue
case err == consensus.ErrPrunedAncestor:
// Block competing with the canonical chain, store in the db, but don't process
// until the competitor TD goes above the canonical TD
currentBlock := bc.CurrentBlock()
localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
if localTd.Cmp(externTd) > 0 {
if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
return i, events, coalescedLogs, err
}
continue
}
// Competitor chain beat canonical, gather all blocks from the common ancestor
var winner []*types.Block
parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
for !bc.HasState(parent.Root()) {
winner = append(winner, parent)
parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
}
for j := 0; j < len(winner)/2; j++ {
winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
}
// Import all the pruned blocks to make the state available
bc.chainmu.Unlock()
_, evs, logs, err := bc.insertChain(winner)
bc.chainmu.Lock()
events, coalescedLogs = evs, logs
if err != nil {
return i, events, coalescedLogs, err
}
case err != nil:
bc.reportBlock(block, nil, err)
return i, events, coalescedLogs, err
}
// Create a new statedb using the parent block and report an
// error if it fails.
var parent *types.Block
if i == 0 {
parent := it.previous()
if parent == nil {
parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
} else {
parent = chain[i-1]
}
state, err := state.New(parent.Root(), bc.stateCache)
if err != nil {
return i, events, coalescedLogs, err
return it.index, events, coalescedLogs, err
}
// Process block using the parent state as reference point.
t0 := time.Now()
receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig)
t1 := time.Now()
if err != nil {
bc.reportBlock(block, receipts, err)
return i, events, coalescedLogs, err
return it.index, events, coalescedLogs, err
}
// Validate the state using the default validator
err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas)
if err != nil {
if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil {
bc.reportBlock(block, receipts, err)
return i, events, coalescedLogs, err
return it.index, events, coalescedLogs, err
}
proctime := time.Since(bstart)
t2 := time.Now()
proctime := time.Since(start)
// Write the block to the chain and get the status.
status, err := bc.WriteBlockWithState(block, receipts, state)
t3 := time.Now()
if err != nil {
return i, events, coalescedLogs, err
return it.index, events, coalescedLogs, err
}
blockInsertTimer.UpdateSince(start)
blockExecutionTimer.Update(t1.Sub(t0))
blockValidationTimer.Update(t2.Sub(t1))
blockWriteTimer.Update(t3.Sub(t2))
switch status {
case CanonStatTy:
log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()),
"txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart)))
log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(),
"uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(),
"elapsed", common.PrettyDuration(time.Since(start)),
"root", block.Root())
coalescedLogs = append(coalescedLogs, logs...)
blockInsertTimer.UpdateSince(bstart)
events = append(events, ChainEvent{block, block.Hash(), logs})
lastCanon = block
@ -1207,78 +1231,153 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
bc.gcproc += proctime
case SideStatTy:
log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
blockInsertTimer.UpdateSince(bstart)
log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(),
"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
"root", block.Root())
events = append(events, ChainSideEvent{block})
}
blockInsertTimer.UpdateSince(start)
stats.processed++
stats.usedGas += usedGas
cache, _ := bc.stateCache.TrieDB().Size()
stats.report(chain, i, cache)
stats.report(chain, it.index, cache)
}
// Any blocks remaining here? The only ones we care about are the future ones
if block != nil && err == consensus.ErrFutureBlock {
if err := bc.addFutureBlock(block); err != nil {
return it.index, events, coalescedLogs, err
}
block, err = it.next()
for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() {
if err := bc.addFutureBlock(block); err != nil {
return it.index, events, coalescedLogs, err
}
stats.queued++
}
}
stats.ignored += it.remaining()
// Append a single chain head event if we've progressed the chain
if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
events = append(events, ChainHeadEvent{lastCanon})
}
return 0, events, coalescedLogs, nil
return it.index, events, coalescedLogs, err
}
// insertStats tracks and reports on block insertion.
type insertStats struct {
queued, processed, ignored int
usedGas uint64
lastIndex int
startTime mclock.AbsTime
}
// statsReportLimit is the time limit during import and export after which we
// always print out progress. This avoids the user wondering what's going on.
const statsReportLimit = 8 * time.Second
// report prints statistics if some number of blocks have been processed
// or more than a few seconds have passed since the last message.
func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
// Fetch the timings for the batch
// insertSidechain is called when an import batch hits upon a pruned ancestor
// error, which happens when a sidechain with a sufficiently old fork-block is
// found.
//
// The method writes all (header-and-body-valid) blocks to disk, then tries to
// switch over to the new chain if the TD exceeded the current chain.
func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
var (
now = mclock.Now()
elapsed = time.Duration(now) - time.Duration(st.startTime)
externTd *big.Int
current = bc.CurrentBlock().NumberU64()
)
// If we're at the last block of the batch or report period reached, log
if index == len(chain)-1 || elapsed >= statsReportLimit {
var (
end = chain[index]
txs = countTransactions(chain[st.lastIndex : index+1])
)
context := []interface{}{
"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
"number", end.Number(), "hash", end.Hash(),
}
if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
}
context = append(context, []interface{}{"cache", cache}...)
// The first sidechain block error is already verified to be ErrPrunedAncestor.
// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
// ones. Any other errors means that the block is invalid, and should not be written
// to disk.
block, err := it.current(), consensus.ErrPrunedAncestor
for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
// Check the canonical state root for that number
if number := block.NumberU64(); current >= number {
if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
// This is most likely a shadow-state attack. When a fork is imported into the
// database, and it eventually reaches a block height which is not pruned, we
// just found that the state already exist! This means that the sidechain block
// refers to a state which already exists in our canon chain.
//
// If left unchecked, we would now proceed importing the blocks, without actually
// having verified the state of the previous blocks.
log.Warn("Sidechain ghost-state attack detected", "number", block.NumberU64(), "sideroot", block.Root(), "canonroot", canonical.Root())
if st.queued > 0 {
context = append(context, []interface{}{"queued", st.queued}...)
// If someone legitimately side-mines blocks, they would still be imported as usual. However,
// we cannot risk writing unverified blocks to disk when they obviously target the pruning
// mechanism.
return it.index, nil, nil, errors.New("sidechain ghost-state attack")
}
}
if st.ignored > 0 {
context = append(context, []interface{}{"ignored", st.ignored}...)
if externTd == nil {
externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1)
}
log.Info("Imported new chain segment", context...)
externTd = new(big.Int).Add(externTd, block.Difficulty())
*st = insertStats{startTime: now, lastIndex: index + 1}
if !bc.HasBlock(block.Hash(), block.NumberU64()) {
start := time.Now()
if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
return it.index, nil, nil, err
}
log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
"root", block.Root())
}
}
}
func countTransactions(chain []*types.Block) (c int) {
for _, b := range chain {
c += len(b.Transactions())
// At this point, we've written all sidechain blocks to database. Loop ended
// either on some other error or all were processed. If there was some other
// error, we can ignore the rest of those blocks.
//
// If the externTd was larger than our local TD, we now need to reimport the previous
// blocks to regenerate the required state
localTd := bc.GetTd(bc.CurrentBlock().Hash(), current)
if localTd.Cmp(externTd) > 0 {
log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd)
return it.index, nil, nil, err
}
return c
// Gather all the sidechain hashes (full blocks may be memory heavy)
var (
hashes []common.Hash
numbers []uint64
)
parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64())
for parent != nil && !bc.HasState(parent.Root) {
hashes = append(hashes, parent.Hash())
numbers = append(numbers, parent.Number.Uint64())
parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
}
if parent == nil {
return it.index, nil, nil, errors.New("missing parent")
}
// Import all the pruned blocks to make the state available
var (
blocks []*types.Block
memory common.StorageSize
)
for i := len(hashes) - 1; i >= 0; i-- {
// Append the next block to our batch
block := bc.GetBlock(hashes[i], numbers[i])
blocks = append(blocks, block)
memory += block.Size()
// If memory use grew too large, import and continue. Sadly we need to discard
// all raised events and logs from notifications since we're too heavy on the
// memory here.
if len(blocks) >= 2048 || memory > 64*1024*1024 {
log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64())
if _, _, _, err := bc.insertChain(blocks, false); err != nil {
return 0, nil, nil, err
}
blocks, memory = blocks[:0], 0
// If the chain is terminating, stop processing blocks
if atomic.LoadInt32(&bc.procInterrupt) == 1 {
log.Debug("Premature abort during blocks processing")
return 0, nil, nil, nil
}
}
}
if len(blocks) > 0 {
log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64())
return bc.insertChain(blocks, false)
}
return 0, nil, nil, nil
}
// reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them
@ -1453,8 +1552,10 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e
bc.addBadBlock(block)
var receiptString string
for _, receipt := range receipts {
receiptString += fmt.Sprintf("\t%v\n", receipt)
for i, receipt := range receipts {
receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n",
i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(),
receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState)
}
log.Error(fmt.Sprintf(`
########## BAD BLOCK #########

143
core/blockchain_insert.go Normal file
View file

@ -0,0 +1,143 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package core
import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
)
// insertStats tracks and reports on block insertion.
type insertStats struct {
queued, processed, ignored int
usedGas uint64
lastIndex int
startTime mclock.AbsTime
}
// statsReportLimit is the time limit during import and export after which we
// always print out progress. This avoids the user wondering what's going on.
const statsReportLimit = 8 * time.Second
// report prints statistics if some number of blocks have been processed
// or more than a few seconds have passed since the last message.
func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
// Fetch the timings for the batch
var (
now = mclock.Now()
elapsed = time.Duration(now) - time.Duration(st.startTime)
)
// If we're at the last block of the batch or report period reached, log
if index == len(chain)-1 || elapsed >= statsReportLimit {
// Count the number of transactions in this segment
var txs int
for _, block := range chain[st.lastIndex : index+1] {
txs += len(block.Transactions())
}
end := chain[index]
// Assemble the log context and send it to the logger
context := []interface{}{
"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
"number", end.Number(), "hash", end.Hash(),
}
if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
}
context = append(context, []interface{}{"cache", cache}...)
if st.queued > 0 {
context = append(context, []interface{}{"queued", st.queued}...)
}
if st.ignored > 0 {
context = append(context, []interface{}{"ignored", st.ignored}...)
}
log.Info("Imported new chain segment", context...)
// Bump the stats reported to the next section
*st = insertStats{startTime: now, lastIndex: index + 1}
}
}
// insertIterator is a helper to assist during chain import.
type insertIterator struct {
chain types.Blocks
results <-chan error
index int
validator Validator
}
// newInsertIterator creates a new iterator based on the given blocks, which are
// assumed to be a contiguous chain.
func newInsertIterator(chain types.Blocks, results <-chan error, validator Validator) *insertIterator {
return &insertIterator{
chain: chain,
results: results,
index: -1,
validator: validator,
}
}
// next returns the next block in the iterator, along with any potential validation
// error for that block. When the end is reached, it will return (nil, nil).
func (it *insertIterator) next() (*types.Block, error) {
if it.index+1 >= len(it.chain) {
it.index = len(it.chain)
return nil, nil
}
it.index++
if err := <-it.results; err != nil {
return it.chain[it.index], err
}
return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
}
// current returns the current block that's being processed.
func (it *insertIterator) current() *types.Block {
if it.index < 0 || it.index+1 >= len(it.chain) {
return nil
}
return it.chain[it.index]
}
// previous returns the previous block was being processed, or nil
func (it *insertIterator) previous() *types.Block {
if it.index < 1 {
return nil
}
return it.chain[it.index-1]
}
// first returns the first block in the it.
func (it *insertIterator) first() *types.Block {
return it.chain[0]
}
// remaining returns the number of remaining blocks.
func (it *insertIterator) remaining() int {
return len(it.chain) - it.index
}
// processed returns the number of processed blocks.
func (it *insertIterator) processed() int {
return it.index + 1
}

8
core/blockchain_test.go
View file

@ -579,11 +579,11 @@ func testInsertNonceError(t *testing.T, full bool) {
blockchain.hc.engine = blockchain.engine
failRes, err = blockchain.InsertHeaderChain(headers, 1)
}
// Check that the returned error indicates the failure.
// Check that the returned error indicates the failure
if failRes != failAt {
t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
t.Errorf("test %d: failure (%v) index mismatch: have %d, want %d", i, err, failRes, failAt)
}
// Check that all no blocks after the failing block have been inserted.
// Check that all blocks after the failing block have been inserted
for j := 0; j < i-failAt; j++ {
if full {
if block := blockchain.GetBlockByNumber(failNum + uint64(j)); block != nil {
@ -1345,7 +1345,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
t.Fatalf("failed to insert shared chain: %v", err)
}
if _, err := chain.InsertChain(original); err != nil {
t.Fatalf("failed to insert shared chain: %v", err)
t.Fatalf("failed to insert original chain: %v", err)
}
// Ensure that the state associated with the forking point is pruned away
if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {

9
core/rawdb/accessors_chain.go
View file

@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
}
}
// HasReceipts verifies the existence of all the transaction receipts belonging
// to a block.
func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
return false
}
return true
}
// ReadReceipts retrieves all the transaction receipts belonging to a block.
func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
// Retrieve the flattened receipt slice

14
core/state/database.go
View file

@ -72,13 +72,19 @@ type Trie interface {
}
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use and retains cached trie nodes in memory. The pool is an optional
// intermediate trie-node memory pool between the low level storage layer and the
// high level trie abstraction.
// concurrent use and retains a few recent expanded trie nodes in memory. To keep
// more historical state in memory, use the NewDatabaseWithCache constructor.
func NewDatabase(db ethdb.Database) Database {
return NewDatabaseWithCache(db, 0)
}
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use and retains both a few recent expanded trie nodes in memory, as
// well as a lot of collapsed RLP trie nodes in a large memory cache.
func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{
db: trie.NewDatabase(db),
db: trie.NewDatabaseWithCache(db, cache),
codeSizeCache: csc,
}
}

2
core/tx_pool.go
View file

@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
// addTxsLocked attempts to queue a batch of transactions if they are valid,
// whilst assuming the transaction pool lock is already held.
func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
// Add the batch of transaction, tracking the accepted ones
// Add the batch of transactions, tracking the accepted ones
dirty := make(map[common.Address]struct{})
errs := make([]error, len(txs))

4
core/types/block.go
View file

@ -81,8 +81,8 @@ type Header struct {
GasUsed uint64 `json:"gasUsed" gencodec:"required"`
Time *big.Int `json:"timestamp" gencodec:"required"`
Extra []byte `json:"extraData" gencodec:"required"`
MixDigest common.Hash `json:"mixHash" gencodec:"required"`
Nonce BlockNonce `json:"nonce" gencodec:"required"`
MixDigest common.Hash `json:"mixHash"`
Nonce BlockNonce `json:"nonce"`
}
// field type overrides for gencodec

20
core/types/gen_header_json.go
View file

@ -13,6 +13,7 @@ import (
var _ = (*headerMarshaling)(nil)
// MarshalJSON marshals as JSON.
func (h Header) MarshalJSON() ([]byte, error) {
type Header struct {
ParentHash common.Hash `json:"parentHash" gencodec:"required"`
@ -28,8 +29,8 @@ func (h Header) MarshalJSON() ([]byte, error) {
GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
Extra hexutil.Bytes `json:"extraData" gencodec:"required"`
MixDigest common.Hash `json:"mixHash" gencodec:"required"`
Nonce BlockNonce `json:"nonce" gencodec:"required"`
MixDigest common.Hash `json:"mixHash"`
Nonce BlockNonce `json:"nonce"`
Hash common.Hash `json:"hash"`
}
var enc Header
@ -52,6 +53,7 @@ func (h Header) MarshalJSON() ([]byte, error) {
return json.Marshal(&enc)
}
// UnmarshalJSON unmarshals from JSON.
func (h *Header) UnmarshalJSON(input []byte) error {
type Header struct {
ParentHash *common.Hash `json:"parentHash" gencodec:"required"`
@ -67,8 +69,8 @@ func (h *Header) UnmarshalJSON(input []byte) error {
GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
Extra *hexutil.Bytes `json:"extraData" gencodec:"required"`
MixDigest *common.Hash `json:"mixHash" gencodec:"required"`
Nonce *BlockNonce `json:"nonce" gencodec:"required"`
MixDigest *common.Hash `json:"mixHash"`
Nonce *BlockNonce `json:"nonce"`
}
var dec Header
if err := json.Unmarshal(input, &dec); err != nil {
@ -126,13 +128,11 @@ func (h *Header) UnmarshalJSON(input []byte) error {
return errors.New("missing required field 'extraData' for Header")
}
h.Extra = *dec.Extra
if dec.MixDigest == nil {
return errors.New("missing required field 'mixHash' for Header")
if dec.MixDigest != nil {
h.MixDigest = *dec.MixDigest
}
h.MixDigest = *dec.MixDigest
if dec.Nonce == nil {
return errors.New("missing required field 'nonce' for Header")
if dec.Nonce != nil {
h.Nonce = *dec.Nonce
}
h.Nonce = *dec.Nonce
return nil
}

6
core/vm/evm.go
View file

@ -339,6 +339,12 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
contract := NewContract(caller, to, new(big.Int), gas)
contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
// We do an AddBalance of zero here, just in order to trigger a touch.
// This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium,
// but is the correct thing to do and matters on other networks, in tests, and potential
// future scenarios
evm.StateDB.AddBalance(addr, bigZero)
// When an error was returned by the EVM or when setting the creation code
// above we revert to the snapshot and consume any gas remaining. Additionally
// when we're in Homestead this also counts for code storage gas errors.

6
eth/api.go
View file

@ -444,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
}
triedb := api.eth.BlockChain().StateCache().TrieDB()
oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0)
if err != nil {
return nil, err
}
newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0)
if err != nil {
return nil, err
}
diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}))
iter := trie.NewIterator(diff)

4
eth/api_tracer.go
View file

@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
// Ensure we have a valid starting state before doing any work
origin := start.NumberU64()
database := state.NewDatabase(api.eth.ChainDb())
database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis
if number := start.NumberU64(); number > 0 {
start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
@ -492,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
}
// Otherwise try to reexec blocks until we find a state or reach our limit
origin := block.NumberU64()
database := state.NewDatabase(api.eth.ChainDb())
database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16)
for i := uint64(0); i < reexec; i++ {
block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)

2
eth/backend.go
View file

@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
EWASMInterpreter: config.EWASMInterpreter,
EVMInterpreter: config.EVMInterpreter,
}
cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout}
)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
if err != nil {

22
eth/config.go
View file

@ -43,15 +43,16 @@ var DefaultConfig = Config{
DatasetsInMem: 1,
DatasetsOnDisk: 2,
},
NetworkId: 1,
LightPeers: 100,
DatabaseCache: 768,
TrieCache: 256,
TrieTimeout: 60 * time.Minute,
MinerGasFloor: 8000000,
MinerGasCeil: 8000000,
MinerGasPrice: big.NewInt(params.GWei),
MinerRecommit: 3 * time.Second,
NetworkId: 1,
LightPeers: 100,
DatabaseCache: 512,
TrieCleanCache: 256,
TrieDirtyCache: 256,
TrieTimeout: 60 * time.Minute,
MinerGasFloor: 8000000,
MinerGasCeil: 8000000,
MinerGasPrice: big.NewInt(params.GWei),
MinerRecommit: 3 * time.Second,
TxPool: core.DefaultTxPoolConfig,
GPO: gasprice.Config{
@ -98,7 +99,8 @@ type Config struct {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
TrieCache int
TrieCleanCache int
TrieDirtyCache int
TrieTimeout time.Duration
// Mining-related options

167
eth/downloader/downloader.go
View file

@ -99,6 +99,7 @@ type Downloader struct {
mode SyncMode // Synchronisation mode defining the strategy used (per sync cycle)
mux *event.TypeMux // Event multiplexer to announce sync operation events
genesis uint64 // Genesis block number to limit sync to (e.g. light client CHT)
queue *queue // Scheduler for selecting the hashes to download
peers *peerSet // Set of active peers from which download can proceed
stateDB ethdb.Database
@ -181,6 +182,9 @@ type BlockChain interface {
// HasBlock verifies a block's presence in the local chain.
HasBlock(common.Hash, uint64) bool
// HasFastBlock verifies a fast block's presence in the local chain.
HasFastBlock(common.Hash, uint64) bool
// GetBlockByHash retrieves a block from the local chain.
GetBlockByHash(common.Hash) *types.Block
@ -430,7 +434,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
}
height := latest.Number.Uint64()
origin, err := d.findAncestor(p, height)
origin, err := d.findAncestor(p, latest)
if err != nil {
return err
}
@ -587,41 +591,107 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) {
}
}
// calculateRequestSpan calculates what headers to request from a peer when trying to determine the
// common ancestor.
// It returns parameters to be used for peer.RequestHeadersByNumber:
// from - starting block number
// count - number of headers to request
// skip - number of headers to skip
// and also returns 'max', the last block which is expected to be returned by the remote peers,
// given the (from,count,skip)
func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) {
var (
from int
count int
MaxCount = MaxHeaderFetch / 16
)
// requestHead is the highest block that we will ask for. If requestHead is not offset,
// the highest block that we will get is 16 blocks back from head, which means we
// will fetch 14 or 15 blocks unnecessarily in the case the height difference
// between us and the peer is 1-2 blocks, which is most common
requestHead := int(remoteHeight) - 1
if requestHead < 0 {
requestHead = 0
}
// requestBottom is the lowest block we want included in the query
// Ideally, we want to include just below own head
requestBottom := int(localHeight - 1)
if requestBottom < 0 {
requestBottom = 0
}
totalSpan := requestHead - requestBottom
span := 1 + totalSpan/MaxCount
if span < 2 {
span = 2
}
if span > 16 {
span = 16
}
count = 1 + totalSpan/span
if count > MaxCount {
count = MaxCount
}
if count < 2 {
count = 2
}
from = requestHead - (count-1)*span
if from < 0 {
from = 0
}
max := from + (count-1)*span
return int64(from), count, span - 1, uint64(max)
}
// findAncestor tries to locate the common ancestor link of the local chain and
// a remote peers blockchain. In the general case when our node was in sync and
// on the correct chain, checking the top N links should already get us a match.
// In the rare scenario when we ended up on a long reorganisation (i.e. none of
// the head links match), we do a binary search to find the common ancestor.
func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) {
func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) {
// Figure out the valid ancestor range to prevent rewrite attacks
floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
var (
floor = int64(-1)
localHeight uint64
remoteHeight = remoteHeader.Number.Uint64()
)
switch d.mode {
case FullSync:
localHeight = d.blockchain.CurrentBlock().NumberU64()
case FastSync:
localHeight = d.blockchain.CurrentFastBlock().NumberU64()
default:
localHeight = d.lightchain.CurrentHeader().Number.Uint64()
}
p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight)
if localHeight >= MaxForkAncestry {
// We're above the max reorg threshold, find the earliest fork point
floor = int64(localHeight - MaxForkAncestry)
if d.mode == FullSync {
ceil = d.blockchain.CurrentBlock().NumberU64()
} else if d.mode == FastSync {
ceil = d.blockchain.CurrentFastBlock().NumberU64()
// If we're doing a light sync, ensure the floor doesn't go below the CHT, as
// all headers before that point will be missing.
if d.mode == LightSync {
// If we dont know the current CHT position, find it
if d.genesis == 0 {
header := d.lightchain.CurrentHeader()
for header != nil {
d.genesis = header.Number.Uint64()
if floor >= int64(d.genesis)-1 {
break
}
header = d.lightchain.GetHeaderByHash(header.ParentHash)
}
}
// We already know the "genesis" block number, cap floor to that
if floor < int64(d.genesis)-1 {
floor = int64(d.genesis) - 1
}
}
}
if ceil >= MaxForkAncestry {
floor = int64(ceil - MaxForkAncestry)
}
p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight)
// Request the topmost blocks to short circuit binary ancestor lookup
head := ceil
if head > height {
head = height
}
from := int64(head) - int64(MaxHeaderFetch)
if from < 0 {
from = 0
}
// Span out with 15 block gaps into the future to catch bad head reports
limit := 2 * MaxHeaderFetch / 16
count := 1 + int((int64(ceil)-from)/16)
if count > limit {
count = limit
}
go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false)
p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip)
go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false)
// Wait for the remote response to the head fetch
number, hash := uint64(0), common.Hash{}
@ -647,9 +717,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
return 0, errEmptyHeaderSet
}
// Make sure the peer's reply conforms to the request
for i := 0; i < len(headers); i++ {
if number := headers[i].Number.Int64(); number != from+int64(i)*16 {
p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number)
for i, header := range headers {
expectNumber := from + int64(i)*int64((skip+1))
if number := header.Number.Int64(); number != expectNumber {
p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
return 0, errInvalidChain
}
}
@ -657,20 +728,24 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
finished = true
for i := len(headers) - 1; i >= 0; i-- {
// Skip any headers that underflow/overflow our requested set
if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil {
if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max {
continue
}
// Otherwise check if we already know the header or not
h := headers[i].Hash()
n := headers[i].Number.Uint64()
if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) {
number, hash = n, h
// If every header is known, even future ones, the peer straight out lied about its head
if number > height && i == limit-1 {
p.log.Warn("Lied about chain head", "reported", height, "found", number)
return 0, errStallingPeer
}
var known bool
switch d.mode {
case FullSync:
known = d.blockchain.HasBlock(h, n)
case FastSync:
known = d.blockchain.HasFastBlock(h, n)
default:
known = d.lightchain.HasHeader(h, n)
}
if known {
number, hash = n, h
break
}
}
@ -694,10 +769,12 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
return number, nil
}
// Ancestor not found, we need to binary search over our chain
start, end := uint64(0), head
start, end := uint64(0), remoteHeight
if floor > 0 {
start = uint64(floor)
}
p.log.Trace("Binary searching for common ancestor", "start", start, "end", end)
for start+1 < end {
// Split our chain interval in two, and request the hash to cross check
check := (start + end) / 2
@ -730,7 +807,17 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
// Modify the search interval based on the response
h := headers[0].Hash()
n := headers[0].Number.Uint64()
if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) {
var known bool
switch d.mode {
case FullSync:
known = d.blockchain.HasBlock(h, n)
case FastSync:
known = d.blockchain.HasFastBlock(h, n)
default:
known = d.lightchain.HasHeader(h, n)
}
if !known {
end = check
break
}

108
eth/downloader/downloader_test.go
View file

@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"math/big"
"strings"
"sync"
"sync/atomic"
"testing"
@ -114,6 +115,15 @@ func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool {
return dl.GetBlockByHash(hash) != nil
}
// HasFastBlock checks if a block is present in the testers canonical chain.
func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool {
dl.lock.RLock()
defer dl.lock.RUnlock()
_, ok := dl.ownReceipts[hash]
return ok
}
// GetHeader retrieves a header from the testers canonical chain.
func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header {
dl.lock.RLock()
@ -234,6 +244,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) {
dl.ownHeaders[block.Hash()] = block.Header()
}
dl.ownBlocks[block.Hash()] = block
dl.ownReceipts[block.Hash()] = make(types.Receipts, 0)
dl.stateDb.Put(block.Root().Bytes(), []byte{0x00})
dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty())
}
@ -374,28 +385,28 @@ func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error {
// assertOwnChain checks if the local chain contains the correct number of items
// of the various chain components.
func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
// Mark this method as a helper to report errors at callsite, not in here
t.Helper()
assertOwnForkedChain(t, tester, 1, []int{length})
}
// assertOwnForkedChain checks if the local forked chain contains the correct
// number of items of the various chain components.
func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
// Initialize the counters for the first fork
headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks
// Mark this method as a helper to report errors at callsite, not in here
t.Helper()
// Initialize the counters for the first fork
headers, blocks, receipts := lengths[0], lengths[0], lengths[0]
if receipts < 0 {
receipts = 1
}
// Update the counters for each subsequent fork
for _, length := range lengths[1:] {
headers += length - common
blocks += length - common
receipts += length - common - fsMinFullBlocks
receipts += length - common
}
switch tester.downloader.mode {
case FullSync:
receipts = 1
case LightSync:
if tester.downloader.mode == LightSync {
blocks, receipts = 1, 1
}
if hs := len(tester.ownHeaders); hs != headers {
@ -1149,7 +1160,9 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
}
func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) {
// Mark this method as a helper to report errors at callsite, not in here
t.Helper()
p := d.Progress()
p.KnownStates, p.PulledStates = 0, 0
want.KnownStates, want.PulledStates = 0, 0
@ -1479,3 +1492,78 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int
}
return nil
}
func TestRemoteHeaderRequestSpan(t *testing.T) {
testCases := []struct {
remoteHeight uint64
localHeight uint64
expected []int
}{
// Remote is way higher. We should ask for the remote head and go backwards
{1500, 1000,
[]int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499},
},
{15000, 13006,
[]int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999},
},
//Remote is pretty close to us. We don't have to fetch as many
{1200, 1150,
[]int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199},
},
// Remote is equal to us (so on a fork with higher td)
// We should get the closest couple of ancestors
{1500, 1500,
[]int{1497, 1499},
},
// We're higher than the remote! Odd
{1000, 1500,
[]int{997, 999},
},
// Check some weird edgecases that it behaves somewhat rationally
{0, 1500,
[]int{0, 2},
},
{6000000, 0,
[]int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999},
},
{0, 0,
[]int{0, 2},
},
}
reqs := func(from, count, span int) []int {
var r []int
num := from
for len(r) < count {
r = append(r, num)
num += span + 1
}
return r
}
for i, tt := range testCases {
from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight)
data := reqs(int(from), count, span)
if max != uint64(data[len(data)-1]) {
t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max)
}
failed := false
if len(data) != len(tt.expected) {
failed = true
t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data))
} else {
for j, n := range data {
if n != tt.expected[j] {
failed = true
break
}
}
}
if failed {
res := strings.Replace(fmt.Sprint(data), " ", ",", -1)
exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1)
fmt.Printf("got: %v\n", res)
fmt.Printf("exp: %v\n", exp)
t.Errorf("test %d: wrong values", i)
}
}
}

2
eth/downloader/queue.go
View file

@ -325,7 +325,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
}
// Make sure no duplicate requests are executed
if _, ok := q.blockTaskPool[hash]; ok {
log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash)
log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash)
continue
}
if _, ok := q.receiptTaskPool[hash]; ok {

28
eth/gen_config.go
View file

@ -30,7 +30,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
TrieCache int
TrieCleanCache int
TrieDirtyCache int
TrieTimeout time.Duration
Etherbase common.Address `toml:",omitempty"`
MinerNotify []string `toml:",omitempty"`
@ -45,6 +46,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
GPO gasprice.Config
EnablePreimageRecording bool
DocRoot string `toml:"-"`
EWASMInterpreter string
EVMInterpreter string
}
var enc Config
enc.Genesis = c.Genesis
@ -58,7 +61,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.SkipBcVersionCheck = c.SkipBcVersionCheck
enc.DatabaseHandles = c.DatabaseHandles
enc.DatabaseCache = c.DatabaseCache
enc.TrieCache = c.TrieCache
enc.TrieCleanCache = c.TrieCleanCache
enc.TrieDirtyCache = c.TrieDirtyCache
enc.TrieTimeout = c.TrieTimeout
enc.Etherbase = c.Etherbase
enc.MinerNotify = c.MinerNotify
@ -74,6 +78,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.EnablePreimageRecording = c.EnablePreimageRecording
enc.DocRoot = c.DocRoot
enc.EWASMInterpreter = c.EWASMInterpreter
enc.EVMInterpreter = c.EVMInterpreter
return &enc, nil
}
@ -91,7 +97,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
SkipBcVersionCheck *bool `toml:"-"`
DatabaseHandles *int `toml:"-"`
DatabaseCache *int
TrieCache *int
TrieCleanCache *int
TrieDirtyCache *int
TrieTimeout *time.Duration
Etherbase *common.Address `toml:",omitempty"`
MinerNotify []string `toml:",omitempty"`
@ -106,6 +113,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
GPO *gasprice.Config
EnablePreimageRecording *bool
DocRoot *string `toml:"-"`
EWASMInterpreter *string
EVMInterpreter *string
}
var dec Config
if err := unmarshal(&dec); err != nil {
@ -144,8 +153,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.DatabaseCache != nil {
c.DatabaseCache = *dec.DatabaseCache
}
if dec.TrieCache != nil {
c.TrieCache = *dec.TrieCache
if dec.TrieCleanCache != nil {
c.TrieCleanCache = *dec.TrieCleanCache
}
if dec.TrieDirtyCache != nil {
c.TrieDirtyCache = *dec.TrieDirtyCache
}
if dec.TrieTimeout != nil {
c.TrieTimeout = *dec.TrieTimeout
@ -189,5 +201,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.DocRoot != nil {
c.DocRoot = *dec.DocRoot
}
if dec.EWASMInterpreter != nil {
c.EWASMInterpreter = *dec.EWASMInterpreter
}
if dec.EVMInterpreter != nil {
c.EVMInterpreter = *dec.EVMInterpreter
}
return nil
}

4
eth/handler.go
View file

@ -653,12 +653,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
trueHead = request.Block.ParentHash()
trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty())
)
// Update the peers total difficulty if better than the previous
// Update the peer's total difficulty if better than the previous
if _, td := p.Head(); trueTD.Cmp(td) > 0 {
p.SetHead(trueHead, trueTD)
// Schedule a sync if above ours. Note, this will not fire a sync for a gap of
// a singe block (as the true TD is below the propagated block), however this
// a single block (as the true TD is below the propagated block), however this
// scenario should easily be covered by the fetcher.
currentBlock := pm.blockchain.CurrentBlock()
if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 {

4
eth/handler_test.go
View file

@ -585,7 +585,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
}
}(peer)
}
timeoutCh := time.NewTimer(time.Millisecond * 100).C
timeout := time.After(300 * time.Millisecond)
var receivedCount int
outer:
for {
@ -597,7 +597,7 @@ outer:
if receivedCount == totalPeers {
break outer
}
case <-timeoutCh:
case <-timeout:
break outer
}
}

21
internal/cmdtest/test_cmd.go
View file

@ -27,6 +27,7 @@ import (
"regexp"
"strings"
"sync"
"syscall"
"testing"
"text/template"
"time"
@ -50,6 +51,8 @@ type TestCmd struct {
stdout *bufio.Reader
stdin io.WriteCloser
stderr *testlogger
// Err will contain the process exit error or interrupt signal error
Err error
}
// Run exec's the current binary using name as argv[0] which will trigger the
@ -182,11 +185,25 @@ func (tt *TestCmd) ExpectExit() {
}
func (tt *TestCmd) WaitExit() {
tt.cmd.Wait()
tt.Err = tt.cmd.Wait()
}
func (tt *TestCmd) Interrupt() {
tt.cmd.Process.Signal(os.Interrupt)
tt.Err = tt.cmd.Process.Signal(os.Interrupt)
}
// ExitStatus exposes the process' OS exit code
// It will only return a valid value after the process has finished.
func (tt *TestCmd) ExitStatus() int {
if tt.Err != nil {
exitErr := tt.Err.(*exec.ExitError)
if exitErr != nil {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
return status.ExitStatus()
}
}
}
return 0
}
// StderrText returns any stderr output written so far.

2
internal/ethapi/api.go
View file

@ -339,7 +339,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
return fetchKeystore(s.am).Lock(addr) == nil
}
// signTransactions sets defaults and signs the given transaction
// signTransaction sets defaults and signs the given transaction
// NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
// and release it after the transaction has been submitted to the tx pool
func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) {

66
les/fetcher.go
View file

@ -149,38 +149,40 @@ func (f *lightFetcher) syncLoop() {
s := requesting
requesting = false
var (
rq *distReq
reqID uint64
rq *distReq
reqID uint64
syncing bool
)
if !f.syncing && !(newAnnounce && s) {
rq, reqID = f.nextRequest()
rq, reqID, syncing = f.nextRequest()
}
syncing := f.syncing
f.lock.Unlock()
if rq != nil {
requesting = true
_, ok := <-f.pm.reqDist.queue(rq)
if !ok {
if _, ok := <-f.pm.reqDist.queue(rq); ok {
if syncing {
f.lock.Lock()
f.syncing = true
f.lock.Unlock()
} else {
go func() {
time.Sleep(softRequestTimeout)
f.reqMu.Lock()
req, ok := f.requested[reqID]
if ok {
req.timeout = true
f.requested[reqID] = req
}
f.reqMu.Unlock()
// keep starting new requests while possible
f.requestChn <- false
}()
}
} else {
f.requestChn <- false
}
if !syncing {
go func() {
time.Sleep(softRequestTimeout)
f.reqMu.Lock()
req, ok := f.requested[reqID]
if ok {
req.timeout = true
f.requested[reqID] = req
}
f.reqMu.Unlock()
// keep starting new requests while possible
f.requestChn <- false
}()
}
}
case reqID := <-f.timeoutChn:
f.reqMu.Lock()
@ -219,6 +221,7 @@ func (f *lightFetcher) syncLoop() {
f.checkSyncedHeaders(p)
f.syncing = false
f.lock.Unlock()
f.requestChn <- false
}
}
}
@ -417,7 +420,7 @@ func (f *lightFetcher) requestedID(reqID uint64) bool {
// nextRequest selects the peer and announced head to be requested next, amount
// to be downloaded starting from the head backwards is also returned
func (f *lightFetcher) nextRequest() (*distReq, uint64) {
func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) {
var (
bestHash common.Hash
bestAmount uint64
@ -427,19 +430,17 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
bestHash, bestAmount, bestTd, bestSyncing = f.findBestRequest()
if bestTd == f.maxConfirmedTd {
return nil, 0
return nil, 0, false
}
f.syncing = bestSyncing
var rq *distReq
reqID := genReqID()
if f.syncing {
if bestSyncing {
rq = f.newFetcherDistReqForSync(bestHash)
} else {
rq = f.newFetcherDistReq(bestHash, reqID, bestAmount)
}
return rq, reqID
return rq, reqID, bestSyncing
}
// findBestRequest finds the best head to request that has been announced by but not yet requested from a known peer.
@ -496,6 +497,9 @@ func (f *lightFetcher) newFetcherDistReqForSync(bestHash common.Hash) *distReq {
},
canSend: func(dp distPeer) bool {
p := dp.(*peer)
f.lock.Lock()
defer f.lock.Unlock()
if p.isOnlyAnnounce {
return false
}
@ -528,6 +532,9 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes
},
canSend: func(dp distPeer) bool {
p := dp.(*peer)
f.lock.Lock()
defer f.lock.Unlock()
if p.isOnlyAnnounce {
return false
}
@ -541,7 +548,7 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes
},
request: func(dp distPeer) func() {
p := dp.(*peer)
f.lock.Lock()
fp := f.peers[p]
if fp != nil {
n := fp.nodeByHash[bestHash]
@ -549,6 +556,7 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes
n.requested = true
}
}
f.lock.Unlock()
cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount))
p.fcServer.QueueRequest(reqID, cost)

1
les/flowcontrol/control.go
View file

@ -82,7 +82,6 @@ func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) {
time := mclock.Now()
peer.recalcBV(time)
peer.bufValue -= cost
peer.recalcBV(time)
rcValue, rcost := peer.cm.processed(peer.cmNode, time)
if rcValue < peer.params.BufLimit {
bv := peer.params.BufLimit - rcValue

2
les/serverpool.go
View file

@ -729,7 +729,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
}
func encodePubkey64(pub *ecdsa.PublicKey) []byte {
return crypto.FromECDSAPub(pub)[:1]
return crypto.FromECDSAPub(pub)[1:]
}
func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {

8
les/ulc_test.go
View file

@ -81,15 +81,7 @@ func TestULCReceiveAnnounce(t *testing.T) {
Td: td.Add(td, big.NewInt(1)),
}
announce.sign(f.Key)
lPeer.SendAnnounce(announce)
time.Sleep(time.Millisecond)
l.PM.peers.lock.Lock()
if len(l.PM.peers.peers) == 0 {
t.Fatal("peer list after receiving message should not be empty")
}
l.PM.peers.lock.Unlock()
}
func TestULCShouldNotSyncWithTwoPeersOneHaveEmptyChain(t *testing.T) {

4
light/postprocess.go
View file

@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co
diskdb: db,
odr: odr,
trieTable: trieTable,
triedb: trie.NewDatabase(trieTable),
triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
sectionSize: size,
}
return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht")
@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
diskdb: db,
odr: odr,
trieTable: trieTable,
triedb: trie.NewDatabase(trieTable),
triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down
parentSize: parentSize,
size: size,
}

2
light/trie.go
View file

@ -108,7 +108,7 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) {
func (t *odrTrie) TryUpdate(key, value []byte) error {
key = crypto.Keccak256(key)
return t.do(key, func() error {
return t.trie.TryDelete(key)
return t.trie.TryUpdate(key, value)
})
}

7
mobile/big.go
View file

@ -84,6 +84,13 @@ func (bi *BigInt) SetString(x string, base int) {
// BigInts represents a slice of big ints.
type BigInts struct{ bigints []*big.Int }
// NewBigInts creates a slice of uninitialized big numbers.
func NewBigInts(size int) *BigInts {
return &BigInts{
bigints: make([]*big.Int, size),
}
}
// Size returns the number of big ints in the slice.
func (bi *BigInts) Size() int {
return len(bi.bigints)

2
node/node.go
View file

@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error {
if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
return err
}
n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace)
n.log.Debug("InProc registered", "namespace", api.Namespace)
}
n.inprocHandler = handler
return nil

2
p2p/discover/table.go
View file

@ -434,7 +434,7 @@ func (tab *Table) loadSeedNodes() {
for i := range seeds {
seed := seeds[i]
age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID())) }}
log.Debug("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age)
tab.add(seed)
}
}

9
p2p/discv5/net.go
View file

@ -567,12 +567,11 @@ loop:
net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte {
if n.state != nil && n.state.canQuery {
return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration
} else {
if n.state == unknown {
net.ping(n, n.addr())
}
return nil
}
if n.state == unknown {
net.ping(n, n.addr())
}
return nil
})
case <-statsDump.C:

43
p2p/protocols/accounting.go
View file

@ -16,29 +16,32 @@
package protocols
import "github.com/ethereum/go-ethereum/metrics"
import (
"time"
"github.com/ethereum/go-ethereum/metrics"
)
//define some metrics
var (
//NOTE: these metrics just define the interfaces and are currently *NOT persisted* over sessions
//All metrics are cumulative
//total amount of units credited
mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", nil)
mBalanceCredit metrics.Counter
//total amount of units debited
mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", nil)
mBalanceDebit metrics.Counter
//total amount of bytes credited
mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", nil)
mBytesCredit metrics.Counter
//total amount of bytes debited
mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", nil)
mBytesDebit metrics.Counter
//total amount of credited messages
mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", nil)
mMsgCredit metrics.Counter
//total amount of debited messages
mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", nil)
mMsgDebit metrics.Counter
//how many times local node had to drop remote peers
mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", nil)
mPeerDrops metrics.Counter
//how many times local node overdrafted and dropped
mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", nil)
mSelfDrops metrics.Counter
)
//Prices defines how prices are being passed on to the accounting instance
@ -105,6 +108,26 @@ func NewAccounting(balance Balance, po Prices) *Accounting {
return ah
}
//SetupAccountingMetrics creates a separate registry for p2p accounting metrics;
//this registry should be independent of any other metrics as it persists at different endpoints.
//It also instantiates the given metrics and starts the persisting go-routine which
//at the passed interval writes the metrics to a LevelDB
func SetupAccountingMetrics(reportInterval time.Duration, path string) *AccountingMetrics {
//create an empty registry
registry := metrics.NewRegistry()
//instantiate the metrics
mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", registry)
mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", registry)
mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", registry)
mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", registry)
mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", registry)
mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", registry)
mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", registry)
mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", registry)
//create the DB and start persisting
return NewAccountingMetrics(registry, reportInterval, path)
}
//Implement Hook.Send
// Send takes a peer, a size and a msg and
// - calculates the cost for the local node sending a msg of size to peer using the Prices interface

10
p2p/protocols/accounting_simulation_test.go
View file

@ -20,7 +20,10 @@ import (
"context"
"flag"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"reflect"
"sync"
"testing"
@ -66,6 +69,13 @@ func init() {
func TestAccountingSimulation(t *testing.T) {
//setup the balances objects for every node
bal := newBalances(*nodes)
//setup the metrics system or tests will fail trying to write metrics
dir, err := ioutil.TempDir("", "account-sim")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
SetupAccountingMetrics(1*time.Second, filepath.Join(dir, "metrics.db"))
//define the node.Service for this test
services := adapters.Services{
"accounting": func(ctx *adapters.ServiceContext) (node.Service, error) {

2
p2p/protocols/protocol.go
View file

@ -381,7 +381,7 @@ func (p *Peer) handleIncoming(handle func(ctx context.Context, msg interface{})
// * arguments
// * context
// * the local handshake to be sent to the remote peer
// * funcion to be called on the remote handshake (can be nil)
// * function to be called on the remote handshake (can be nil)
// * expects a remote handshake back of the same type
// * the dialing peer needs to send the handshake first and then waits for remote
// * the listening peer waits for the remote handshake and then sends it

147
p2p/protocols/reporter.go Normal file
View file

@ -0,0 +1,147 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package protocols
import (
"encoding/binary"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/syndtr/goleveldb/leveldb"
)
//AccountMetrics abstracts away the metrics DB and
//the reporter to persist metrics
type AccountingMetrics struct {
reporter *reporter
}
//Close will be called when the node is being shutdown
//for a graceful cleanup
func (am *AccountingMetrics) Close() {
close(am.reporter.quit)
am.reporter.db.Close()
}
//reporter is an internal structure used to write p2p accounting related
//metrics to a LevelDB. It will periodically write the accrued metrics to the DB.
type reporter struct {
reg metrics.Registry //the registry for these metrics (independent of other metrics)
interval time.Duration //duration at which the reporter will persist metrics
db *leveldb.DB //the actual DB
quit chan struct{} //quit the reporter loop
}
//NewMetricsDB creates a new LevelDB instance used to persist metrics defined
//inside p2p/protocols/accounting.go
func NewAccountingMetrics(r metrics.Registry, d time.Duration, path string) *AccountingMetrics {
var val = make([]byte, 8)
var err error
//Create the LevelDB
db, err := leveldb.OpenFile(path, nil)
if err != nil {
log.Error(err.Error())
return nil
}
//Check for all defined metrics that there is a value in the DB
//If there is, assign it to the metric. This means that the node
//has been running before and that metrics have been persisted.
metricsMap := map[string]metrics.Counter{
"account.balance.credit": mBalanceCredit,
"account.balance.debit": mBalanceDebit,
"account.bytes.credit": mBytesCredit,
"account.bytes.debit": mBytesDebit,
"account.msg.credit": mMsgCredit,
"account.msg.debit": mMsgDebit,
"account.peerdrops": mPeerDrops,
"account.selfdrops": mSelfDrops,
}
//iterate the map and get the values
for key, metric := range metricsMap {
val, err = db.Get([]byte(key), nil)
//until the first time a value is being written,
//this will return an error.
//it could be beneficial though to log errors later,
//but that would require a different logic
if err == nil {
metric.Inc(int64(binary.BigEndian.Uint64(val)))
}
}
//create the reporter
rep := &reporter{
reg: r,
interval: d,
db: db,
quit: make(chan struct{}),
}
//run the go routine
go rep.run()
m := &AccountingMetrics{
reporter: rep,
}
return m
}
//run is the goroutine which periodically sends the metrics to the configured LevelDB
func (r *reporter) run() {
intervalTicker := time.NewTicker(r.interval)
for {
select {
case <-intervalTicker.C:
//at each tick send the metrics
if err := r.save(); err != nil {
log.Error("unable to send metrics to LevelDB", "err", err)
//If there is an error in writing, exit the routine; we assume here that the error is
//severe and don't attempt to write again.
//Also, this should prevent leaking when the node is stopped
return
}
case <-r.quit:
//graceful shutdown
return
}
}
}
//send the metrics to the DB
func (r *reporter) save() error {
//create a LevelDB Batch
batch := leveldb.Batch{}
//for each metric in the registry (which is independent)...
r.reg.Each(func(name string, i interface{}) {
metric, ok := i.(metrics.Counter)
if ok {
//assuming every metric here to be a Counter (separate registry)
//...create a snapshot...
ms := metric.Snapshot()
byteVal := make([]byte, 8)
binary.BigEndian.PutUint64(byteVal, uint64(ms.Count()))
//...and save the value to the DB
batch.Put([]byte(name), byteVal)
}
})
return r.db.Write(&batch, nil)
}

77
p2p/protocols/reporter_test.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package protocols
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
"github.com/ethereum/go-ethereum/log"
)
//TestReporter tests that the metrics being collected for p2p accounting
//are being persisted and available after restart of a node.
//It simulates restarting by just recreating the DB as if the node had restarted.
func TestReporter(t *testing.T) {
//create a test directory
dir, err := ioutil.TempDir("", "reporter-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
//setup the metrics
log.Debug("Setting up metrics first time")
reportInterval := 5 * time.Millisecond
metrics := SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db"))
log.Debug("Done.")
//do some metrics
mBalanceCredit.Inc(12)
mBytesCredit.Inc(34)
mMsgDebit.Inc(9)
//give the reporter time to write the metrics to DB
time.Sleep(20 * time.Millisecond)
//set the metrics to nil - this effectively simulates the node having shut down...
mBalanceCredit = nil
mBytesCredit = nil
mMsgDebit = nil
//close the DB also, or we can't create a new one
metrics.Close()
//setup the metrics again
log.Debug("Setting up metrics second time")
metrics = SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db"))
defer metrics.Close()
log.Debug("Done.")
//now check the metrics, they should have the same value as before "shutdown"
if mBalanceCredit.Count() != 12 {
t.Fatalf("Expected counter to be %d, but is %d", 12, mBalanceCredit.Count())
}
if mBytesCredit.Count() != 34 {
t.Fatalf("Expected counter to be %d, but is %d", 23, mBytesCredit.Count())
}
if mMsgDebit.Count() != 9 {
t.Fatalf("Expected counter to be %d, but is %d", 9, mMsgDebit.Count())
}
}

30
params/config.go
View file

@ -49,10 +49,10 @@ var (
// MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network.
MainnetTrustedCheckpoint = &TrustedCheckpoint{
Name: "mainnet",
SectionIndex: 195,
SectionHead: common.HexToHash("0x1cdd2a84cf6c1261ffccc88f6bcefb513abd7934a96c1e909fbf74767560f16b"),
CHTRoot: common.HexToHash("0xe453333c20391d16b91b6fe11c104704f62c8dba15f69db73b4cdf7e100105eb"),
BloomRoot: common.HexToHash("0x47f30069473072e00d2cdca146dce40f0aad243dfc8221bf810822c091674efe"),
SectionIndex: 206,
SectionHead: common.HexToHash("0x9fa677c7c0580136f5a86d9b2fd29b112e531f0284396298b8809bcb6787b538"),
CHTRoot: common.HexToHash("0x7f32dfb29e341b4c8c10ea2e06a812bcea470366f635b7a8b3d0856684cd76f4"),
BloomRoot: common.HexToHash("0x0169e174f0a8172aec217556d8a25c7ba7ca52aacff170325230a75740ff1eaf"),
}
// TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network.
@ -73,10 +73,10 @@ var (
// TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network.
TestnetTrustedCheckpoint = &TrustedCheckpoint{
Name: "testnet",
SectionIndex: 126,
SectionHead: common.HexToHash("0x48f7dd4c9c60be04bf15fd4d0bcac46ddd8caf6b01d6fb8f8e1f7955cdd1337a"),
CHTRoot: common.HexToHash("0x6e54cb80a1884881ea1a114243af9012c95e0296b47f103b5ab124313968508e"),
BloomRoot: common.HexToHash("0xb55accf6dce6455b47db8510d15eff38d0ed7378829f3036d26b48e7d15da3f6"),
SectionIndex: 136,
SectionHead: common.HexToHash("0xe5d80bb08d92bbc12dfe510c64cba01eafcbb4ba585e7c7ab7f8a93c6f295ab3"),
CHTRoot: common.HexToHash("0xe3ca77ab0cb51eec74f4f7458e36aee207c68768387b39cb0bcff0940a6264d8"),
BloomRoot: common.HexToHash("0x30c8eeadac5539d3dcd6e88915d1a07cb2f3a1d6ebe7e553e3ee783c04c68c2d"),
}
// RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network.
@ -100,10 +100,10 @@ var (
// RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network.
RinkebyTrustedCheckpoint = &TrustedCheckpoint{
Name: "rinkeby",
SectionIndex: 93,
SectionHead: common.HexToHash("0xdefb94aa217ab38f2919f7318d1d5476bd2aabf1ec9148047fe03e555615e0b4"),
CHTRoot: common.HexToHash("0x52c98c2fe508a8332c27dc10538f3fead43306e2b22b597587763c2fe6586da6"),
BloomRoot: common.HexToHash("0x93d83be0c1b12f732b1a027ecdfb16f39b0d020b8c10bfb90e76f3b01adfc5b6"),
SectionIndex: 103,
SectionHead: common.HexToHash("0x9f38b903852831bf4fa7992f7fd43d8b26da2deb82b421fb845cf6faee54e056"),
CHTRoot: common.HexToHash("0x2d710c2cea468d2e604838000d658ee213e4abb07f90c4f71f5cd7f8510aa708"),
BloomRoot: common.HexToHash("0xcc401060280c2cc82697ea5ecef8cac61e52063c37533a2e9609332419704d5f"),
}
// AllEthashProtocolChanges contains every protocol change (EIPs) introduced
@ -111,16 +111,16 @@ var (
//
// This configuration is intentionally not using keyed fields to force anyone
// adding flags to the config to also have to set these fields.
AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
// AllCliqueProtocolChanges contains every protocol change (EIPs) introduced
// and accepted by the Ethereum core developers into the Clique consensus.
//
// This configuration is intentionally not using keyed fields to force anyone
// adding flags to the config to also have to set these fields.
AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}}
TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil}
TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil}
TestRules = TestChainConfig.Rules(new(big.Int))
)

2
params/version.go
View file

@ -23,7 +23,7 @@ import (
const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 8 // Minor version component of the current release
VersionPatch = 18 // Patch version component of the current release
VersionPatch = 20 // Patch version component of the current release
VersionMeta = "unstable" // Version metadata to append to the version string
)

2
rpc/client_example_test.go
View file

@ -25,7 +25,7 @@ import (
"github.com/ethereum/go-ethereum/rpc"
)
// In this example, our client whishes to track the latest 'block number'
// In this example, our client wishes to track the latest 'block number'
// known to the server. The server supports two methods:
//
// eth_getBlockByNumber("latest", {})

2
rpc/doc.go
View file

@ -32,7 +32,7 @@ An example method:
func (s *CalcService) Add(a, b int) (int, error)
When the returned error isn't nil the returned integer is ignored and the error is
send back to the client. Otherwise the returned integer is send back to the client.
sent back to the client. Otherwise the returned integer is sent back to the client.
Optional arguments are supported by accepting pointer values as arguments. E.g.
if we want to do the addition in an optional finite field we can accept a mod

1
swarm/OWNERS
View file

@ -7,7 +7,6 @@ swarm
├── fuse ────────────────── @jmozah, @holisticode
├── grafana_dashboards ──── @nonsense
├── metrics ─────────────── @nonsense, @holisticode
├── multihash ───────────── @nolash
├── network ─────────────── ethersphere
│ ├── bitvector ───────── @zelig, @janos, @gbalint
│ ├── priorityqueue ───── @zelig, @janos, @gbalint

27
swarm/api/api.go
View file

@ -42,7 +42,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
@ -417,7 +416,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
return reader, mimeType, status, nil, err
}
// get the data of the update
_, rsrcData, err := a.feed.GetContent(entry.Feed)
_, contentAddr, err := a.feed.GetContent(entry.Feed)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
@ -425,23 +424,23 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
return reader, mimeType, status, nil, err
}
// extract multihash
decodedMultihash, err := multihash.FromMultihash(rsrcData)
if err != nil {
// extract content hash
if len(contentAddr) != storage.AddressLength {
apiGetInvalid.Inc(1)
status = http.StatusUnprocessableEntity
log.Warn("invalid multihash in feed update", "err", err)
return reader, mimeType, status, nil, err
errorMessage := fmt.Sprintf("invalid swarm hash in feed update. Expected %d bytes. Got %d", storage.AddressLength, len(contentAddr))
log.Warn(errorMessage)
return reader, mimeType, status, nil, errors.New(errorMessage)
}
manifestAddr = storage.Address(decodedMultihash)
log.Trace("feed update contains multihash", "key", manifestAddr)
manifestAddr = storage.Address(contentAddr)
log.Trace("feed update contains swarm hash", "key", manifestAddr)
// get the manifest the multihash digest points to
// get the manifest the swarm hash points to
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err))
log.Warn(fmt.Sprintf("loadManifestTrie (feed update) error: %v", err))
return reader, mimeType, status, nil, err
}
@ -451,8 +450,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
if entry == nil {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path)
log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path)
err = fmt.Errorf("manifest (feed update) entry for '%s' not found", path)
log.Trace("manifest (feed update) entry not found", "key", manifestAddr, "path", path)
return reader, mimeType, status, nil, err
}
}
@ -472,7 +471,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
// no entry found
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest entry for '%s' not found", path)
err = fmt.Errorf("Not found: could not find resource '%s'", path)
log.Trace("manifest entry not found", "key", contentAddr, "path", path)
}
return

129
swarm/api/client/client_test.go
View file

@ -25,18 +25,17 @@ import (
"sort"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/api"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func serverFunc(api *api.API) testutil.TestServer {
func serverFunc(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}
@ -49,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) {
}
func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
client := NewClient(srv.URL)
@ -90,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
}
func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
client := NewClient(srv.URL)
@ -188,7 +187,7 @@ func newTestDirectory(t *testing.T) string {
// TestClientUploadDownloadDirectory tests uploading and downloading a
// directory of files to a swarm manifest
func TestClientUploadDownloadDirectory(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
dir := newTestDirectory(t)
@ -254,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) {
}
func testClientFileList(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
dir := newTestDirectory(t)
@ -312,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
// TestClientMultipartUpload tests uploading files to swarm using a multipart
// upload
func TestClientMultipartUpload(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// define an uploader which uploads testDirFiles with some data
@ -369,58 +368,99 @@ func newTestSigner() (*feed.GenericSigner, error) {
return feed.NewGenericSigner(privKey), nil
}
// test the transparent resolving of multihash feed updates with bzz:// scheme
// Test the transparent resolving of feed updates with bzz:// scheme
//
// first upload data, and store the multihash to the resulting manifest in a feed update
// retrieving the update with the multihash should return the manifest pointing directly to the data
// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
// This effectively uses a feed to store a pointer to content rather than the content itself
// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
// and raw retrieve of that hash should return the data
func TestClientCreateFeedMultihash(t *testing.T) {
func TestClientBzzWithFeed(t *testing.T) {
signer, _ := newTestSigner()
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
client := NewClient(srv.URL)
// Initialize a Swarm test server
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
swarmClient := NewClient(srv.URL)
defer srv.Close()
// add the data our multihash aliased manifest will point to
databytes := []byte("bar")
// put together some data for our test:
dataBytes := []byte(`
//
// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
//
// MANIFEST HASH --> DATA
//
// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
//
// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
//
// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
// stays constant, we have effectively created a fixed address to changing content. (Applause)
//
// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2)
//
`)
swarmHash, err := client.UploadRaw(bytes.NewReader(databytes), int64(len(databytes)), false)
if err != nil {
t.Fatalf("Error uploading raw test data: %s", err)
// Create a virtual File out of memory containing the above data
f := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(dataBytes)),
},
}
s := common.FromHex(swarmHash)
mh := multihash.ToMultihash(s)
// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
manifestAddressHex, err := swarmClient.Upload(f, "", false)
if err != nil {
t.Fatalf("Error creating manifest: %s", err)
}
// our feed topic
topic, _ := feed.NewTopic("foo.eth", nil)
// convert the hex-encoded manifest hash to a 32-byte slice
manifestAddress := common.FromHex(manifestAddressHex)
createRequest := feed.NewFirstRequest(topic)
if len(manifestAddress) != storage.AddressLength {
t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress))
}
createRequest.SetData(mh)
if err := createRequest.Sign(signer); err != nil {
// Now create a **feed manifest**. For that, we need a topic:
topic, _ := feed.NewTopic("interesting topic indeed", nil)
// Build a feed request to update data
request := feed.NewFirstRequest(topic)
// Put the 32-byte address of the manifest into the feed update
request.SetData(manifestAddress)
// Sign the update
if err := request.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
feedManifestHash, err := client.CreateFeedWithManifest(createRequest)
// Publish the update and at the same time request a **feed manifest** to be created
feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request)
if err != nil {
t.Fatalf("Error creating feed manifest: %s", err)
}
correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
if feedManifestHash != correctManifestAddrHex {
t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash)
// Check we have received the exact **feed manifest** to be expected
// given the topic and user signing the updates:
correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
if feedManifestAddressHex != correctFeedManifestAddrHex {
t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex)
}
// Check we get a not found error when trying to get feed updates with a made-up manifest
_, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
_, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if err != ErrNoFeedUpdatesFound {
t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err)
}
reader, err := client.QueryFeed(nil, correctManifestAddrHex)
// If we query the feed directly we should get **manifest hash** back:
reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving feed updates: %s", err)
}
@ -429,10 +469,27 @@ func TestClientCreateFeedMultihash(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(mh, gotData) {
t.Fatalf("Expected: %v, got %v", mh, gotData)
//Check that indeed the **manifest hash** is retrieved
if !bytes.Equal(manifestAddress, gotData) {
t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
}
// Now the final test we were looking for: Use bzz://<feed-manifest> and that should resolve all manifests
// and return the original data directly:
f, err = swarmClient.Download(feedManifestAddressHex, "")
if err != nil {
t.Fatal(err)
}
gotData, err = ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
// Check that we get back the original data:
if !bytes.Equal(dataBytes, gotData) {
t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
}
}
// TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client.
@ -440,7 +497,7 @@ func TestClientCreateUpdateFeed(t *testing.T) {
signer, _ := newTestSigner()
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
client := NewClient(srv.URL)
defer srv.Close()

2
swarm/api/http/middleware.go
View file

@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writer := newLoggingResponseWriter(w)
h.ServeHTTP(writer, r)
log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode)
})
}

12
swarm/api/http/response_test.go
View file

@ -24,12 +24,10 @@ import (
"testing"
"golang.org/x/net/html"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestError(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@ -55,7 +53,7 @@ func TestError(t *testing.T) {
}
func Test404Page(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@ -81,7 +79,7 @@ func Test404Page(t *testing.T) {
}
func Test500Page(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@ -106,7 +104,7 @@ func Test500Page(t *testing.T) {
}
}
func Test500PageWith0xHashPrefix(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) {
}
func TestJsonResponse(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response

136
swarm/api/http/server_test.go
View file

@ -20,7 +20,6 @@ import (
"archive/tar"
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"flag"
@ -46,7 +45,6 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/swarm/testutil"
@ -58,7 +56,7 @@ func init() {
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
func serverFunc(api *api.API) testutil.TestServer {
func serverFunc(api *api.API) TestServer {
return NewServer(api, "")
}
@ -70,60 +68,91 @@ func newTestSigner() (*feed.GenericSigner, error) {
return feed.NewGenericSigner(privKey), nil
}
// test the transparent resolving of multihash-containing feed updates with bzz:// scheme
// Test the transparent resolving of feed updates with bzz:// scheme
//
// first upload data, and store the multihash to the resulting manifest in a feed update
// retrieving the update with the multihash should return the manifest pointing directly to the data
// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
// This effectively uses a feed to store a pointer to content rather than the content itself
// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
// and raw retrieve of that hash should return the data
func TestBzzFeedMultihash(t *testing.T) {
func TestBzzWithFeed(t *testing.T) {
signer, _ := newTestSigner()
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
// Initialize Swarm test server
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// add the data our multihash aliased manifest will point to
databytes := "bar"
testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL)
resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes)))
// put together some data for our test:
dataBytes := []byte(`
//
// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
//
// MANIFEST HASH --> DATA
//
// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
//
// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
//
// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
// stays constant, we have effectively created a fixed address to changing content. (Applause)
//
// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) ...
//
`)
// POST data to bzz and get back a content-addressed **manifest hash** pointing to it.
resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader([]byte(dataBytes)))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
}
b, err := ioutil.ReadAll(resp.Body)
manifestAddressHex, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
s := common.FromHex(string(b))
mh := multihash.ToMultihash(s)
log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
manifestAddress := common.FromHex(string(manifestAddressHex))
topic, _ := feed.NewTopic("foo.eth", nil)
log.Info("added data", "manifest", string(manifestAddressHex))
// At this point we have uploaded the data and have a manifest pointing to it
// Now store that manifest address in a feed update.
// We also want a feed manifest, so we can use it to refer to the feed.
// First, create a topic for our feed:
topic, _ := feed.NewTopic("interesting topic indeed", nil)
// Create a feed update request:
updateRequest := feed.NewFirstRequest(topic)
updateRequest.SetData(mh)
// Store the **manifest address** as data into the feed update.
updateRequest.SetData(manifestAddress)
// Sign the update
if err := updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
log.Info("added data", "data", common.ToHex(manifestAddress))
testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL))
// Build the feed update http request:
feedUpdateURL, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL))
if err != nil {
t.Fatal(err)
}
query := testUrl.Query()
query := feedUpdateURL.Query()
body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted
query.Set("manifest", "1") // indicate we want a manifest back
testUrl.RawQuery = query.Encode()
query.Set("manifest", "1") // indicate we want a feed manifest back
feedUpdateURL.RawQuery = query.Encode()
// create the multihash update
resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
// submit the feed update request to Swarm
resp, err = http.Post(feedUpdateURL.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@ -131,24 +160,25 @@ func TestBzzFeedMultihash(t *testing.T) {
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
}
b, err = ioutil.ReadAll(resp.Body)
feedManifestAddressHex, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
rsrcResp := &storage.Address{}
err = json.Unmarshal(b, rsrcResp)
feedManifestAddress := &storage.Address{}
err = json.Unmarshal(feedManifestAddressHex, feedManifestAddress)
if err != nil {
t.Fatalf("data %s could not be unmarshaled: %v", b, err)
t.Fatalf("data %s could not be unmarshaled: %v", feedManifestAddressHex, err)
}
correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
if rsrcResp.Hex() != correctManifestAddrHex {
t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
correctManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
if feedManifestAddress.Hex() != correctManifestAddrHex {
t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestAddress.Hex())
}
// get bzz manifest transparent feed update resolve
testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testBzzUrl)
getBzzURL := fmt.Sprintf("%s/bzz:/%s", srv.URL, feedManifestAddress)
resp, err = http.Get(getBzzURL)
if err != nil {
t.Fatal(err)
}
@ -156,37 +186,30 @@ func TestBzzFeedMultihash(t *testing.T) {
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
}
b, err = ioutil.ReadAll(resp.Body)
retrievedData, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, []byte(databytes)) {
t.Fatalf("retrieved data mismatch, expected %x, got %x", databytes, b)
if !bytes.Equal(retrievedData, []byte(dataBytes)) {
t.Fatalf("retrieved data mismatch, expected %x, got %x", dataBytes, retrievedData)
}
}
// Test Swarm feeds using the raw update methods
func TestBzzFeed(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
signer, _ := newTestSigner()
defer srv.Close()
// data of update 1
update1Data := make([]byte, 666)
update1Data := testutil.RandomBytes(1, 666)
update1Timestamp := srv.CurrentTime
_, err := rand.Read(update1Data)
if err != nil {
t.Fatal(err)
}
//data for update 2
update2Data := []byte("foo")
topic, _ := feed.NewTopic("foo.eth", nil)
updateRequest := feed.NewFirstRequest(topic)
if err != nil {
t.Fatal(err)
}
updateRequest.SetData(update1Data)
if err := updateRequest.Sign(signer); err != nil {
@ -253,7 +276,8 @@ func TestBzzFeed(t *testing.T) {
t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex())
}
// get bzz manifest transparent feed update resolve
// take the chance to have bzz: crash on resolving a feed update that does not contain
// a swarm hash:
testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testBzzUrl)
if err != nil {
@ -261,7 +285,7 @@ func TestBzzFeed(t *testing.T) {
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK")
t.Fatal("Expected error status since feed update does not contain a Swarm hash. Received 200 OK")
}
_, err = ioutil.ReadAll(resp.Body)
if err != nil {
@ -450,7 +474,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
addr := [3]storage.Address{}
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
for i, mf := range testmanifest {
@ -688,7 +712,7 @@ func TestBzzTar(t *testing.T) {
}
func testBzzTar(encrypted bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"}
fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
@ -823,7 +847,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) {
}
func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// create a manifest with some data at the root path
@ -878,7 +902,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) {
}
func TestMethodsNotAllowed(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
databytes := "bar"
for _, c := range []struct {
@ -937,7 +961,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string
}
func TestGet(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
for _, testCase := range []struct {
@ -1020,7 +1044,7 @@ func TestGet(t *testing.T) {
}
func TestModify(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
swarmClient := swarm.NewClient(srv.URL)
@ -1121,7 +1145,7 @@ func TestMultiPartUpload(t *testing.T) {
// POST /bzz:/ Content-Type: multipart/form-data
verbose := false
// Setup Swarm
srv := testutil.NewTestSwarmServer(t, serverFunc, nil)
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
url := fmt.Sprintf("%s/bzz:/", srv.URL)
@ -1152,7 +1176,7 @@ func TestMultiPartUpload(t *testing.T) {
// TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver
func TestBzzGetFileWithResolver(t *testing.T) {
resolver := newTestResolveValidator("")
srv := testutil.NewTestSwarmServer(t, serverFunc, resolver)
srv := NewTestSwarmServer(t, serverFunc, resolver)
defer srv.Close()
fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"}
fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}

2
swarm/testutil/http.go → swarm/api/http/test_server.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package testutil
package http
import (
"io/ioutil"

71
swarm/bmt/bmt_test.go
View file

@ -18,10 +18,8 @@ package bmt
import (
"bytes"
crand "crypto/rand"
"encoding/binary"
"fmt"
"io"
"math/rand"
"sync"
"sync/atomic"
@ -29,6 +27,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
// the actual data length generated (could be longer than max datalength of the BMT)
@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) {
})
// run the tests
for _, x := range tests {
for i, x := range tests {
for segmentCount := x.from; segmentCount <= x.to; segmentCount++ {
for length := 1; length <= segmentCount*32; length++ {
t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) {
data := make([]byte, length)
if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF {
t.Fatal(err)
}
data := testutil.RandomBytes(i, length)
expected := x.expected(data)
actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data)
if !bytes.Equal(actual, expected) {
@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) {
// tests sequential write with entire max size written in one go
func TestSyncHasherCorrectness(t *testing.T) {
data := newData(BufferSize)
data := testutil.RandomBytes(1, BufferSize)
hasher := sha3.NewKeccak256
size := hasher().Size()
@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) {
// tests order-neutral concurrent writes with entire max size written in one go
func TestAsyncCorrectness(t *testing.T) {
data := newData(BufferSize)
data := testutil.RandomBytes(1, BufferSize)
hasher := sha3.NewKeccak256
size := hasher().Size()
whs := []whenHash{first, last, random}
@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) {
bmt := New(pool)
for i := 0; i < 100; i++ {
data := newData(BufferSize)
data := testutil.RandomBytes(1, BufferSize)
n := rand.Intn(bmt.Size())
err := testHasherCorrectness(bmt, hasher, data, n, segmentCount)
if err != nil {
@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) {
for i := 0; i < cycles; i++ {
go func() {
bmt := New(pool)
data := newData(BufferSize)
data := testutil.RandomBytes(1, BufferSize)
n := rand.Intn(bmt.Size())
errc <- testHasherCorrectness(bmt, hasher, data, n, 128)
}()
@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) {
defer pool.Drain(0)
n := count * 32
bmt := New(pool)
data := newData(n)
data := testutil.RandomBytes(1, n)
rbmt := NewRefHasher(hasher, count)
refHash := rbmt.Hash(data)
expHash := syncHash(bmt, nil, data)
@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) {
// benchmarks simple sha3 hash on chunks
func benchmarkSHA3(t *testing.B, n int) {
data := newData(n)
data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
h := hasher()
@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) {
func benchmarkBMTBaseline(t *testing.B, n int) {
hasher := sha3.NewKeccak256
hashSize := hasher().Size()
data := newData(hashSize)
data := testutil.RandomBytes(1, hashSize)
t.ReportAllocs()
t.ResetTimer()
@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) {
// benchmarks BMT Hasher
func benchmarkBMT(t *testing.B, n int) {
data := newData(n)
data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
bmt := New(pool)
@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) {
// benchmarks BMT hasher with asynchronous concurrent segment/section writes
func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
data := newData(n)
data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
bmt := New(pool).NewAsyncWriter(double)
idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
shuffle(len(idxs), func(i int, j int) {
rand.Shuffle(len(idxs), func(i int, j int) {
idxs[i], idxs[j] = idxs[j], idxs[i]
})
@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
// benchmarks 100 concurrent bmt hashes with pool capacity
func benchmarkPool(t *testing.B, poolsize, n int) {
data := newData(n)
data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
pool := NewTreePool(hasher, segmentCount, poolsize)
cycles := 100
@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) {
// benchmarks the reference hasher
func benchmarkRefHasher(t *testing.B, n int) {
data := newData(n)
data := testutil.RandomBytes(1, n)
hasher := sha3.NewKeccak256
rbmt := NewRefHasher(hasher, 128)
@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) {
}
}
func newData(bufferSize int) []byte {
data := make([]byte, bufferSize)
_, err := io.ReadFull(crand.Reader, data)
if err != nil {
panic(err.Error())
}
return data
}
// Hash hashes the data and the span using the bmt hasher
func syncHash(h *Hasher, span, data []byte) []byte {
h.ResetWithLength(span)
@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) {
section := data[i*secsize : end]
segments = append(segments, section)
}
shuffle(n, func(i int, j int) {
rand.Shuffle(n, func(i int, j int) {
idxs[i], idxs[j] = idxs[j], idxs[i]
})
return idxs, segments
@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s
}
return <-c
}
// this is also in swarm/network_test.go
// shuffle pseudo-randomizes the order of elements.
// n is the number of elements. Shuffle panics if n < 0.
// swap swaps the elements with indexes i and j.
func shuffle(n int, swap func(i, j int)) {
if n < 0 {
panic("invalid argument to Shuffle")
}
// Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
// Shuffle really ought not be called with n that doesn't fit in 32 bits.
// Not only will it take a very long time, but with 2³¹! possible permutations,
// there's no way that any PRNG can have a big enough internal state to
// generate even a minuscule percentage of the possible permutations.
// Nevertheless, the right API signature accepts an int n, so handle it as best we can.
i := n - 1
for ; i > 1<<31-1-1; i-- {
j := int(rand.Int63n(int64(i + 1)))
swap(i, j)
}
for ; i > 0; i-- {
j := int(rand.Int31n(int32(i + 1)))
swap(i, j)
}
}

163
swarm/fuse/swarmfs_test.go
View file

@ -20,20 +20,19 @@ package fuse
import (
"bytes"
"crypto/rand"
"flag"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"testing"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/testutil"
colorable "github.com/mattn/go-colorable"
)
@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) {
}
}
func getRandomBytes(size int) []byte {
contents := make([]byte, size)
rand.Read(contents)
return contents
}
func isDirEmpty(name string) bool {
f, err := os.Open(name)
if err != nil {
@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) {
dat.testMountDir = filepath.Join(dat.testDir, "testMountDir")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)}
dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)}
dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)}
dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)}
dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)}
dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)}
dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)}
dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)}
dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)}
dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)}
dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)}
dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)}
dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)}
dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)}
dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)}
dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)}
dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)}
dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)}
dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)}
dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)}
dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)}
dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)}
dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)}
dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount1")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount2")
dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount3")
dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount4")
dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5")
dat.testMountDir = filepath.Join(dat.testDir, "max-mount5")
dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) {
if err != nil {
t.Fatalf("Couldn't create upload dir 6: %v", err)
}
dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
testMountDir6 := filepath.Join(dat.testDir, "max-mount6")
err = os.MkdirAll(testMountDir6, 0777)
if err != nil {
@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) {
}
// mount a different hash in already mounted point
dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2")
if err3 != nil {
t.Fatalf("Error creating second upload dir: %v", err3)
@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) {
//we need to manually close the file before mount for this test
//but let's defer too in case of errors
defer d.Close()
_, err = d.Write(getRandomBytes(10))
_, err = d.Write(testutil.RandomBytes(1, 10))
if err != nil {
t.Fatalf("Couldn't write to file: %v", err)
}
@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "seek-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1")
dat.testMountDir = filepath.Join(dat.testDir, "create-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) {
}
defer d.Close()
log.Debug("Opened file")
contents := make([]byte, 11)
_, err = rand.Read(contents)
if err != nil {
t.Fatalf("Could not rand read contents %v", err)
}
contents := testutil.RandomBytes(1, 11)
log.Debug("content read")
_, err = d.Write(contents)
if err != nil {
@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount")
dat.files = make(map[string]fileInfo)
dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) {
}
defer d.Close()
log.Debug("File opened")
contents := make([]byte, 11)
_, err = rand.Read(contents)
if err != nil {
t.Fatalf("Error filling random bytes into byte array %v", err)
}
contents := testutil.RandomBytes(1, 11)
log.Debug("Content read")
_, err = d.Write(contents)
if err != nil {
@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool)
}
defer d.Close()
log.Debug("File opened")
contents := make([]byte, 11)
_, err = rand.Read(contents)
if err != nil {
t.Fatalf("Error writing random bytes to byte array: %v", err)
}
contents := testutil.RandomBytes(1, 11)
log.Debug("content read")
_, err = d.Write(contents)
if err != nil {
@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload")
dat.testMountDir = filepath.Join(dat.testDir, "remove-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload")
dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) {
}
defer d.Close()
log.Debug("file opened")
contents := make([]byte, 11)
_, err = rand.Read(contents)
if err != nil {
t.Fatalf("Error writing random bytes to byte array: %v", err)
}
contents := testutil.RandomBytes(1, 11)
log.Debug("content read")
_, err = d.Write(contents)
if err != nil {
@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload")
dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
dat.files = make(map[string]fileInfo)
dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount")
dat.files = make(map[string]fileInfo)
dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) {
dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload")
dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount")
dat.files = make(map[string]fileInfo)
dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)}
dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)}
dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)}
dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)}
dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)}
dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)}
dat, err = ta.uploadAndMount(dat, t)
if err != nil {
@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount")
dat.files = make(map[string]fileInfo)
line1 := make([]byte, 10)
_, err = rand.Read(line1)
if err != nil {
t.Fatalf("Error writing random bytes to byte array: %v", err)
}
line1 := testutil.RandomBytes(1, 10)
dat.files["1.txt"] = fileInfo{0700, 333, 444, line1}
@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) {
}
defer fd.Close()
log.Debug("file opened")
line2 := make([]byte, 5)
_, err = rand.Read(line2)
if err != nil {
t.Fatalf("Error writing random bytes to byte array: %v", err)
}
line2 := testutil.RandomBytes(1, 5)
log.Debug("line read")
_, err = fd.Seek(int64(len(line1)), 0)
if err != nil {

92
swarm/multihash/multihash.go
View file

@ -1,92 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package multihash
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
)
const (
defaultMultihashLength = 32
defaultMultihashTypeCode = 0x1b
)
var (
multihashTypeCode uint8
MultihashLength = defaultMultihashLength
)
func init() {
multihashTypeCode = defaultMultihashTypeCode
MultihashLength = defaultMultihashLength
}
// check if valid swarm multihash
func isSwarmMultihashType(code uint8) bool {
return code == multihashTypeCode
}
// GetMultihashLength returns the digest length of the provided multihash
// It will fail if the multihash is not a valid swarm mulithash
func GetMultihashLength(data []byte) (int, int, error) {
cursor := 0
typ, c := binary.Uvarint(data)
if c <= 0 {
return 0, 0, errors.New("unreadable hashtype field")
}
if !isSwarmMultihashType(uint8(typ)) {
return 0, 0, fmt.Errorf("hash code %x is not a swarm hashtype", typ)
}
cursor += c
hashlength, c := binary.Uvarint(data[cursor:])
if c <= 0 {
return 0, 0, errors.New("unreadable length field")
}
cursor += c
// we cheekily assume hashlength < maxint
inthashlength := int(hashlength)
if len(data[c:]) < inthashlength {
return 0, 0, errors.New("length mismatch")
}
return inthashlength, cursor, nil
}
// FromMulithash returns the digest portion of the multihash
// It will fail if the multihash is not a valid swarm multihash
func FromMultihash(data []byte) ([]byte, error) {
hashLength, _, err := GetMultihashLength(data)
if err != nil {
return nil, err
}
return data[len(data)-hashLength:], nil
}
// ToMulithash wraps the provided digest data with a swarm mulithash header
func ToMultihash(hashData []byte) []byte {
buf := bytes.NewBuffer(nil)
b := make([]byte, 8)
c := binary.PutUvarint(b, uint64(multihashTypeCode))
buf.Write(b[:c])
c = binary.PutUvarint(b, uint64(len(hashData)))
buf.Write(b[:c])
buf.Write(hashData)
return buf.Bytes()
}

53
swarm/multihash/multihash_test.go
View file

@ -1,53 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package multihash
import (
"bytes"
"math/rand"
"testing"
)
// parse multihash, and check that invalid multihashes fail
func TestCheckMultihash(t *testing.T) {
hashbytes := make([]byte, 32)
c, err := rand.Read(hashbytes)
if err != nil {
t.Fatal(err)
} else if c < 32 {
t.Fatal("short read")
}
expected := ToMultihash(hashbytes)
l, hl, _ := GetMultihashLength(expected)
if l != 32 {
t.Fatalf("expected length %d, got %d", 32, l)
} else if hl != 2 {
t.Fatalf("expected header length %d, got %d", 2, hl)
}
if _, _, err := GetMultihashLength(expected[1:]); err == nil {
t.Fatal("expected failure on corrupt header")
}
if _, _, err := GetMultihashLength(expected[:len(expected)-2]); err == nil {
t.Fatal("expected failure on short content")
}
dh, _ := FromMultihash(expected)
if !bytes.Equal(dh, hashbytes) {
t.Fatalf("expected content hash %x, got %x", hashbytes, dh)
}
}

153
swarm/network/kademlia.go
View file

@ -81,14 +81,15 @@ func NewKadParams() *KadParams {
// Kademlia is a table of live peers and a db of known peers (node records)
type Kademlia struct {
lock sync.RWMutex
*KadParams // Kademlia configuration parameters
base []byte // immutable baseaddress of the table
addrs *pot.Pot // pots container for known peer addresses
conns *pot.Pot // pots container for live peer connections
depth uint8 // stores the last current depth of saturation
nDepth int // stores the last neighbourhood depth
nDepthC chan int // returned by DepthC function to signal neighbourhood depth change
addrCountC chan int // returned by AddrCountC function to signal peer count change
*KadParams // Kademlia configuration parameters
base []byte // immutable baseaddress of the table
addrs *pot.Pot // pots container for known peer addresses
conns *pot.Pot // pots container for live peer connections
depth uint8 // stores the last current depth of saturation
nDepth int // stores the last neighbourhood depth
nDepthC chan int // returned by DepthC function to signal neighbourhood depth change
addrCountC chan int // returned by AddrCountC function to signal peer count change
Pof func(pot.Val, pot.Val, int) (int, bool) // function for calculating kademlia routing distance between two addresses
}
// NewKademlia creates a Kademlia table for base address addr
@ -103,6 +104,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia {
KadParams: params,
addrs: pot.NewPot(nil, 0),
conns: pot.NewPot(nil, 0),
Pof: pof,
}
}
@ -175,7 +177,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) {
k.lock.Lock()
defer k.lock.Unlock()
minsize := k.MinBinSize
depth := k.neighbourhoodDepth()
depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
// if there is a callable neighbour within the current proxBin, connect
// this makes sure nearest neighbour set is fully connected
var ppo int
@ -289,6 +291,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) {
// neighbourhood depth on each change.
// Not receiving from the returned channel will block On function
// when the neighbourhood depth is changed.
// TODO: Why is this exported, and if it should be; why can't we have more subscribers than one?
func (k *Kademlia) NeighbourhoodDepthC() <-chan int {
k.lock.Lock()
defer k.lock.Unlock()
@ -305,7 +308,7 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() {
// It provides signaling of neighbourhood depth change.
// This part of the code is sending new neighbourhood depth to nDepthC if that condition is met.
if k.nDepthC != nil {
nDepth := k.neighbourhoodDepth()
nDepth := depthForPot(k.conns, k.MinProxBinSize, k.base)
if nDepth != k.nDepth {
k.nDepth = nDepth
k.nDepthC <- nDepth
@ -361,7 +364,7 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con
var startPo int
var endPo int
kadDepth := k.neighbourhoodDepth()
kadDepth := depthForPot(k.conns, k.MinProxBinSize, k.base)
k.conns.EachBin(base, pof, o, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
if startPo > 0 && endPo != k.MaxProxDisplay {
@ -395,7 +398,7 @@ func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int, bool) bool) {
if len(base) == 0 {
base = k.base
}
depth := k.neighbourhoodDepth()
depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
k.conns.EachNeighbour(base, pof, func(val pot.Val, po int) bool {
if po > o {
return true
@ -417,7 +420,7 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool
if len(base) == 0 {
base = k.base
}
depth := k.neighbourhoodDepth()
depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
k.addrs.EachNeighbour(base, pof, func(val pot.Val, po int) bool {
if po > o {
return true
@ -426,21 +429,72 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool
})
}
// neighbourhoodDepth returns the proximity order that defines the distance of
func (k *Kademlia) NeighbourhoodDepth() (depth int) {
k.lock.RLock()
defer k.lock.RUnlock()
return depthForPot(k.conns, k.MinProxBinSize, k.base)
}
// depthForPot returns the proximity order that defines the distance of
// the nearest neighbour set with cardinality >= MinProxBinSize
// if there is altogether less than MinProxBinSize peers it returns 0
// caller must hold the lock
func (k *Kademlia) neighbourhoodDepth() (depth int) {
if k.conns.Size() < k.MinProxBinSize {
func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) {
if p.Size() <= minProxBinSize {
return 0
}
// total number of peers in iteration
var size int
// true if iteration has all prox peers
var b bool
// last po recorded in iteration
var lastPo int
f := func(v pot.Val, i int) bool {
// po == 256 means that addr is the pivot address(self)
if i == 256 {
return true
}
size++
depth = i
return size < k.MinProxBinSize
// this means we have all nn-peers.
// depth is by default set to the bin of the farthest nn-peer
if size == minProxBinSize {
b = true
depth = i
return true
}
// if there are empty bins between farthest nn and current node,
// the depth should recalculated to be
// the farthest of those empty bins
//
// 0 abac ccde
// 1 2a2a
// 2 589f <--- nearest non-nn
// ============ DEPTH 3 ===========
// 3 <--- don't count as empty bins
// 4 <--- don't count as empty bins
// 5 cbcb cdcd <---- furthest nn
// 6 a1a2 b3c4
if b && i < depth {
depth = i + 1
lastPo = i
return false
}
lastPo = i
return true
}
p.EachNeighbour(pivotAddr, pof, f)
// cover edge case where more than one farthest nn
// AND we only have nn-peers
if lastPo == depth {
depth = 0
}
k.conns.EachNeighbour(k.base, pof, f)
return depth
}
@ -500,7 +554,7 @@ func (k *Kademlia) string() string {
liverows := make([]string, k.MaxProxDisplay)
peersrows := make([]string, k.MaxProxDisplay)
depth := k.neighbourhoodDepth()
depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
rest := k.conns.Size()
k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
var rowlen int
@ -570,6 +624,7 @@ type PeerPot struct {
// as hexadecimal representations of the address.
// used for testing only
func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot {
// create a table of all nodes for health check
np := pot.NewPot(nil, 0)
for _, addr := range addrs {
@ -578,34 +633,47 @@ func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot {
ppmap := make(map[string]*PeerPot)
for i, a := range addrs {
pl := 256
prev := 256
// actual kademlia depth
depth := depthForPot(np, kadMinProxSize, a)
// upon entering a new iteration
// this will hold the value the po should be
// if it's one higher than the po in the last iteration
prevPo := 256
// all empty bins which are outside neighbourhood depth
var emptyBins []int
// all nn-peers
var nns [][]byte
np.EachNeighbour(addrs[i], pof, func(val pot.Val, po int) bool {
a := val.([]byte)
np.EachNeighbour(a, pof, func(val pot.Val, po int) bool {
addr := val.([]byte)
// po == 256 means that addr is the pivot address(self)
if po == 256 {
return true
}
if pl == 256 || pl == po {
nns = append(nns, a)
// iterate through the neighbours, going from the closest to the farthest
// we calculate the nearest neighbours that should be in the set
// depth in this case equates to:
// 1. Within all bins that are higher or equal than depth there are
// at least minProxBinSize peers connected
// 2. depth-1 bin is not empty
if po >= depth {
nns = append(nns, addr)
prevPo = depth - 1
return true
}
if pl == 256 && len(nns) >= kadMinProxSize {
pl = po
prev = po
for j := prevPo; j > po; j-- {
emptyBins = append(emptyBins, j)
}
if prev < pl {
for j := prev; j > po; j-- {
emptyBins = append(emptyBins, j)
}
}
prev = po - 1
prevPo = po - 1
return true
})
for j := prev; j >= 0; j-- {
emptyBins = append(emptyBins, j)
}
log.Trace(fmt.Sprintf("%x NNS: %s", addrs[i][:4], LogAddrs(nns)))
log.Trace(fmt.Sprintf("%x NNS: %s, emptyBins: %s", addrs[i][:4], LogAddrs(nns), logEmptyBins(emptyBins)))
ppmap[common.Bytes2Hex(a)] = &PeerPot{nns, emptyBins}
}
return ppmap
@ -620,7 +688,7 @@ func (k *Kademlia) saturation(n int) int {
prev++
return prev == po && size >= n
})
depth := k.neighbourhoodDepth()
depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
if depth < prev {
return depth
}
@ -633,8 +701,11 @@ func (k *Kademlia) full(emptyBins []int) (full bool) {
prev := 0
e := len(emptyBins)
ok := true
depth := k.neighbourhoodDepth()
depth := depthForPot(k.conns, k.MinProxBinSize, k.base)
k.conns.EachBin(k.base, pof, 0, func(po, _ int, _ func(func(val pot.Val, i int) bool) bool) bool {
if po >= depth {
return false
}
if prev == depth+1 {
return true
}

89
swarm/network/kademlia_test.go
View file

@ -25,6 +25,9 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/swarm/pot"
)
@ -73,6 +76,76 @@ func Register(k *Kademlia, regs ...string) {
}
}
// tests the validity of neighborhood depth calculations
//
// in particular, it tests that if there are one or more consecutive
// empty bins above the farthest "nearest neighbor-peer" then
// the depth should be set at the farthest of those empty bins
//
// TODO: Make test adapt to change in MinProxBinSize
func TestNeighbourhoodDepth(t *testing.T) {
baseAddressBytes := RandomAddr().OAddr
kad := NewKademlia(baseAddressBytes, NewKadParams())
baseAddress := pot.NewAddressFromBytes(baseAddressBytes)
closerAddress := pot.RandomAddressAt(baseAddress, 7)
closerPeer := newTestDiscoveryPeer(closerAddress, kad)
kad.On(closerPeer)
depth := kad.NeighbourhoodDepth()
if depth != 0 {
t.Fatalf("expected depth 0, was %d", depth)
}
sameAddress := pot.RandomAddressAt(baseAddress, 7)
samePeer := newTestDiscoveryPeer(sameAddress, kad)
kad.On(samePeer)
depth = kad.NeighbourhoodDepth()
if depth != 0 {
t.Fatalf("expected depth 0, was %d", depth)
}
midAddress := pot.RandomAddressAt(baseAddress, 4)
midPeer := newTestDiscoveryPeer(midAddress, kad)
kad.On(midPeer)
depth = kad.NeighbourhoodDepth()
if depth != 5 {
t.Fatalf("expected depth 5, was %d", depth)
}
kad.Off(midPeer)
depth = kad.NeighbourhoodDepth()
if depth != 0 {
t.Fatalf("expected depth 0, was %d", depth)
}
fartherAddress := pot.RandomAddressAt(baseAddress, 1)
fartherPeer := newTestDiscoveryPeer(fartherAddress, kad)
kad.On(fartherPeer)
depth = kad.NeighbourhoodDepth()
if depth != 2 {
t.Fatalf("expected depth 2, was %d", depth)
}
midSameAddress := pot.RandomAddressAt(baseAddress, 4)
midSamePeer := newTestDiscoveryPeer(midSameAddress, kad)
kad.Off(closerPeer)
kad.On(midPeer)
kad.On(midSamePeer)
depth = kad.NeighbourhoodDepth()
if depth != 2 {
t.Fatalf("expected depth 2, was %d", depth)
}
kad.Off(fartherPeer)
log.Trace(kad.string())
time.Sleep(time.Millisecond)
depth = kad.NeighbourhoodDepth()
if depth != 0 {
t.Fatalf("expected depth 0, was %d", depth)
}
}
func testSuggestPeer(k *Kademlia, expAddr string, expPo int, expWant bool) error {
addr, o, want := k.SuggestPeer()
if binStr(addr) != expAddr {
@ -376,7 +449,7 @@ func TestKademliaHiveString(t *testing.T) {
Register(k, "10000000", "10000001")
k.MaxProxDisplay = 8
h := k.String()
expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n========================================================================="
expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n========================================================================="
if expH[104:] != h[104:] {
t.Fatalf("incorrect hive output. expected %v, got %v", expH, h)
}
@ -644,3 +717,17 @@ func TestKademliaCase5(t *testing.T) {
"78fafa0809929a1279ece089a51d12457c2d8416dff859aeb2ccc24bb50df5ec", "1dd39b1257e745f147cbbc3cadd609ccd6207c41056dbc4254bba5d2527d3ee5", "5f61dd66d4d94aec8fcc3ce0e7885c7edf30c43143fa730e2841c5d28e3cd081", "8aa8b0472cb351d967e575ad05c4b9f393e76c4b01ef4b3a54aac5283b78abc9", "4502f385152a915b438a6726ce3ea9342e7a6db91a23c2f6bee83a885ed7eb82", "718677a504249db47525e959ef1784bed167e1c46f1e0275b9c7b588e28a3758", "7c54c6ed1f8376323896ed3a4e048866410de189e9599dd89bf312ca4adb96b5", "18e03bd3378126c09e799a497150da5c24c895aedc84b6f0dbae41fc4bac081a", "23db76ac9e6e58d9f5395ca78252513a7b4118b4155f8462d3d5eec62486cadc", "40ae0e8f065e96c7adb7fa39505136401f01780481e678d718b7f6dbb2c906ec", "c1539998b8bae19d339d6bbb691f4e9daeb0e86847545229e80fe0dffe716e92", "ed139d73a2699e205574c08722ca9f030ad2d866c662f1112a276b91421c3cb9", "5bdb19584b7a36d09ca689422ef7e6bb681b8f2558a6b2177a8f7c812f631022", "636c9de7fe234ffc15d67a504c69702c719f626c17461d3f2918e924cd9d69e2", "de4455413ff9335c440d52458c6544191bd58a16d85f700c1de53b62773064ea", "de1963310849527acabc7885b6e345a56406a8f23e35e436b6d9725e69a79a83", "a80a50a467f561210a114cba6c7fb1489ed43a14d61a9edd70e2eb15c31f074d", "7804f12b8d8e6e4b375b242058242068a3809385e05df0e64973cde805cf729c", "60f9aa320c02c6f2e6370aa740cf7cea38083fa95fca8c99552cda52935c1520", "d8da963602390f6c002c00ce62a84b514edfce9ebde035b277a957264bb54d21", "8463d93256e026fe436abad44697152b9a56ac8e06a0583d318e9571b83d073c", "9a3f78fcefb9a05e40a23de55f6153d7a8b9d973ede43a380bf46bb3b3847de1", "e3bb576f4b3760b9ca6bff59326f4ebfc4a669d263fb7d67ab9797adea54ed13", "4d5cdbd6dcca5bdf819a0fe8d175dc55cc96f088d37462acd5ea14bc6296bdbe", "5a0ed28de7b5258c727cb85447071c74c00a5fbba9e6bc0393bc51944d04ab2a", "61e4ddb479c283c638f4edec24353b6cc7a3a13b930824aad016b0996ca93c47", "7e3610868acf714836cafaaa7b8c009a9ac6e3a6d443e5586cf661530a204ee2", "d74b244d4345d2c86e30a097105e4fb133d53c578320285132a952cdaa64416e", "cfeed57d0f935bfab89e3f630a7c97e0b1605f0724d85a008bbfb92cb47863a8", "580837af95055670e20d494978f60c7f1458dc4b9e389fc7aa4982b2aca3bce3", "df55c0c49e6c8a83d82dfa1c307d3bf6a20e18721c80d8ec4f1f68dc0a137ced", "5f149c51ce581ba32a285439a806c063ced01ccd4211cd024e6a615b8f216f95", "1eb76b00aeb127b10dd1b7cd4c3edeb4d812b5a658f0feb13e85c4d2b7c6fe06", "7a56ba7c3fb7cbfb5561a46a75d95d7722096b45771ec16e6fa7bbfab0b35dfe", "4bae85ad88c28470f0015246d530adc0cd1778bdd5145c3c6b538ee50c4e04bd", "afd1892e2a7145c99ec0ebe9ded0d3fec21089b277a68d47f45961ec5e39e7e0", "953138885d7b36b0ef79e46030f8e61fd7037fbe5ce9e0a94d728e8c8d7eab86", "de761613ef305e4f628cb6bf97d7b7dc69a9d513dc233630792de97bcda777a6", "3f3087280063d09504c084bbf7fdf984347a72b50d097fd5b086ffabb5b3fb4c", "7d18a94bb1ebfdef4d3e454d2db8cb772f30ca57920dd1e402184a9e598581a0", "a7d6fbdc9126d9f10d10617f49fb9f5474ffe1b229f76b7dd27cebba30eccb5d", "fad0246303618353d1387ec10c09ee991eb6180697ed3470ed9a6b377695203d", "1cf66e09ea51ee5c23df26615a9e7420be2ac8063f28f60a3bc86020e94fe6f3", "8269cdaa153da7c358b0b940791af74d7c651cd4d3f5ed13acfe6d0f2c539e7f", "90d52eaaa60e74bf1c79106113f2599471a902d7b1c39ac1f55b20604f453c09", "9788fd0c09190a3f3d0541f68073a2f44c2fcc45bb97558a7c319f36c25a75b3", "10b68fc44157ecfdae238ee6c1ce0333f906ad04d1a4cb1505c8e35c3c87fbb0", "e5284117fdf3757920475c786e0004cb00ba0932163659a89b36651a01e57394", "403ad51d911e113dcd5f9ff58c94f6d278886a2a4da64c3ceca2083282c92de3",
)
}
func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer {
rw := &p2p.MsgPipeRW{}
p := p2p.NewPeer(enode.ID{}, "foo", []p2p.Cap{})
pp := protocols.NewPeer(p, rw, &protocols.Spec{})
bp := &BzzPeer{
Peer: pp,
BzzAddr: &BzzAddr{
OAddr: addr.Bytes(),
UAddr: []byte(fmt.Sprintf("%x", addr[:])),
},
}
return NewPeer(bp, kad)
}

4
swarm/network/protocol.go
View file

@ -44,7 +44,7 @@ const (
// BzzSpec is the spec of the generic swarm handshake
var BzzSpec = &protocols.Spec{
Name: "bzz",
Version: 7,
Version: 8,
MaxMsgSize: 10 * 1024 * 1024,
Messages: []interface{}{
HandshakeMsg{},
@ -54,7 +54,7 @@ var BzzSpec = &protocols.Spec{
// DiscoverySpec is the spec for the bzz discovery subprotocols
var DiscoverySpec = &protocols.Spec{
Name: "hive",
Version: 6,
Version: 8,
MaxMsgSize: 10 * 1024 * 1024,
Messages: []interface{}{
peersMsg{},

2
swarm/network/protocol_test.go
View file

@ -31,7 +31,7 @@ import (
)
const (
TestProtocolVersion = 7
TestProtocolVersion = 8
TestProtocolNetworkID = 3
)

114
swarm/network/simulation/events.go
View file

@ -20,16 +20,18 @@ import (
"context"
"sync"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations"
)
// PeerEvent is the type of the channel returned by Simulation.PeerEvents.
type PeerEvent struct {
// NodeID is the ID of node that the event is caught on.
NodeID enode.ID
// PeerID is the ID of the peer node that the event is caught on.
PeerID enode.ID
// Event is the event that is caught.
Event *p2p.PeerEvent
Event *simulations.Event
// Error is the error that may have happened during event watching.
Error error
}
@ -37,9 +39,13 @@ type PeerEvent struct {
// PeerEventsFilter defines a filter on PeerEvents to exclude messages with
// defined properties. Use PeerEventsFilter methods to set required options.
type PeerEventsFilter struct {
t *p2p.PeerEventType
protocol *string
msgCode *uint64
eventType simulations.EventType
connUp *bool
msgReceive *bool
protocol *string
msgCode *uint64
}
// NewPeerEventsFilter returns a new PeerEventsFilter instance.
@ -47,20 +53,48 @@ func NewPeerEventsFilter() *PeerEventsFilter {
return &PeerEventsFilter{}
}
// Type sets the filter to only one peer event type.
func (f *PeerEventsFilter) Type(t p2p.PeerEventType) *PeerEventsFilter {
f.t = &t
// Connect sets the filter to events when two nodes connect.
func (f *PeerEventsFilter) Connect() *PeerEventsFilter {
f.eventType = simulations.EventTypeConn
b := true
f.connUp = &b
return f
}
// Drop sets the filter to events when two nodes disconnect.
func (f *PeerEventsFilter) Drop() *PeerEventsFilter {
f.eventType = simulations.EventTypeConn
b := false
f.connUp = &b
return f
}
// ReceivedMessages sets the filter to only messages that are received.
func (f *PeerEventsFilter) ReceivedMessages() *PeerEventsFilter {
f.eventType = simulations.EventTypeMsg
b := true
f.msgReceive = &b
return f
}
// SentMessages sets the filter to only messages that are sent.
func (f *PeerEventsFilter) SentMessages() *PeerEventsFilter {
f.eventType = simulations.EventTypeMsg
b := false
f.msgReceive = &b
return f
}
// Protocol sets the filter to only one message protocol.
func (f *PeerEventsFilter) Protocol(p string) *PeerEventsFilter {
f.eventType = simulations.EventTypeMsg
f.protocol = &p
return f
}
// MsgCode sets the filter to only one msg code.
func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter {
f.eventType = simulations.EventTypeMsg
f.msgCode = &c
return f
}
@ -80,19 +114,8 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ...
go func(id enode.ID) {
defer s.shutdownWG.Done()
client, err := s.Net.GetNode(id).Client()
if err != nil {
subsWG.Done()
eventC <- PeerEvent{NodeID: id, Error: err}
return
}
events := make(chan *p2p.PeerEvent)
sub, err := client.Subscribe(ctx, "admin", events, "peerEvents")
if err != nil {
subsWG.Done()
eventC <- PeerEvent{NodeID: id, Error: err}
return
}
events := make(chan *simulations.Event)
sub := s.Net.Events().Subscribe(events)
defer sub.Unsubscribe()
subsWG.Done()
@ -110,28 +133,55 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ...
case <-s.Done():
return
case e := <-events:
// ignore control events
if e.Control {
continue
}
match := len(filters) == 0 // if there are no filters match all events
for _, f := range filters {
if f.t != nil && *f.t != e.Type {
continue
if f.eventType == simulations.EventTypeConn && e.Conn != nil {
if *f.connUp != e.Conn.Up {
continue
}
// all connection filter parameters matched, break the loop
match = true
break
}
if f.protocol != nil && *f.protocol != e.Protocol {
continue
if f.eventType == simulations.EventTypeMsg && e.Msg != nil {
if f.msgReceive != nil && *f.msgReceive != e.Msg.Received {
continue
}
if f.protocol != nil && *f.protocol != e.Msg.Protocol {
continue
}
if f.msgCode != nil && *f.msgCode != e.Msg.Code {
continue
}
// all message filter parameters matched, break the loop
match = true
break
}
if f.msgCode != nil && e.MsgCode != nil && *f.msgCode != *e.MsgCode {
continue
}
var peerID enode.ID
switch e.Type {
case simulations.EventTypeConn:
peerID = e.Conn.One
if peerID == id {
peerID = e.Conn.Other
}
case simulations.EventTypeMsg:
peerID = e.Msg.One
if peerID == id {
peerID = e.Msg.Other
}
// all filter parameters matched, break the loop
match = true
break
}
if match {
select {
case eventC <- PeerEvent{NodeID: id, Event: e}:
case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Event: e}:
case <-ctx.Done():
if err := ctx.Err(); err != nil {
select {
case eventC <- PeerEvent{NodeID: id, Error: err}:
case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Error: err}:
case <-s.Done():
}
}

17
swarm/network/simulation/example_test.go
View file

@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/network/simulation"
@ -34,6 +33,10 @@ import (
// BucketKeyKademlia key. This allows to use WaitTillHealthy to block until
// all nodes have the their Kadmlias healthy.
func ExampleSimulation_WaitTillHealthy() {
log.Error("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
return
sim := simulation.New(map[string]simulation.ServiceFunc{
"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
addr := network.NewAddr(ctx.Config.Node())
@ -87,7 +90,7 @@ func ExampleSimulation_PeerEvents() {
log.Error("peer event", "err", e.Error)
continue
}
log.Info("peer event", "node", e.NodeID, "peer", e.Event.Peer, "msgcode", e.Event.MsgCode)
log.Info("peer event", "node", e.NodeID, "peer", e.PeerID, "type", e.Event.Type)
}
}()
}
@ -100,7 +103,7 @@ func ExampleSimulation_PeerEvents_disconnections() {
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
@ -109,7 +112,7 @@ func ExampleSimulation_PeerEvents_disconnections() {
log.Error("peer drop", "err", d.Error)
continue
}
log.Warn("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Warn("peer drop", "node", d.NodeID, "peer", d.PeerID)
}
}()
}
@ -124,8 +127,8 @@ func ExampleSimulation_PeerEvents_multipleFilters() {
context.Background(),
sim.NodeIDs(),
// Watch when bzz messages 1 and 4 are received.
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(1),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(4),
simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(1),
simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(4),
)
go func() {
@ -134,7 +137,7 @@ func ExampleSimulation_PeerEvents_multipleFilters() {
log.Error("bzz message", "err", m.Error)
continue
}
log.Info("bzz message", "node", m.NodeID, "peer", m.Event.Peer)
log.Info("bzz message", "node", m.NodeID, "peer", m.PeerID)
}
}()
}

1
swarm/network/simulation/kademlia.go
View file

@ -33,6 +33,7 @@ var BucketKeyKademlia BucketKey = "kademlia"
// WaitTillHealthy is blocking until the health of all kademlias is true.
// If error is not nil, a map of kademlia that was found not healthy is returned.
// TODO: Check correctness since change in kademlia depth calculation logic
func (s *Simulation) WaitTillHealthy(ctx context.Context, kadMinProxSize int) (ill map[enode.ID]*network.Kademlia, err error) {
// Prepare PeerPot map for checking Kademlia health
var ppmap map[string]*network.PeerPot

1
swarm/network/simulation/kademlia_test.go
View file

@ -28,6 +28,7 @@ import (
)
func TestWaitTillHealthy(t *testing.T) {
sim := New(map[string]ServiceFunc{
"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
addr := network.NewAddr(ctx.Config.Node())

35
swarm/network/simulation/node_test.go
View file

@ -160,6 +160,41 @@ func TestAddNodeWithService(t *testing.T) {
}
}
func TestAddNodeMultipleServices(t *testing.T) {
sim := New(map[string]ServiceFunc{
"noop1": noopServiceFunc,
"noop2": noopService2Func,
})
defer sim.Close()
id, err := sim.AddNode()
if err != nil {
t.Fatal(err)
}
n := sim.Net.GetNode(id).Node.(*adapters.SimNode)
if n.Service("noop1") == nil {
t.Error("service noop1 not found on node")
}
if n.Service("noop2") == nil {
t.Error("service noop2 not found on node")
}
}
func TestAddNodeDuplicateServiceError(t *testing.T) {
sim := New(map[string]ServiceFunc{
"noop1": noopServiceFunc,
"noop2": noopServiceFunc,
})
defer sim.Close()
wantErr := "duplicate service: *simulation.noopService"
_, err := sim.AddNode()
if err.Error() != wantErr {
t.Errorf("got error %q, want %q", err, wantErr)
}
}
func TestAddNodes(t *testing.T) {
sim := New(noopServiceFuncMap)
defer sim.Close()

7
swarm/network/simulation/simulation.go
View file

@ -68,6 +68,10 @@ type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Se
// New creates a new Simulation instance with new
// simulations.Network initialized with provided services.
// Services map must have unique keys as service names and
// every ServiceFunc must return a node.Service of the unique type.
// This restriction is required by node.Node.Start() function
// which is used to start node.Service returned by ServiceFunc.
func New(services map[string]ServiceFunc) (s *Simulation) {
s = &Simulation{
buckets: make(map[enode.ID]*sync.Map),
@ -76,6 +80,9 @@ func New(services map[string]ServiceFunc) (s *Simulation) {
adapterServices := make(map[string]adapters.ServiceFunc, len(services))
for name, serviceFunc := range services {
// Scope this variables correctly
// as they will be in the adapterServices[name] function accessed later.
name, serviceFunc := name, serviceFunc
s.serviceNames = append(s.serviceNames, name)
adapterServices[name] = func(ctx *adapters.ServiceContext) (node.Service, error) {
b := new(sync.Map)

13
swarm/network/simulation/simulation_test.go
View file

@ -205,3 +205,16 @@ func (t *noopService) Start(server *p2p.Server) error {
func (t *noopService) Stop() error {
return nil
}
// a helper function for most basic noop service
// of a different type then noopService to test
// multiple services on one node.
func noopService2Func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
return new(noopService2), nil, nil
}
// noopService2 is the service that does not do anything
// but implements node.Service interface.
type noopService2 struct {
noopService
}

4
swarm/network/simulations/overlay.go
View file

@ -64,12 +64,12 @@ func init() {
type Simulation struct {
mtx sync.Mutex
stores map[enode.ID]*state.InmemoryStore
stores map[enode.ID]state.Store
}
func NewSimulation() *Simulation {
return &Simulation{
stores: make(map[enode.ID]*state.InmemoryStore),
stores: make(map[enode.ID]state.Store),
}
}

27
swarm/network/stream/common_test.go
View file

@ -18,7 +18,6 @@ package stream
import (
"context"
crand "crypto/rand"
"errors"
"flag"
"fmt"
@ -39,7 +38,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/pot"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
"github.com/ethereum/go-ethereum/swarm/testutil"
colorable "github.com/mattn/go-colorable"
)
@ -69,21 +68,6 @@ func init() {
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
func createGlobalStore() (string, *mockdb.GlobalStore, error) {
var globalStore *mockdb.GlobalStore
globalStoreDir, err := ioutil.TempDir("", "global.store")
if err != nil {
log.Error("Error initiating global store temp directory!", "err", err)
return "", nil, err
}
globalStore, err = mockdb.NewGlobalStore(globalStoreDir)
if err != nil {
log.Error("Error initiating global store!", "err", err)
return "", nil, err
}
return globalStoreDir, globalStore, nil
}
func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
// setup
addr := network.RandomAddr() // tested peers peer address
@ -114,7 +98,7 @@ func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest
delivery := NewDelivery(to, netStore)
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions)
streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil)
teardown := func() {
streamer.Close()
removeDataDir()
@ -230,12 +214,7 @@ func generateRandomFile() (string, error) {
//generate a random file size between minFileSize and maxFileSize
fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
b := make([]byte, fileSize*1024)
_, err := crand.Read(b)
if err != nil {
log.Error("Error generating random file.", "err", err)
return "", err
}
b := testutil.RandomBytes(1, fileSize*1024)
return string(b), nil
}

4
swarm/network/stream/delivery.go
View file

@ -169,7 +169,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req *
go func() {
chunk, err := d.chunkStore.Get(ctx, req.Addr)
if err != nil {
log.Warn("ChunkStore.Get can not retrieve chunk", "err", err)
log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err)
return
}
if req.SkipCheck {
@ -255,7 +255,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) (
}
sp = d.getPeer(id)
if sp == nil {
log.Warn("Delivery.RequestFromPeers: peer not found", "id", id)
//log.Warn("Delivery.RequestFromPeers: peer not found", "id", id)
return true
}
spID = &id

25
swarm/network/stream/delivery_test.go
View file

@ -19,9 +19,7 @@ package stream
import (
"bytes"
"context"
crand "crypto/rand"
"fmt"
"io"
"os"
"sync"
"testing"
@ -39,6 +37,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/network/simulation"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
//Tests initializing a retrieve request
@ -291,7 +290,7 @@ func TestRequestFromPeers(t *testing.T) {
Peer: protocolsPeer,
}, to)
to.On(peer)
r := NewRegistry(addr.ID(), delivery, nil, nil, nil)
r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil)
// an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished
sp := &Peer{
@ -332,7 +331,7 @@ func TestRequestFromPeersWithLightNode(t *testing.T) {
Peer: protocolsPeer,
}, to)
to.On(peer)
r := NewRegistry(addr.ID(), delivery, nil, nil, nil)
r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil)
// an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished
sp := &Peer{
Peer: protocolsPeer,
@ -454,6 +453,8 @@ func TestDeliveryFromNodes(t *testing.T) {
}
func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) {
t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
node := ctx.Config.Node()
@ -481,7 +482,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
SkipCheck: skipCheck,
Syncing: SyncingDisabled,
Retrieval: RetrievalEnabled,
})
}, nil)
bucket.Store(bucketKeyRegistry, r)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
@ -530,7 +531,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
//now we can actually upload a (random) file to the round-robin store
size := chunkCount * chunkSize
log.Debug("Storing data to file store")
fileHash, wait, err := roundRobinFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
// wait until all chunks stored
if err != nil {
return err
@ -566,13 +567,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal(d.Error)
}
}
@ -656,7 +657,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
Syncing: SyncingDisabled,
Retrieval: RetrievalDisabled,
SyncUpdateDelay: 0,
})
}, nil)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
bucket.Store(bucketKeyFileStore, fileStore)
@ -698,13 +699,13 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
b.Fatal(d.Error)
}
}
@ -719,7 +720,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
for i := 0; i < chunkCount; i++ {
// create actual size real chunks
ctx := context.TODO()
hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false)
hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false)
if err != nil {
b.Fatalf("expected no error. got %v", err)
}

15
swarm/network/stream/intervals_test.go
View file

@ -18,10 +18,8 @@ package stream
import (
"context"
crand "crypto/rand"
"encoding/binary"
"fmt"
"io"
"os"
"sync"
"testing"
@ -29,13 +27,13 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/network/simulation"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestIntervalsLive(t *testing.T) {
@ -54,6 +52,8 @@ func TestIntervalsLiveAndHistory(t *testing.T) {
}
func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
nodes := 2
chunkCount := dataChunkCount
externalStreamName := "externalStream"
@ -86,7 +86,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
Retrieval: RetrievalDisabled,
Syncing: SyncingRegisterOnly,
SkipCheck: skipCheck,
})
}, nil)
bucket.Store(bucketKeyRegistry, r)
r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) {
@ -130,7 +130,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
fileStore := item.(*storage.FileStore)
size := chunkCount * chunkSize
_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
_, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
if err != nil {
log.Error("Store error: %v", "err", err)
t.Fatal(err)
@ -154,7 +155,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
@ -165,7 +166,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal(d.Error)
}
}

3
swarm/network/stream/snapshot_retrieval_test.go
View file

@ -130,7 +130,7 @@ func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s no
Retrieval: RetrievalEnabled,
Syncing: SyncingAutoSubscribe,
SyncUpdateDelay: 3 * time.Second,
})
}, nil)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
bucket.Store(bucketKeyFileStore, fileStore)
@ -246,6 +246,7 @@ simulation's `action` function.
The snapshot should have 'streamer' in its service list.
*/
func runRetrievalTest(chunkCount int, nodeCount int) error {
sim := simulation.New(retrievalSimServiceMap)
defer sim.Close()

51
swarm/network/stream/snapshot_sync_test.go
View file

@ -17,9 +17,7 @@ package stream
import (
"context"
crand "crypto/rand"
"fmt"
"io"
"os"
"runtime"
"sync"
@ -29,7 +27,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
@ -38,7 +35,9 @@ import (
"github.com/ethereum/go-ethereum/swarm/pot"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
const MaxTimeout = 600
@ -168,7 +167,7 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic
Retrieval: RetrievalDisabled,
Syncing: SyncingAutoSubscribe,
SyncUpdateDelay: 3 * time.Second,
})
}, nil)
bucket.Store(bucketKeyRegistry, r)
@ -183,6 +182,8 @@ func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Servic
}
func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
sim := simulation.New(simServiceMap)
defer sim.Close()
@ -211,12 +212,12 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal("unexpected disconnect")
cancelSimRun()
}
@ -270,20 +271,9 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio
// File retrieval check is repeated until all uploaded files are retrieved from all nodes
// or until the timeout is reached.
var gDir string
var globalStore *mockdb.GlobalStore
var globalStore mock.GlobalStorer
if *useMockStore {
gDir, globalStore, err = createGlobalStore()
if err != nil {
return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
}
defer func() {
os.RemoveAll(gDir)
err := globalStore.Close()
if err != nil {
log.Error("Error closing global store! %v", "err", err)
}
}()
globalStore = mockmem.NewGlobalStore()
}
REPEAT:
for {
@ -341,6 +331,8 @@ assuming that the snapshot file identifies a healthy
kademlia network. The snapshot should have 'streamer' in its service list.
*/
func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) error {
t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
n := ctx.Config.Node()
@ -362,7 +354,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
Retrieval: RetrievalDisabled,
Syncing: SyncingRegisterOnly,
})
}, nil)
bucket.Store(bucketKeyRegistry, r)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
@ -403,12 +395,12 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal("unexpected disconnect")
cancelSimRun()
}
@ -429,7 +421,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
var subscriptionCount int
filter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(4)
filter := simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("stream").MsgCode(4)
eventC := sim.PeerEvents(ctx, nodeIDs, filter)
for j, node := range nodeIDs {
@ -478,14 +470,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int)
return err
}
var gDir string
var globalStore *mockdb.GlobalStore
var globalStore mock.GlobalStorer
if *useMockStore {
gDir, globalStore, err = createGlobalStore()
if err != nil {
return fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
}
defer os.RemoveAll(gDir)
globalStore = mockmem.NewGlobalStore()
}
// File retrieval check is repeated until all uploaded files are retrieved from all nodes
// or until the timeout is reached.
@ -603,7 +590,7 @@ func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, lstore *storage.Lo
size := chunkSize
var rootAddrs []storage.Address
for i := 0; i < chunkCount; i++ {
rk, wait, err := fileStore.Store(context.TODO(), io.LimitReader(crand.Reader, int64(size)), int64(size), false)
rk, wait, err := fileStore.Store(context.TODO(), testutil.RandomReader(i, size), int64(size), false)
if err != nil {
return nil, err
}

76
swarm/network/stream/stream.go
View file

@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"math"
"reflect"
"sync"
"time"
@ -87,6 +88,9 @@ type Registry struct {
intervalsStore state.Store
autoRetrieval bool //automatically subscribe to retrieve request stream
maxPeerServers int
spec *protocols.Spec //this protocol's spec
balance protocols.Balance //implements protocols.Balance, for accounting
prices protocols.Prices //implements protocols.Prices, provides prices to accounting
}
// RegistryOptions holds optional values for NewRegistry constructor.
@ -99,7 +103,7 @@ type RegistryOptions struct {
}
// NewRegistry is Streamer constructor
func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry {
func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions, balance protocols.Balance) *Registry {
if options == nil {
options = &RegistryOptions{}
}
@ -119,7 +123,10 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
intervalsStore: intervalsStore,
autoRetrieval: retrieval,
maxPeerServers: options.MaxPeerServers,
balance: balance,
}
streamer.setupSpec()
streamer.api = NewAPI(streamer)
delivery.getPeer = streamer.getPeer
@ -228,6 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy
return streamer
}
//we need to construct a spec instance per node instance
func (r *Registry) setupSpec() {
//first create the "bare" spec
r.createSpec()
//if balance is nil, this node has been started without swap support (swapEnabled flag is false)
if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() {
//swap is enabled, so setup the hook
r.spec.Hook = protocols.NewAccounting(r.balance, r.prices)
}
}
// RegisterClient registers an incoming streamer constructor
func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) {
r.clientMu.Lock()
@ -492,7 +510,7 @@ func (r *Registry) updateSyncing() {
}
func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peer := protocols.NewPeer(p, rw, Spec)
peer := protocols.NewPeer(p, rw, r.spec)
bp := network.NewBzzPeer(peer)
np := network.NewPeer(bp, r.delivery.kad)
r.delivery.kad.On(np)
@ -716,35 +734,43 @@ func (c *clientParams) clientCreated() {
close(c.clientCreatedC)
}
// Spec is the spec of the streamer protocol
var Spec = &protocols.Spec{
Name: "stream",
Version: 8,
MaxMsgSize: 10 * 1024 * 1024,
Messages: []interface{}{
UnsubscribeMsg{},
OfferedHashesMsg{},
WantedHashesMsg{},
TakeoverProofMsg{},
SubscribeMsg{},
RetrieveRequestMsg{},
ChunkDeliveryMsgRetrieval{},
SubscribeErrorMsg{},
RequestSubscriptionMsg{},
QuitMsg{},
ChunkDeliveryMsgSyncing{},
},
//GetSpec returns the streamer spec to callers
//This used to be a global variable but for simulations with
//multiple nodes its fields (notably the Hook) would be overwritten
func (r *Registry) GetSpec() *protocols.Spec {
return r.spec
}
func (r *Registry) createSpec() {
// Spec is the spec of the streamer protocol
var spec = &protocols.Spec{
Name: "stream",
Version: 8,
MaxMsgSize: 10 * 1024 * 1024,
Messages: []interface{}{
UnsubscribeMsg{},
OfferedHashesMsg{},
WantedHashesMsg{},
TakeoverProofMsg{},
SubscribeMsg{},
RetrieveRequestMsg{},
ChunkDeliveryMsgRetrieval{},
SubscribeErrorMsg{},
RequestSubscriptionMsg{},
QuitMsg{},
ChunkDeliveryMsgSyncing{},
},
}
r.spec = spec
}
func (r *Registry) Protocols() []p2p.Protocol {
return []p2p.Protocol{
{
Name: Spec.Name,
Version: Spec.Version,
Length: Spec.Length(),
Name: r.spec.Name,
Version: r.spec.Version,
Length: r.spec.Length(),
Run: r.runProtocol,
// NodeInfo: ,
// PeerInfo: ,
},
}
}

202
swarm/network/stream/syncer_test.go
View file

@ -18,9 +18,7 @@ package stream
import (
"context"
crand "crypto/rand"
"fmt"
"io"
"io/ioutil"
"math"
"os"
@ -30,7 +28,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/swarm/log"
@ -38,7 +35,9 @@ import (
"github.com/ethereum/go-ethereum/swarm/network/simulation"
"github.com/ethereum/go-ethereum/swarm/state"
"github.com/ethereum/go-ethereum/swarm/storage"
mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
const dataChunkCount = 200
@ -50,7 +49,7 @@ func TestSyncerSimulation(t *testing.T) {
testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1)
}
func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
address := common.BytesToAddress(id.Bytes())
mockStore := globalStore.NewNodeStore(address)
params := storage.NewDefaultLocalStoreParams()
@ -69,11 +68,12 @@ func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network
}
func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) {
t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted")
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
var store storage.ChunkStore
var globalStore *mockdb.GlobalStore
var gDir, datadir string
var datadir string
node := ctx.Config.Node()
addr := network.NewAddr(node)
@ -81,11 +81,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
addr.OAddr[0] = byte(0)
if *useMockStore {
gDir, globalStore, err = createGlobalStore()
if err != nil {
return nil, nil, fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
}
store, datadir, err = createMockStore(globalStore, node.ID(), addr)
store, datadir, err = createMockStore(mockmem.NewGlobalStore(), node.ID(), addr)
} else {
store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
}
@ -96,13 +92,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
cleanup = func() {
store.Close()
os.RemoveAll(datadir)
if *useMockStore {
err := globalStore.Close()
if err != nil {
log.Error("Error closing global store! %v", "err", err)
}
os.RemoveAll(gDir)
}
}
localStore := store.(*storage.LocalStore)
netStore, err := storage.NewNetStore(localStore, nil)
@ -120,7 +109,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
Retrieval: RetrievalDisabled,
Syncing: SyncingAutoSubscribe,
SkipCheck: skipCheck,
})
}, nil)
fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
bucket.Store(bucketKeyFileStore, fileStore)
@ -152,13 +141,13 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
t.Fatal(d.Error)
}
}
@ -183,7 +172,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
}
fileStore := item.(*storage.FileStore)
size := chunkCount * chunkSize
_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
_, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false)
if err != nil {
t.Fatal(err.Error())
}
@ -245,3 +234,170 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
t.Fatal(result.Error)
}
}
//TestSameVersionID just checks that if the version is not changed,
//then streamer peers see each other
func TestSameVersionID(t *testing.T) {
//test version ID
v := uint(1)
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
var store storage.ChunkStore
var datadir string
node := ctx.Config.Node()
addr := network.NewAddr(node)
store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
if err != nil {
return nil, nil, err
}
bucket.Store(bucketKeyStore, store)
cleanup = func() {
store.Close()
os.RemoveAll(datadir)
}
localStore := store.(*storage.LocalStore)
netStore, err := storage.NewNetStore(localStore, nil)
if err != nil {
return nil, nil, err
}
bucket.Store(bucketKeyDB, netStore)
kad := network.NewKademlia(addr.Over(), network.NewKadParams())
delivery := NewDelivery(kad, netStore)
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
bucket.Store(bucketKeyDelivery, delivery)
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
Retrieval: RetrievalDisabled,
Syncing: SyncingAutoSubscribe,
}, nil)
//assign to each node the same version ID
r.spec.Version = v
bucket.Store(bucketKeyRegistry, r)
return r, cleanup, nil
},
})
defer sim.Close()
//connect just two nodes
log.Info("Adding nodes to simulation")
_, err := sim.AddNodesAndConnectChain(2)
if err != nil {
t.Fatal(err)
}
log.Info("Starting simulation")
ctx := context.Background()
//make sure they have time to connect
time.Sleep(200 * time.Millisecond)
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
//get the pivot node's filestore
nodes := sim.UpNodeIDs()
item, ok := sim.NodeItem(nodes[0], bucketKeyRegistry)
if !ok {
return fmt.Errorf("No filestore")
}
registry := item.(*Registry)
//the peers should connect, thus getting the peer should not return nil
if registry.getPeer(nodes[1]) == nil {
t.Fatal("Expected the peer to not be nil, but it is")
}
return nil
})
if result.Error != nil {
t.Fatal(result.Error)
}
log.Info("Simulation ended")
}
//TestDifferentVersionID proves that if the streamer protocol version doesn't match,
//then the peers are not connected at streamer level
func TestDifferentVersionID(t *testing.T) {
//create a variable to hold the version ID
v := uint(0)
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
var store storage.ChunkStore
var datadir string
node := ctx.Config.Node()
addr := network.NewAddr(node)
store, datadir, err = createTestLocalStorageForID(node.ID(), addr)
if err != nil {
return nil, nil, err
}
bucket.Store(bucketKeyStore, store)
cleanup = func() {
store.Close()
os.RemoveAll(datadir)
}
localStore := store.(*storage.LocalStore)
netStore, err := storage.NewNetStore(localStore, nil)
if err != nil {
return nil, nil, err
}
bucket.Store(bucketKeyDB, netStore)
kad := network.NewKademlia(addr.Over(), network.NewKadParams())
delivery := NewDelivery(kad, netStore)
netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
bucket.Store(bucketKeyDelivery, delivery)
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
Retrieval: RetrievalDisabled,
Syncing: SyncingAutoSubscribe,
}, nil)
//increase the version ID for each node
v++
r.spec.Version = v
bucket.Store(bucketKeyRegistry, r)
return r, cleanup, nil
},
})
defer sim.Close()
//connect the nodes
log.Info("Adding nodes to simulation")
_, err := sim.AddNodesAndConnectChain(2)
if err != nil {
t.Fatal(err)
}
log.Info("Starting simulation")
ctx := context.Background()
//make sure they have time to connect
time.Sleep(200 * time.Millisecond)
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
//get the pivot node's filestore
nodes := sim.UpNodeIDs()
item, ok := sim.NodeItem(nodes[0], bucketKeyRegistry)
if !ok {
return fmt.Errorf("No filestore")
}
registry := item.(*Registry)
//getting the other peer should fail due to the different version numbers
if registry.getPeer(nodes[1]) != nil {
t.Fatal("Expected the peer to be nil, but it is not")
}
return nil
})
if result.Error != nil {
t.Fatal(result.Error)
}
log.Info("Simulation ended")
}

Some files were not shown because too many files have changed in this diff Show more