From a1eb9c7d13240fd250866219a502d0cdc9924e06 Mon Sep 17 00:00:00 2001 From: Giulio M Date: Wed, 8 Aug 2018 09:33:06 +0200 Subject: [PATCH 001/126] swarm/api/http: fixed list leaf links (#17342) --- swarm/api/http/server_test.go | 6 +++--- swarm/api/http/templates.go | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 3ac60596b5..7934e37eba 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -576,7 +576,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { pageFragments: []string{ fmt.Sprintf("Swarm index of bzz:/%s/a/", ref), `b/`, - `a`, + fmt.Sprintf(`a`, ref), }, }, { @@ -584,8 +584,8 @@ func testBzzGetPath(encrypted bool, t *testing.T) { json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`, pageFragments: []string{ fmt.Sprintf("Swarm index of bzz:/%s/a/b/", ref), - `b`, - `c`, + fmt.Sprintf(`b`, ref), + fmt.Sprintf(`c`, ref), }, }, { diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go index 1cd42ca371..986f5f8873 100644 --- a/swarm/api/http/templates.go +++ b/swarm/api/http/templates.go @@ -18,6 +18,7 @@ package http import ( "encoding/hex" + "fmt" "html/template" "path" @@ -45,7 +46,10 @@ func init() { { templateName: "bzz-list", partial: bzzList, - funcs: template.FuncMap{"basename": path.Base}, + funcs: template.FuncMap{ + "basename": path.Base, + "leaflink": leafLink, + }, }, { templateName: "landing-page", @@ -62,6 +66,10 @@ func init() { faviconBytes = bytes } +func leafLink(URI api.URI, manifestEntry api.ManifestEntry) string { + return fmt.Sprintf("/bzz:/%s/%s", URI.Addr, manifestEntry.Path) +} + const bzzList = `{{ define "content" }}

Swarm index of {{ .URI }}

@@ -83,10 +91,11 @@ const bzzList = `{{ define "content" }} DIR - - {{ end }} {{ range .List.Entries }} + {{ end }} + {{ range .List.Entries }} - {{ basename .Path }} + {{ basename .Path }} {{ .ContentType }} {{ .Size }} From 8051a0768a2af6c36b04ffa6fb225a45986d9b89 Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Wed, 8 Aug 2018 21:08:40 +0800 Subject: [PATCH 002/126] trie: fix comment typo (#17350) --- trie/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/database.go b/trie/database.go index 8675b9f0a0..7df45fe2df 100644 --- a/trie/database.go +++ b/trie/database.go @@ -51,7 +51,7 @@ const secureKeyLength = 11 + 32 // DatabaseReader wraps the Get and Has method of a backing store for the trie. type DatabaseReader interface { - // Get retrieves the value associated with key form the database. + // Get retrieves the value associated with key from the database. Get(key []byte) (value []byte, err error) // Has retrieves whether a key is present in the database. From abbb219933504a2aa739353a1cb6157cdd0cf145 Mon Sep 17 00:00:00 2001 From: Jay Date: Thu, 9 Aug 2018 14:56:35 +0900 Subject: [PATCH 003/126] rpc: fix a subscription name (#17345) --- rpc/client_example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go index 8276a9eadd..9c21c12d5d 100644 --- a/rpc/client_example_test.go +++ b/rpc/client_example_test.go @@ -66,7 +66,7 @@ func subscribeBlocks(client *rpc.Client, subch chan Block) { defer cancel() // Subscribe to new blocks. - sub, err := client.EthSubscribe(ctx, subch, "newBlocks") + sub, err := client.EthSubscribe(ctx, subch, "newHeads") if err != nil { fmt.Println("subscribe error:", err) return From 834057592f68eecc45382794c0fed96e594e14d1 Mon Sep 17 00:00:00 2001 From: libotony Date: Thu, 9 Aug 2018 15:03:42 +0800 Subject: [PATCH 004/126] p2p/discv5: fix negative index after uint convert to int (#17274) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index 9b0bd0c80a..4c39c05533 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -1228,7 +1228,7 @@ func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) { if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash { return nil, errors.New("topic hash mismatch") } - if data.Idx < 0 || int(data.Idx) >= len(data.Topics) { + if int(data.Idx) < 0 || int(data.Idx) >= len(data.Topics) { return nil, errors.New("topic index out of range") } return pongpkt.data.(*pong), nil From 11bbc660823246b9fc25e4b994121e30a9f17306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 8 Aug 2018 17:16:38 +0300 Subject: [PATCH 005/126] eth, trie: fix tracer GC which accidentally pruned the metaroot --- eth/api_tracer.go | 8 ++++++-- trie/database.go | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 623e5ed1bd..722e2a6e32 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -297,7 +297,9 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl database.TrieDB().Reference(root, common.Hash{}) } // Dereference all past tries we ourselves are done working with - database.TrieDB().Dereference(proot) + if proot != (common.Hash{}) { + database.TrieDB().Dereference(proot) + } proot = root // TODO(karalabe): Do we need the preimages? Won't they accumulate too much? @@ -526,7 +528,9 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* return nil, err } database.TrieDB().Reference(root, common.Hash{}) - database.TrieDB().Dereference(proot) + if proot != (common.Hash{}) { + database.TrieDB().Dereference(proot) + } proot = root } nodes, imgs := database.TrieDB().Size() diff --git a/trie/database.go b/trie/database.go index 7df45fe2df..d0691b637e 100644 --- a/trie/database.go +++ b/trie/database.go @@ -431,6 +431,11 @@ func (db *Database) reference(child common.Hash, parent common.Hash) { // Dereference removes an existing reference from a root node. func (db *Database) Dereference(root common.Hash) { + // Sanity check to ensure that the meta-root is not removed + if root == (common.Hash{}) { + log.Error("Attempted to dereference the trie cache meta root") + return + } db.lock.Lock() defer db.lock.Unlock() From 7b5c3758250ffc78c7a5ce14c1b736a38d548423 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 9 Aug 2018 11:37:00 +0200 Subject: [PATCH 006/126] cmd/swarm: remove shadow err (#17360) --- cmd/swarm/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index ff085fd94f..cda8c41c32 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -132,7 +132,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) { log.Debug(printConfig(config)) } -//override the current config with whatever is in the config file, if a config file has been provided +//configFileOverride overrides the current config with the config file, if a config file has been provided func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) { var err error @@ -142,7 +142,8 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" { utils.Fatalf("Config file flag provided with invalid file path") } - f, err := os.Open(filepath) + var f *os.File + f, err = os.Open(filepath) if err != nil { return nil, err } From d3e4c2dcb00eb61c32dd3a9b94a317727c2449a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 9 Aug 2018 16:14:30 +0200 Subject: [PATCH 007/126] cmd/swarm: disable TestCLISwarmFs fuse test on darwin (#17340) --- cmd/swarm/fs_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index a2b730bd5d..4f38b094bb 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// +build linux darwin freebsd +// +build linux freebsd package main @@ -43,12 +43,12 @@ type testFile struct { } // TestCLISwarmFs is a high-level test of swarmfs +// +// This test fails on travis for macOS as this executable exits with code 1 +// and without any log messages in the log: +// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse. +// This is the reason for this file not being built on darwin architecture. func TestCLISwarmFs(t *testing.T) { - // This test fails on travis as this executable exits with code 1 - // and without any log messages in the log. - // /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse - t.Skip() - cluster := newTestCluster(t, 3) defer cluster.Shutdown() From 3bcb501c8fefeec1bc8aac26686460cae86f2ccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 9 Aug 2018 16:15:07 +0200 Subject: [PATCH 008/126] swarm/api: close tar writer in GetDirectoryTar to flush and clean (#17339) --- swarm/api/api.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/swarm/api/api.go b/swarm/api/api.go index ad4bd7dcb1..b418c45e1c 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -525,6 +525,10 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err return nil }) + // close tar writer before closing pipew + // to flush remaining data to pipew + // regardless of error value + tw.Close() if err != nil { apiGetTarFail.Inc(1) pipew.CloseWithError(err) From 45eaef24319897f5b1679c9d1aa7d88702cce905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 9 Aug 2018 16:15:59 +0200 Subject: [PATCH 009/126] cmd/swarm: solve rare cases of using the same random port in tests (#17352) --- cmd/swarm/config_test.go | 14 ++++ cmd/swarm/run_test.go | 138 +++++++++++++++++++++++++++++++++------ 2 files changed, 132 insertions(+), 20 deletions(-) diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index d5011e3a70..02198f878e 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "os" "os/exec" "testing" @@ -559,3 +560,16 @@ func TestValidateConfig(t *testing.T) { } } } + +func assignTCPPort() (string, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + l.Close() + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return "", err + } + return port, nil +} diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index a70c4686dd..90d3c98ba6 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -17,12 +17,15 @@ package main import ( + "context" "fmt" "io/ioutil" "net" "os" "path/filepath" "runtime" + "sync" + "syscall" "testing" "time" @@ -218,14 +221,12 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { } // assign ports - httpPort, err := assignTCPPort() - if err != nil { - t.Fatal(err) - } - p2pPort, err := assignTCPPort() + ports, err := getAvailableTCPPorts(2) if err != nil { t.Fatal(err) } + p2pPort := ports[0] + httpPort := ports[1] // start the node node.Cmd = runSwarm(t, @@ -246,6 +247,17 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { } }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // ensure that all ports have active listeners + // so that the next node will not get the same + // when calling getAvailableTCPPorts + err = waitTCPPorts(ctx, ports...) + if err != nil { + t.Fatal(err) + } + // wait for the node to start for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { node.Client, err = rpc.Dial(conf.IPCEndpoint()) @@ -280,14 +292,12 @@ func newTestNode(t *testing.T, dir string) *testNode { node := &testNode{Dir: dir} // assign ports - httpPort, err := assignTCPPort() - if err != nil { - t.Fatal(err) - } - p2pPort, err := assignTCPPort() + ports, err := getAvailableTCPPorts(2) if err != nil { t.Fatal(err) } + p2pPort := ports[0] + httpPort := ports[1] // start the node node.Cmd = runSwarm(t, @@ -308,6 +318,17 @@ func newTestNode(t *testing.T, dir string) *testNode { } }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // ensure that all ports have active listeners + // so that the next node will not get the same + // when calling getAvailableTCPPorts + err = waitTCPPorts(ctx, ports...) + if err != nil { + t.Fatal(err) + } + // wait for the node to start for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { node.Client, err = rpc.Dial(conf.IPCEndpoint()) @@ -343,15 +364,92 @@ func (n *testNode) Shutdown() { } } -func assignTCPPort() (string, error) { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return "", err +// getAvailableTCPPorts returns a set of ports that +// nothing is listening on at the time. +// +// Function assignTCPPort cannot be called in sequence +// and guardantee that the same port will be returned in +// different calls as the listener is closed within the function, +// not after all listeners are started and selected unique +// available ports. +func getAvailableTCPPorts(count int) (ports []string, err error) { + for i := 0; i < count; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + // defer close in the loop to be sure the same port will not + // be selected in the next iteration + defer l.Close() + + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return nil, err + } + ports = append(ports, port) + } + return ports, nil +} + +// waitTCPPorts blocks until tcp connections can be +// established on all provided ports. It runs all +// ports dialers in parallel, and returns the first +// encountered error. +// See waitTCPPort also. +func waitTCPPorts(ctx context.Context, ports ...string) error { + var err error + // mu locks err variable that is assigned in + // other goroutines + var mu sync.Mutex + + // cancel is canceling all goroutines + // when the firs error is returned + // to prevent unnecessary waiting + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var wg sync.WaitGroup + for _, port := range ports { + wg.Add(1) + go func(port string) { + defer wg.Done() + + e := waitTCPPort(ctx, port) + + mu.Lock() + defer mu.Unlock() + if e != nil && err == nil { + err = e + cancel() + } + }(port) + } + wg.Wait() + + return err +} + +// waitTCPPort blocks until tcp connection can be established +// ona provided port. It has a 3 minute timeout as maximum, +// to prevent long waiting, but it can be shortened with +// a provided context instance. Dialer has a 10 second timeout +// in every iteration, and connection refused error will be +// retried in 100 milliseconds periods. +func waitTCPPort(ctx context.Context, port string) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + + for { + c, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, "tcp", "127.0.0.1:"+port) + if err != nil { + if operr, ok := err.(*net.OpError); ok { + if syserr, ok := operr.Err.(*os.SyscallError); ok && syserr.Err == syscall.ECONNREFUSED { + time.Sleep(100 * time.Millisecond) + continue + } + } + return err + } + return c.Close() } - l.Close() - _, port, err := net.SplitHostPort(l.Addr().String()) - if err != nil { - return "", err - } - return port, nil } From f0998415ba9a73f0add32f9b5aed2aec98b9a7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 8 Aug 2018 12:15:08 +0300 Subject: [PATCH 010/126] cmd, consensus/ethash, eth: miner push notifications --- cmd/geth/main.go | 3 +- cmd/geth/usage.go | 1 + cmd/utils/flags.go | 14 +++- consensus/ethash/algorithm_test.go | 2 +- consensus/ethash/consensus.go | 2 +- consensus/ethash/ethash.go | 18 +++-- consensus/ethash/ethash_test.go | 43 +++++------ consensus/ethash/sealer.go | 90 ++++++++++++++-------- consensus/ethash/sealer_test.go | 115 +++++++++++++++++++++++++++++ eth/backend.go | 8 +- eth/config.go | 1 + les/backend.go | 2 +- 12 files changed, 226 insertions(+), 73 deletions(-) create mode 100644 consensus/ethash/sealer_test.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 77ef6afe24..d556ad92c3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -98,8 +98,9 @@ var ( utils.MaxPendingPeersFlag, utils.EtherbaseFlag, utils.GasPriceFlag, - utils.MinerThreadsFlag, utils.MiningEnabledFlag, + utils.MinerThreadsFlag, + utils.MinerNotifyFlag, utils.TargetGasLimitFlag, utils.NATFlag, utils.NoDiscoverFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 6a12a66cc2..9d63c68f7f 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -185,6 +185,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.MiningEnabledFlag, utils.MinerThreadsFlag, + utils.MinerNotifyFlag, utils.EtherbaseFlag, utils.TargetGasLimitFlag, utils.GasPriceFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 522ad06b61..d6142f246c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -24,7 +24,6 @@ import ( "math/big" "os" "path/filepath" - "runtime" "strconv" "strings" "time" @@ -318,9 +317,13 @@ var ( Usage: "Enable mining", } MinerThreadsFlag = cli.IntFlag{ - Name: "minerthreads", + Name: "miner.threads", Usage: "Number of CPU threads to use for mining", - Value: runtime.NumCPU(), + Value: 0, + } + MinerNotifyFlag = cli.StringFlag{ + Name: "miner.notify", + Usage: "Comma separated HTTP URL list to notify of new work packages", } TargetGasLimitFlag = cli.Uint64Flag{ Name: "targetgaslimit", @@ -1093,6 +1096,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(MinerThreadsFlag.Name) { cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name) } + if ctx.GlobalIsSet(MinerNotifyFlag.Name) { + cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") + } if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } @@ -1293,7 +1299,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir), DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem, DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk, - }) + }, nil) } } if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index e7625f7c00..db22cccd0c 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { go func(idx int) { defer pend.Done() - ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}) + ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil) defer ethash.Close() if err := ethash.VerifySeal(nil, block.Header()); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index eb0f73d98b..e18a06d52a 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -493,7 +493,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head if !bytes.Equal(header.MixDigest[:], digest) { return errInvalidMixDigest } - target := new(big.Int).Div(maxUint256, header.Difficulty) + target := new(big.Int).Div(two256, header.Difficulty) if new(big.Int).SetBytes(result).Cmp(target) > 0 { return errInvalidPoW } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 0cb3059b9d..19c94deb6b 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -45,11 +45,11 @@ import ( var ErrInvalidDumpMagic = errors.New("invalid dump magic") var ( - // maxUint256 is a big integer representing 2^256-1 - maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) + // two256 is a big integer representing 2^256 + two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) // sharedEthash is a full instance that can be shared between multiple users. - sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}) + sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil) // algorithmRevision is the data structure version used for file naming. algorithmRevision = 23 @@ -447,8 +447,10 @@ type Ethash struct { exitCh chan chan error // Notification channel to exiting backend threads } -// New creates a full sized ethash PoW scheme and starts a background thread for remote mining. -func New(config Config) *Ethash { +// New creates a full sized ethash PoW scheme and starts a background thread for +// remote mining, also optionally notifying a batch of remote services of new work +// packages. +func New(config Config, notify []string) *Ethash { if config.CachesInMem <= 0 { log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) config.CachesInMem = 1 @@ -473,13 +475,13 @@ func New(config Config) *Ethash { submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote() + go ethash.remote(notify) return ethash } // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. -func NewTester() *Ethash { +func NewTester(notify []string) *Ethash { ethash := &Ethash{ config: Config{PowMode: ModeTest}, caches: newlru("cache", 1, newCache), @@ -494,7 +496,7 @@ func NewTester() *Ethash { submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote() + go ethash.remote(notify) return ethash } diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index ccdd30fb0f..87ac17c2b2 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -32,17 +32,18 @@ import ( // Tests that ethash works correctly in test mode. func TestTestMode(t *testing.T) { - head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} - ethash := NewTester() + ethash := NewTester(nil) defer ethash.Close() - block, err := ethash.Seal(nil, types.NewBlockWithHeader(head), nil) + + block, err := ethash.Seal(nil, types.NewBlockWithHeader(header), nil) if err != nil { t.Fatalf("failed to seal block: %v", err) } - head.Nonce = types.EncodeNonce(block.Nonce()) - head.MixDigest = block.MixDigest() - if err := ethash.VerifySeal(nil, head); err != nil { + header.Nonce = types.EncodeNonce(block.Nonce()) + header.MixDigest = block.MixDigest() + if err := ethash.VerifySeal(nil, header); err != nil { t.Fatalf("unexpected verification error: %v", err) } } @@ -55,7 +56,7 @@ func TestCacheFileEvict(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpdir) - e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}) + e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil) defer e.Close() workers := 8 @@ -78,21 +79,21 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { if block < 0 { block = 0 } - head := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} - e.VerifySeal(nil, head) + header := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} + e.VerifySeal(nil, header) } } func TestRemoteSealer(t *testing.T) { - ethash := NewTester() + ethash := NewTester(nil) defer ethash.Close() + api := &API{ethash} if _, err := api.GetWork(); err != errNoMiningWork { t.Error("expect to return an error indicate there is no mining work") } - - head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} - block := types.NewBlockWithHeader(head) + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) // Push new work. ethash.Seal(nil, block, nil) @@ -108,16 +109,14 @@ func TestRemoteSealer(t *testing.T) { if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { t.Error("expect to return false when submit a fake solution") } - // Push new block with same block number to replace the original one. - head = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} - block = types.NewBlockWithHeader(head) + header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} + block = types.NewBlockWithHeader(header) ethash.Seal(nil, block, nil) if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { t.Error("expect to return the latest pushed work") } - // Push block with higher block number. newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} newBlock := types.NewBlockWithHeader(newHead) @@ -130,19 +129,18 @@ func TestRemoteSealer(t *testing.T) { func TestHashRate(t *testing.T) { var ( - ethash = NewTester() - api = &API{ethash} hashrate = []hexutil.Uint64{100, 200, 300} expect uint64 ids = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")} ) - + ethash := NewTester(nil) defer ethash.Close() if tot := ethash.Hashrate(); tot != 0 { t.Error("expect the result should be zero") } + api := &API{ethash} for i := 0; i < len(hashrate); i += 1 { if res := api.SubmitHashRate(hashrate[i], ids[i]); !res { t.Error("remote miner submit hashrate failed") @@ -155,9 +153,8 @@ func TestHashRate(t *testing.T) { } func TestClosedRemoteSealer(t *testing.T) { - ethash := NewTester() - // Make sure exit channel has been listened - time.Sleep(1 * time.Second) + ethash := NewTester(nil) + time.Sleep(1 * time.Second) // ensure exit channel is listening ethash.Close() api := &API{ethash} diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index a9449d4060..03d8484739 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -17,11 +17,14 @@ package ethash import ( + "bytes" crand "crypto/rand" + "encoding/json" "errors" "math" "math/big" "math/rand" + "net/http" "runtime" "sync" "time" @@ -109,7 +112,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s var ( header = block.Header() hash = header.HashNoNonce().Bytes() - target = new(big.Int).Div(maxUint256, header.Difficulty) + target = new(big.Int).Div(two256, header.Difficulty) number = header.Number.Uint64() dataset = ethash.dataset(number) ) @@ -161,40 +164,65 @@ search: runtime.KeepAlive(dataset) } -// remote starts a standalone goroutine to handle remote mining related stuff. -func (ethash *Ethash) remote() { +// remote is a standalone goroutine to handle remote mining related stuff. +func (ethash *Ethash) remote(notify []string) { var ( - works = make(map[common.Hash]*types.Block) - rates = make(map[common.Hash]hashrate) - currentWork *types.Block - ) + works = make(map[common.Hash]*types.Block) + rates = make(map[common.Hash]hashrate) - // getWork returns a work package for external miner. + currentBlock *types.Block + currentWork [3]string + + notifyTransport = &http.Transport{} + notifyClient = &http.Client{ + Transport: notifyTransport, + Timeout: time.Second, + } + notifyReqs = make([]*http.Request, len(notify)) + ) + // notifyWork notifies all the specified mining endpoints of the availability of + // new work to be processed. + notifyWork := func() { + work := currentWork + blob, _ := json.Marshal(work) + + for i, url := range notify { + // Terminate any previously pending request and create the new work + if notifyReqs[i] != nil { + notifyTransport.CancelRequest(notifyReqs[i]) + } + notifyReqs[i], _ = http.NewRequest("POST", url, bytes.NewReader(blob)) + notifyReqs[i].Header.Set("Content-Type", "application/json") + + // Push the new work concurrently to all the remote nodes + go func(req *http.Request, url string) { + res, err := notifyClient.Do(req) + if err != nil { + log.Warn("Failed to notify remote miner", "err", err) + } else { + log.Trace("Notified remote miner", "miner", url, "hash", log.Lazy{Fn: func() common.Hash { return common.HexToHash(work[0]) }}, "target", work[2]) + res.Body.Close() + } + }(notifyReqs[i], url) + } + } + // makeWork creates a work package for external miner. // // The work package consists of 3 strings: // result[0], 32 bytes hex encoded current block header pow-hash // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty - getWork := func() ([3]string, error) { - var res [3]string - if currentWork == nil { - return res, errNoMiningWork - } - res[0] = currentWork.HashNoNonce().Hex() - res[1] = common.BytesToHash(SeedHash(currentWork.NumberU64())).Hex() + makeWork := func(block *types.Block) { + hash := block.HashNoNonce() - // Calculate the "target" to be returned to the external sealer. - n := big.NewInt(1) - n.Lsh(n, 255) - n.Div(n, currentWork.Difficulty()) - n.Lsh(n, 1) - res[2] = common.BytesToHash(n.Bytes()).Hex() + currentWork[0] = hash.Hex() + currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() + currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() // Trace the seal work fetched by remote sealer. - works[currentWork.HashNoNonce()] = currentWork - return res, nil + currentBlock = block + works[hash] = block } - // submitWork verifies the submitted pow solution, returning // whether the solution was accepted or not (not can be both a bad pow as well as // any other error, like no pending work or stale mining result). @@ -238,21 +266,23 @@ func (ethash *Ethash) remote() { for { select { case block := <-ethash.workCh: - if currentWork != nil && block.ParentHash() != currentWork.ParentHash() { + if currentBlock != nil && block.ParentHash() != currentBlock.ParentHash() { // Start new round mining, throw out all previous work. works = make(map[common.Hash]*types.Block) } // Update current work with new received block. // Note same work can be past twice, happens when changing CPU threads. - currentWork = block + makeWork(block) + + // Notify and requested URLs of the new work availability + notifyWork() case work := <-ethash.fetchWorkCh: // Return current mining work to remote miner. - miningWork, err := getWork() - if err != nil { - work.errc <- err + if currentBlock == nil { + work.errc <- errNoMiningWork } else { - work.res <- miningWork + work.res <- currentWork } case result := <-ethash.submitWorkCh: diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go new file mode 100644 index 0000000000..6d8a770495 --- /dev/null +++ b/consensus/ethash/sealer_test.go @@ -0,0 +1,115 @@ +package ethash + +import ( + "encoding/json" + "io/ioutil" + "math/big" + "net" + "net/http" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// Tests whether remote HTTP servers are correctly notified of new work. +func TestRemoteNotify(t *testing.T) { + // Start a simple webserver to capture notifications + sink := make(chan [3]string) + + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Fatalf("failed to read miner notification: %v", err) + } + var work [3]string + if err := json.Unmarshal(blob, &work); err != nil { + t.Fatalf("failed to unmarshal miner notification: %v", err) + } + sink <- work + }), + } + // Open a custom listener to extract its local address + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to open notification server: %v", err) + } + defer listener.Close() + + go server.Serve(listener) + + // Create the custom ethash engine + ethash := NewTester([]string{"http://" + listener.Addr().String()}) + defer ethash.Close() + + // Stream a work task and ensure the notification bubbles out + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) + + ethash.Seal(nil, block, nil) + select { + case work := <-sink: + if want := header.HashNoNonce().Hex(); work[0] != want { + t.Errorf("work packet hash mismatch: have %s, want %s", work[0], want) + } + if want := common.BytesToHash(SeedHash(header.Number.Uint64())).Hex(); work[1] != want { + t.Errorf("work packet seed mismatch: have %s, want %s", work[1], want) + } + target := new(big.Int).Div(new(big.Int).Lsh(big.NewInt(1), 256), header.Difficulty) + if want := common.BytesToHash(target.Bytes()).Hex(); work[2] != want { + t.Errorf("work packet target mismatch: have %s, want %s", work[2], want) + } + case <-time.After(time.Second): + t.Fatalf("notification timed out") + } +} + +// Tests that pushing work packages fast to the miner doesn't cause any daa race +// issues in the notifications. +func TestRemoteMultiNotify(t *testing.T) { + // Start a simple webserver to capture notifications + sink := make(chan [3]string, 1024) + + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Fatalf("failed to read miner notification: %v", err) + } + var work [3]string + if err := json.Unmarshal(blob, &work); err != nil { + t.Fatalf("failed to unmarshal miner notification: %v", err) + } + sink <- work + }), + } + // Open a custom listener to extract its local address + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to open notification server: %v", err) + } + defer listener.Close() + + go server.Serve(listener) + + // Create the custom ethash engine + ethash := NewTester([]string{"http://" + listener.Addr().String()}) + defer ethash.Close() + + // Stream a lot of work task and ensure all the notifications bubble out + for i := 0; i < cap(sink); i++ { + header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) + + ethash.Seal(nil, block, nil) + } + for i := 0; i < cap(sink); i++ { + select { + case <-sink: + case <-time.After(250 * time.Millisecond): + t.Fatalf("notification %d timed out", i) + } + } +} diff --git a/eth/backend.go b/eth/backend.go index 32946a0ab3..865534b198 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -124,7 +124,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { chainConfig: chainConfig, eventMux: ctx.EventMux, accountManager: ctx.AccountManager, - engine: CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb), + engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb), shutdownChan: make(chan bool), networkID: config.NetworkId, gasPrice: config.GasPrice, @@ -210,7 +210,7 @@ func CreateDB(ctx *node.ServiceContext, config *Config, name string) (ethdb.Data } // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service -func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chainConfig *params.ChainConfig, db ethdb.Database) consensus.Engine { +func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up if chainConfig.Clique != nil { return clique.New(chainConfig.Clique, db) @@ -222,7 +222,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai return ethash.NewFaker() case ethash.ModeTest: log.Warn("Ethash used in test mode") - return ethash.NewTester() + return ethash.NewTester(nil) case ethash.ModeShared: log.Warn("Ethash used in shared mode") return ethash.NewShared() @@ -234,7 +234,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai DatasetDir: config.DatasetDir, DatasetsInMem: config.DatasetsInMem, DatasetsOnDisk: config.DatasetsOnDisk, - }) + }, notify) engine.SetThreads(-1) // Disable CPU mining return engine } diff --git a/eth/config.go b/eth/config.go index 426d2bf1ef..0c82f29232 100644 --- a/eth/config.go +++ b/eth/config.go @@ -97,6 +97,7 @@ type Config struct { // Mining-related options Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` + MinerNotify []string `toml:",omitempty"` ExtraData []byte `toml:",omitempty"` GasPrice *big.Int diff --git a/les/backend.go b/les/backend.go index 952d92cc2a..178bc1e0e4 100644 --- a/les/backend.go +++ b/les/backend.go @@ -102,7 +102,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { peers: peers, reqDist: newRequestDistributor(peers, quitSync), accountManager: ctx.AccountManager, - engine: eth.CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb), + engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), shutdownChan: make(chan bool), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), From 3ec5dda4d2dd0dec6d5bd465752f30e8f6ce208c Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 10 Aug 2018 13:49:37 +0200 Subject: [PATCH 011/126] swarm/api/http: added logging to denote request ended (#17371) --- swarm/api/http/middleware.go | 2 +- swarm/api/http/response.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index d338a782ce..c0d8d1a408 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -64,8 +64,8 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writer := newLoggingResponseWriter(w) - h.ServeHTTP(writer, r) + log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) }) } diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index 9f4788d35e..f050e706a8 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -84,15 +84,16 @@ func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) } func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { + w.WriteHeader(params.Code) if params.Code >= 400 { - w.Header().Del("Cache-Control") //avoid sending cache headers for errors! + w.Header().Del("Cache-Control") w.Header().Del("ETag") } acceptHeader := r.Header.Get("Accept") - // this cannot be in a switch form since an Accept header can be in the form of "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8" + // this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8" if strings.Contains(acceptHeader, "application/json") { if err := respondJSON(w, r, params); err != nil { RespondError(w, r, "Internal server error", http.StatusInternalServerError) From 6d1e292eefa70b5cb76cd03ff61fc6c4550d7c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Fri, 10 Aug 2018 16:12:55 +0200 Subject: [PATCH 012/126] Manifest cli fix and upload defaultpath only once (#17375) * cmd/swarm: fix manifest subcommands and add tests * cmd/swarm: manifest update: update default entry for non-encrypted uploads * swarm/api: upload defaultpath file only once * swarm/api/client: improve UploadDirectory default path handling * cmd/swarm: support absolute and relative default path values * cmd/swarm: fix a typo in test * cmd/swarm: check encrypted uploads in manifest update tests --- cmd/swarm/main.go | 10 +- cmd/swarm/manifest.go | 234 ++++++------- cmd/swarm/manifest_test.go | 579 ++++++++++++++++++++++++++++++++ cmd/swarm/upload.go | 11 + cmd/swarm/upload_test.go | 81 +++++ swarm/api/api.go | 22 +- swarm/api/client/client.go | 31 +- swarm/api/client/client_test.go | 2 +- swarm/api/http/server.go | 4 +- swarm/api/manifest.go | 17 +- 10 files changed, 840 insertions(+), 151 deletions(-) create mode 100644 cmd/swarm/manifest_test.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 258f24d320..ac09ae9981 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -322,23 +322,23 @@ Downloads a swarm bzz uri to the given dir. When no dir is provided, working dir Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", Subcommands: []cli.Command{ { - Action: add, + Action: manifestAdd, CustomHelpTemplate: helpTemplate, Name: "add", Usage: "add a new path to the manifest", - ArgsUsage: " []", + ArgsUsage: " ", Description: "Adds a new path to the manifest", }, { - Action: update, + Action: manifestUpdate, CustomHelpTemplate: helpTemplate, Name: "update", Usage: "update the hash for an already existing path in the manifest", - ArgsUsage: " []", + ArgsUsage: " ", Description: "Update the hash for an already existing path in the manifest", }, { - Action: remove, + Action: manifestRemove, CustomHelpTemplate: helpTemplate, Name: "remove", Usage: "removes a path from the manifest", diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 82166edf6c..0216ffc1dd 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -18,10 +18,8 @@ package main import ( - "encoding/json" "fmt" - "mime" - "path/filepath" + "os" "strings" "github.com/ethereum/go-ethereum/cmd/utils" @@ -30,127 +28,118 @@ import ( "gopkg.in/urfave/cli.v1" ) -const bzzManifestJSON = "application/bzz-manifest+json" - -func add(ctx *cli.Context) { +// manifestAdd adds a new entry to the manifest at the given path. +// New entry hash, the last argument, must be the hash of a manifest +// with only one entry, which meta-data will be added to the original manifest. +// On success, this function will print new (updated) manifest's hash. +func manifestAdd(ctx *cli.Context) { args := ctx.Args() - if len(args) < 3 { - utils.Fatalf("Need at least three arguments []") + if len(args) != 3 { + utils.Fatalf("Need exactly three arguments ") } var ( mhash = args[0] path = args[1] hash = args[2] - - ctype string - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - if len(args) > 3 { - ctype = args[3] - } else { - ctype = mime.TypeByExtension(filepath.Ext(path)) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + + m, _, err := client.DownloadManifest(hash) + if err != nil { + utils.Fatalf("Error downloading manifest to add: %v", err) + } + l := len(m.Entries) + if l == 0 { + utils.Fatalf("No entries in manifest %s", hash) + } else if l > 1 { + utils.Fatalf("Too many entries in manifest %s", hash) } - newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype) + newManifest := addEntryToManifest(client, mhash, path, m.Entries[0]) fmt.Println(newManifest) - - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } } -func update(ctx *cli.Context) { - +// manifestUpdate replaces an existing entry of the manifest at the given path. +// New entry hash, the last argument, must be the hash of a manifest +// with only one entry, which meta-data will be added to the original manifest. +// On success, this function will print hash of the updated manifest. +func manifestUpdate(ctx *cli.Context) { args := ctx.Args() - if len(args) < 3 { - utils.Fatalf("Need at least three arguments ") + if len(args) != 3 { + utils.Fatalf("Need exactly three arguments ") } var ( mhash = args[0] path = args[1] hash = args[2] - - ctype string - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - if len(args) > 3 { - ctype = args[3] - } else { - ctype = mime.TypeByExtension(filepath.Ext(path)) + + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + + m, _, err := client.DownloadManifest(hash) + if err != nil { + utils.Fatalf("Error downloading manifest to update: %v", err) + } + l := len(m.Entries) + if l == 0 { + utils.Fatalf("No entries in manifest %s", hash) + } else if l > 1 { + utils.Fatalf("Too many entries in manifest %s", hash) } - newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype) + newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true) + if defaultEntryUpdated { + // Print informational message to stderr + // allowing the user to get the new manifest hash from stdout + // without the need to parse the complete output. + fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too") + } fmt.Println(newManifest) - - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } } -func remove(ctx *cli.Context) { +// manifestRemove removes an existing entry of the manifest at the given path. +// On success, this function will print hash of the manifest which does not +// contain the path. +func manifestRemove(ctx *cli.Context) { args := ctx.Args() - if len(args) < 2 { - utils.Fatalf("Need at least two arguments ") + if len(args) != 2 { + utils.Fatalf("Need exactly two arguments ") } var ( mhash = args[0] path = args[1] - - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - newManifest := removeEntryFromManifest(ctx, mhash, path) - fmt.Println(newManifest) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } + newManifest := removeEntryFromManifest(client, mhash, path) + fmt.Println(newManifest) } -func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string { - - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) - longestPathEntry = api.ManifestEntry{} - ) +func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string { + var longestPathEntry = api.ManifestEntry{} mroot, isEncrypted, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } - //TODO: check if the "hash" to add is valid and present in swarm - _, _, err = client.DownloadManifest(hash) - if err != nil { - utils.Fatalf("Hash to add is not present: %v", err) - } - // See if we path is in this Manifest or do we have to dig deeper - for _, entry := range mroot.Entries { - if path == entry.Path { + for _, e := range mroot.Entries { + if path == e.Path { utils.Fatalf("Path %s already present, not adding anything", path) } else { - if entry.ContentType == bzzManifestJSON { - prfxlen := strings.HasPrefix(path, entry.Path) + if e.ContentType == api.ManifestType { + prfxlen := strings.HasPrefix(path, e.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { - longestPathEntry = entry + longestPathEntry = e } } } @@ -159,25 +148,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin if longestPathEntry.Path != "" { // Load the child Manifest add the entry there newPath := path[len(longestPathEntry.Path):] - newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) + newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if longestPathEntry.Path == entry.Path { - entry.Hash = newHash + for _, e := range mroot.Entries { + if longestPathEntry.Path == e.Path { + e.Hash = newHash } - newMRoot.Entries = append(newMRoot.Entries, entry) + newMRoot.Entries = append(newMRoot.Entries, e) } mroot = newMRoot } else { // Add the entry in the leaf Manifest - newEntry := api.ManifestEntry{ - Hash: hash, - Path: path, - ContentType: ctype, - } - mroot.Entries = append(mroot.Entries, newEntry) + entry.Path = path + mroot.Entries = append(mroot.Entries, entry) } newManifestHash, err := client.UploadManifest(mroot, isEncrypted) @@ -185,14 +170,16 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin utils.Fatalf("Manifest upload failed: %v", err) } return newManifestHash - } -func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string { - +// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash +// finding the path recursively through all nested manifests. Argument isRoot is used for default +// entry update detection. If the updated entry has the same hash as the default entry, then the +// default entry in root manifest will be updated too. +// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and +// a a bool that is true if default entry is updated. +func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) { var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) newEntry = api.ManifestEntry{} longestPathEntry = api.ManifestEntry{} ) @@ -202,17 +189,18 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st utils.Fatalf("Manifest download failed: %v", err) } - //TODO: check if the "hash" with which to update is valid and present in swarm - // See if we path is in this Manifest or do we have to dig deeper - for _, entry := range mroot.Entries { - if path == entry.Path { - newEntry = entry + for _, e := range mroot.Entries { + if path == e.Path { + newEntry = e + // keep the reference of the hash of the entry that should be replaced + // for default entry detection + oldHash = e.Hash } else { - if entry.ContentType == bzzManifestJSON { - prfxlen := strings.HasPrefix(path, entry.Path) + if e.ContentType == api.ManifestType { + prfxlen := strings.HasPrefix(path, e.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { - longestPathEntry = entry + longestPathEntry = e } } } @@ -225,50 +213,50 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st if longestPathEntry.Path != "" { // Load the child Manifest add the entry there newPath := path[len(longestPathEntry.Path):] - newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) + var newHash string + newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if longestPathEntry.Path == entry.Path { - entry.Hash = newHash + for _, e := range mroot.Entries { + if longestPathEntry.Path == e.Path { + e.Hash = newHash } - newMRoot.Entries = append(newMRoot.Entries, entry) + newMRoot.Entries = append(newMRoot.Entries, e) } mroot = newMRoot } - if newEntry.Path != "" { + // update the manifest if the new entry is found and + // check if default entry should be updated + if newEntry.Path != "" || isRoot { // Replace the hash for leaf Manifest newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if newEntry.Path == entry.Path { - myEntry := api.ManifestEntry{ - Hash: hash, - Path: entry.Path, - ContentType: ctype, - } - newMRoot.Entries = append(newMRoot.Entries, myEntry) - } else { + for _, e := range mroot.Entries { + if newEntry.Path == e.Path { + entry.Path = e.Path newMRoot.Entries = append(newMRoot.Entries, entry) + } else if isRoot && e.Path == "" && e.Hash == oldHash { + entry.Path = e.Path + newMRoot.Entries = append(newMRoot.Entries, entry) + defaultEntryUpdated = true + } else { + newMRoot.Entries = append(newMRoot.Entries, e) } } mroot = newMRoot } - newManifestHash, err := client.UploadManifest(mroot, isEncrypted) + newManifestHash, err = client.UploadManifest(mroot, isEncrypted) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } - return newManifestHash + return newManifestHash, oldHash, defaultEntryUpdated } -func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { - +func removeEntryFromManifest(client *swarm.Client, mhash, path string) string { var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) entryToRemove = api.ManifestEntry{} longestPathEntry = api.ManifestEntry{} ) @@ -283,7 +271,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if path == entry.Path { entryToRemove = entry } else { - if entry.ContentType == bzzManifestJSON { + if entry.ContentType == api.ManifestType { prfxlen := strings.HasPrefix(path, entry.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { longestPathEntry = entry @@ -299,7 +287,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if longestPathEntry.Path != "" { // Load the child Manifest remove the entry there newPath := path[len(longestPathEntry.Path):] - newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath) + newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go new file mode 100644 index 0000000000..08fe0b2eb7 --- /dev/null +++ b/cmd/swarm/manifest_test.go @@ -0,0 +1,579 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/swarm/api" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" +) + +// TestManifestChange tests manifest add, update and remove +// cli commands without encryption. +func TestManifestChange(t *testing.T) { + testManifestChange(t, false) +} + +// TestManifestChange tests manifest add, update and remove +// cli commands with encryption enabled. +func TestManifestChangeEncrypted(t *testing.T) { + testManifestChange(t, true) +} + +// testManifestChange performs cli commands: +// - manifest add +// - manifest update +// - manifest remove +// on a manifest, testing the functionality of this +// comands on paths that are in root manifest or a nested one. +// Argument encrypt controls whether to use encryption or not. +func testManifestChange(t *testing.T, encrypt bool) { + t.Parallel() + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + origDir := filepath.Join(tmp, "orig") + if err := os.Mkdir(origDir, 0777); err != nil { + t.Fatal(err) + } + + indexDataFilename := filepath.Join(origDir, "index.html") + err = ioutil.WriteFile(indexDataFilename, []byte("

Test

"), 0666) + if err != nil { + t.Fatal(err) + } + // Files paths robots.txt and robots.html share the same prefix "robots." + // which will result a manifest with a nested manifest under path "robots.". + // This will allow testing manifest changes on both root and nested manifest. + err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("No Robots Allowed"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + indexDataFilename, + "up", + origDir, + } + if encrypt { + args = append(args, "--encrypt") + } + + origManifestHash := runSwarmExpectHash(t, args...) + + checkHashLength(t, origManifestHash, encrypt) + + client := swarm.NewClient(cluster.Nodes[0].URL) + + // upload a new file and use its manifest to add it the original manifest. + t.Run("add", func(t *testing.T) { + humansData := []byte("Ann\nBob") + humansDataFilename := filepath.Join(tmp, "humans.txt") + err = ioutil.WriteFile(humansDataFilename, humansData, 0666) + if err != nil { + t.Fatal(err) + } + + humansManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + humansDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "add", + origManifestHash, + "humans.txt", + humansManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "humans.txt" { + found = true + if e.Size != int64(len(humansData)) { + t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for humans.txt") + } + ct := "text/plain; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no humans.txt in new manifest") + } + + checkFile(t, client, newManifestHash, "humans.txt", humansData) + }) + + // upload a new file and use its manifest to add it the original manifest, + // but ensure that the file will be in the nested manifest of the original one. + t.Run("add nested", func(t *testing.T) { + robotsData := []byte(`{"disallow": "/"}`) + robotsDataFilename := filepath.Join(tmp, "robots.json") + err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666) + if err != nil { + t.Fatal(err) + } + + robotsManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + robotsDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "add", + origManifestHash, + "robots.json", + robotsManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "json" { + found = true + if e.Size != int64(len(robotsData)) { + t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for robots.json") + } + ct := "application/json" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break loop + } + } + } + } + if !found { + t.Fatal("no robots.json in new manifest") + } + + checkFile(t, client, newManifestHash, "robots.json", robotsData) + }) + + // upload a new file and use its manifest to change the file it the original manifest. + t.Run("update", func(t *testing.T) { + indexData := []byte("

Ethereum Swarm

") + indexDataFilename := filepath.Join(tmp, "index.html") + err = ioutil.WriteFile(indexDataFilename, indexData, 0666) + if err != nil { + t.Fatal(err) + } + + indexManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + indexDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "index.html", + indexManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "index.html" { + found = true + if e.Size != int64(len(indexData)) { + t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for index.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no index.html in new manifest") + } + + checkFile(t, client, newManifestHash, "index.html", indexData) + + // check default entry change + checkFile(t, client, newManifestHash, "", indexData) + }) + + // upload a new file and use its manifest to change the file it the original manifest, + // but ensure that the file is in the nested manifest of the original one. + t.Run("update nested", func(t *testing.T) { + robotsData := []byte(`Only humans allowed!!!`) + robotsDataFilename := filepath.Join(tmp, "robots.html") + err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666) + if err != nil { + t.Fatal(err) + } + + humansManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + robotsDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "robots.html", + humansManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "html" { + found = true + if e.Size != int64(len(robotsData)) { + t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for robots.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break loop + } + } + } + } + if !found { + t.Fatal("no robots.html in new manifest") + } + + checkFile(t, client, newManifestHash, "robots.html", robotsData) + }) + + // remove a file from the manifest. + t.Run("remove", func(t *testing.T) { + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "remove", + origManifestHash, + "mutants.txt", + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "mutants.txt" { + found = true + break + } + } + if found { + t.Fatal("mutants.txt is not removed") + } + }) + + // remove a file from the manifest, but ensure that the file is in + // the nested manifest of the original one. + t.Run("remove nested", func(t *testing.T) { + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "remove", + origManifestHash, + "robots.html", + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "html" { + found = true + break loop + } + } + } + } + if found { + t.Fatal("robots.html in not removed") + } + }) +} + +// TestNestedDefaultEntryUpdate tests if the default entry is updated +// if the file in nested manifest used for it is also updated. +func TestNestedDefaultEntryUpdate(t *testing.T) { + testNestedDefaultEntryUpdate(t, false) +} + +// TestNestedDefaultEntryUpdateEncrypted tests if the default entry +// of encrypted upload is updated if the file in nested manifest +// used for it is also updated. +func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { + testNestedDefaultEntryUpdate(t, true) +} + +func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { + t.Parallel() + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + origDir := filepath.Join(tmp, "orig") + if err := os.Mkdir(origDir, 0777); err != nil { + t.Fatal(err) + } + + indexData := []byte("

Test

") + indexDataFilename := filepath.Join(origDir, "index.html") + err = ioutil.WriteFile(indexDataFilename, indexData, 0666) + if err != nil { + t.Fatal(err) + } + // Add another file with common prefix as the default entry to test updates of + // default entry with nested manifests. + err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + indexDataFilename, + "up", + origDir, + } + if encrypt { + args = append(args, "--encrypt") + } + + origManifestHash := runSwarmExpectHash(t, args...) + + checkHashLength(t, origManifestHash, encrypt) + + client := swarm.NewClient(cluster.Nodes[0].URL) + + newIndexData := []byte("

Ethereum Swarm

") + newIndexDataFilename := filepath.Join(tmp, "index.html") + err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666) + if err != nil { + t.Fatal(err) + } + + newIndexManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + newIndexDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "index.html", + newIndexManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "index." { + found = true + newManifest = downloadManifest(t, client, e.Hash, encrypt) + break + } + } + if !found { + t.Fatal("no index. path in new manifest") + } + + found = false + for _, e := range newManifest.Entries { + if e.Path == "html" { + found = true + if e.Size != int64(len(newIndexData)) { + t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for index.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no html in new manifest") + } + + checkFile(t, client, newManifestHash, "index.html", newIndexData) + + // check default entry change + checkFile(t, client, newManifestHash, "", newIndexData) +} + +func runSwarmExpectHash(t *testing.T, args ...string) (hash string) { + t.Helper() + hashRegexp := `[a-f\d]{64,128}` + up := runSwarm(t, args...) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + return matches[0] +} + +func checkHashLength(t *testing.T, hash string, encrypted bool) { + t.Helper() + l := len(hash) + if encrypted && l != 128 { + t.Errorf("expected hash length 128, got %v", l) + } + if !encrypted && l != 64 { + t.Errorf("expected hash length 64, got %v", l) + } +} + +func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) { + t.Helper() + m, isEncrypted, err := client.DownloadManifest(hash) + if err != nil { + t.Fatal(err) + } + + if encrypted != isEncrypted { + t.Error("new manifest encryption flag is not correct") + } + return m +} + +func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) { + t.Helper() + f, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + + got, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, expected) { + t.Errorf("expected file content %q, got %q", expected, got) + } +} diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 8ba0e7c5f0..9eae2a3f80 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -98,6 +98,17 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } + if defaultPath != "" { + // construct absolute default path + absDefaultPath, _ := filepath.Abs(defaultPath) + absFile, _ := filepath.Abs(file) + // make sure absolute directory ends with only one "/" + // to trim it from absolute default path and get relative default path + absFile = strings.TrimRight(absFile, "/") + "/" + if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) { + defaultPath = strings.TrimPrefix(absDefaultPath, absFile) + } + } return client.UploadDirectory(file, defaultPath, "", toEncrypt) } } else { diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 2afc9b3a11..c3199dadc6 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -273,3 +273,84 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } } } + +// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute +// default paths and with encryption. +func TestCLISwarmUpDefaultPath(t *testing.T) { + testCLISwarmUpDefaultPath(false, false, t) + testCLISwarmUpDefaultPath(false, true, t) + testCLISwarmUpDefaultPath(true, false, t) + testCLISwarmUpDefaultPath(true, true, t) +} + +func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("

Test

"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666) + if err != nil { + t.Fatal(err) + } + + defaultPath := "index.html" + if absDefaultPath { + defaultPath = filepath.Join(tmp, defaultPath) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + defaultPath, + "up", + tmp, + } + if toEncrypt { + args = append(args, "--encrypt") + } + + up := runSwarm(t, args...) + hashRegexp := `[a-f\d]{64,128}` + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + hash := matches[0] + + client := swarm.NewClient(cluster.Nodes[0].URL) + + m, isEncrypted, err := client.DownloadManifest(hash) + if err != nil { + t.Fatal(err) + } + + if toEncrypt != isEncrypted { + t.Error("downloaded manifest is not encrypted") + } + + var found bool + var entriesCount int + for _, e := range m.Entries { + entriesCount++ + if e.Path == "" { + found = true + } + } + + if !found { + t.Error("manifest default entry was not found") + } + + if entriesCount != 3 { + t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3) + } +} diff --git a/swarm/api/api.go b/swarm/api/api.go index b418c45e1c..99d971b105 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -704,11 +704,12 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] return fkey, newMkey.String(), nil } -func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) { +func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath, defaultPath string, mw *ManifestWriter) (storage.Address, error) { apiUploadTarCount.Inc(1) var contentKey storage.Address tr := tar.NewReader(bodyReader) defer bodyReader.Close() + var defaultPathFound bool for { hdr, err := tr.Next() if err == io.EOF { @@ -737,6 +738,25 @@ func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestP apiUploadTarFail.Inc(1) return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err) } + if hdr.Name == defaultPath { + entry := &ManifestEntry{ + Hash: contentKey.Hex(), + Path: "", // default entry + ContentType: hdr.Xattrs["user.swarm.content-type"], + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: hdr.ModTime, + } + contentKey, err = mw.AddEntry(ctx, nil, entry) + if err != nil { + apiUploadTarFail.Inc(1) + return nil, fmt.Errorf("error adding default manifest entry from tar stream: %s", err) + } + defaultPathFound = true + } + } + if defaultPath != "" && !defaultPathFound { + return contentKey, fmt.Errorf("default path %q not found", defaultPath) } return contentKey, nil } diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index b3a5e929d0..8a9efe3608 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -138,7 +138,7 @@ func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, er if file.Size <= 0 { return "", errors.New("file size must be greater than zero") } - return c.TarUpload(manifest, &FileUploader{file}, toEncrypt) + return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt) } // Download downloads a file with the given path from the swarm manifest with @@ -175,7 +175,15 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo } else if !stat.IsDir() { return "", fmt.Errorf("not a directory: %s", dir) } - return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt) + if defaultPath != "" { + if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir) + } + return "", fmt.Errorf("default path: %v", err) + } + } + return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt) } // DownloadDirectory downloads the files contained in a swarm manifest under @@ -389,21 +397,11 @@ func (u UploaderFunc) Upload(upload UploadFn) error { // DirectoryUploader uploads all files in a directory, optionally uploading // a file to the default path type DirectoryUploader struct { - Dir string - DefaultPath string + Dir string } // Upload performs the upload of the directory and default path func (d *DirectoryUploader) Upload(upload UploadFn) error { - if d.DefaultPath != "" { - file, err := Open(d.DefaultPath) - if err != nil { - return err - } - if err := upload(file); err != nil { - return err - } - } return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { if err != nil { return err @@ -441,7 +439,7 @@ type UploadFn func(file *File) error // TarUpload uses the given Uploader to upload files to swarm as a tar stream, // returning the resulting manifest hash -func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) { +func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { reqR, reqW := io.Pipe() defer reqR.Close() addr := hash @@ -458,6 +456,11 @@ func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (stri return "", err } req.Header.Set("Content-Type", "application/x-tar") + if defaultPath != "" { + q := req.URL.Query() + q.Set("defaultpath", defaultPath) + req.URL.RawQuery = q.Encode() + } // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index dc608e3f1f..ae82a91d79 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -194,7 +194,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { // upload the directory client := NewClient(srv.URL) - defaultPath := filepath.Join(dir, testDirFiles[0]) + defaultPath := testDirFiles[0] hash, err := client.UploadDirectory(dir, defaultPath, "", false) if err != nil { t.Fatalf("error uploading directory: %s", err) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index bd6949de6c..5a5c42adc0 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -336,7 +336,9 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) { log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context())) - key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, mw) + defaultPath := r.URL.Query().Get("defaultpath") + + key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, defaultPath, mw) if err != nil { return nil, err } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index fbd143f295..2a163dd39c 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -106,13 +106,18 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC } // AddEntry stores the given data and adds the resulting key to the manifest -func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (storage.Address, error) { - key, _, err := m.api.Store(ctx, data, e.Size, m.trie.encrypted) - if err != nil { - return nil, err - } +func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (key storage.Address, err error) { entry := newManifestTrieEntry(e, nil) - entry.Hash = key.Hex() + if data != nil { + key, _, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) + if err != nil { + return nil, err + } + entry.Hash = key.Hex() + } + if entry.Hash == "" { + return key, errors.New("missing entry hash") + } m.trie.addEntry(entry, m.quitC) return key, nil } From fb368723acf83e64c71e1eaa403e7cda06e6ce5e Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Mon, 13 Aug 2018 16:40:52 +0800 Subject: [PATCH 013/126] core: fix comment typo (#17376) --- core/tx_cacher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tx_cacher.go b/core/tx_cacher.go index 6d989c83d9..bcaa5ead38 100644 --- a/core/tx_cacher.go +++ b/core/tx_cacher.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// senderCacher is a concurrent tranaction sender recoverer anc cacher. +// senderCacher is a concurrent transaction sender recoverer anc cacher. var senderCacher = newTxSenderCacher(runtime.NumCPU()) // txSenderCacherRequest is a request for recovering transaction senders with a @@ -45,7 +45,7 @@ type txSenderCacher struct { } // newTxSenderCacher creates a new transaction sender background cacher and starts -// as many procesing goroutines as allowed by the GOMAXPROCS on construction. +// as many processing goroutines as allowed by the GOMAXPROCS on construction. func newTxSenderCacher(threads int) *txSenderCacher { cacher := &txSenderCacher{ tasks: make(chan *txSenderCacherRequest, threads), From e07e507d1af687cd64b263038ebd3cb7be74fb40 Mon Sep 17 00:00:00 2001 From: Eugene Valeyev Date: Mon, 13 Aug 2018 17:27:25 +0300 Subject: [PATCH 014/126] whisper: fixed broken partial topic filtering Changes in #15811 broke partial topic filtering. Re-enable it. --- whisper/whisperv5/filter.go | 2 +- whisper/whisperv5/filter_test.go | 8 +++---- whisper/whisperv6/filter.go | 17 --------------- whisper/whisperv6/filter_test.go | 36 -------------------------------- 4 files changed, 5 insertions(+), 58 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index 3190334ebb..9550a7e389 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -220,7 +220,7 @@ func matchSingleTopic(topic TopicType, bt []byte) bool { bt = bt[:TopicLength] } - if len(bt) < TopicLength { + if len(bt) == 0 { return false } diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 01034a3513..c01c22668c 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -829,16 +829,16 @@ func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) { } } -func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { +func TestMatchSingleTopic_PartialTopic_ReturnTrue(t *testing.T) { bt := []byte("tes") - topic := BytesToTopic(bt) + topic := BytesToTopic([]byte("test")) - if matchSingleTopic(topic, bt) { + if !matchSingleTopic(topic, bt) { t.FailNow() } } -func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) { +func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { bt := []byte("test") topic := BytesToTopic([]byte("not_equal")) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index 2f170ddebe..6a5b79674b 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -250,23 +250,6 @@ func (f *Filter) MatchEnvelope(envelope *Envelope) bool { return f.PoW <= 0 || envelope.pow >= f.PoW } -func matchSingleTopic(topic TopicType, bt []byte) bool { - if len(bt) > TopicLength { - bt = bt[:TopicLength] - } - - if len(bt) < TopicLength { - return false - } - - for j, b := range bt { - if topic[j] != b { - return false - } - } - return true -} - // IsPubKeyEqual checks that two public keys are equal func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { if !ValidatePublicKey(a) { diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 0bb7986c39..82e4aa0241 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -829,39 +829,3 @@ func TestVariableTopics(t *testing.T) { } } } - -func TestMatchSingleTopic_ReturnTrue(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic(bt) - - if !matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) { - bt := []byte("test with tail") - topic := BytesToTopic([]byte("test")) - - if !matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { - bt := []byte("tes") - topic := BytesToTopic(bt) - - if matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic([]byte("not_equal")) - - if matchSingleTopic(topic, bt) { - t.FailNow() - } -} From 8a040de60bd6b740ebe87cd8e1fe6bfdb6635d2f Mon Sep 17 00:00:00 2001 From: Yao Zengzeng Date: Tue, 14 Aug 2018 19:25:36 +0800 Subject: [PATCH 015/126] README.md: fix some typos (#17381) Signed-off-by: YaoZengzeng --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86fcdeee4d..c6bc91af17 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ This command will: * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console), (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API) as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs). - This too is optional and if you leave it out you can always attach to an already running Geth instance + This tool is optional and if you leave it out you can always attach to an already running Geth instance with `geth attach`. ### Full node on the Ethereum test network From 97887d98da703a31040bceee13bce9ee77fca673 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 14 Aug 2018 16:03:56 +0200 Subject: [PATCH 016/126] swarm/network, swarm/storage: validate chunk size (#17397) * swarm/network, swarm/storage: validate default chunk size * swarm/bmt, swarm/network, swarm/storage: update BMT hash initialisation * swarm/bmt: move segmentCount to tests * swarm/chunk: change chunk.DefaultSize to be untyped const * swarm/storage: add size validator * swarm/storage: add chunk size validation to localstore * swarm/storage: move validation from localstore to validator * swarm/storage: global chunk rules in MRU --- swarm/bmt/bmt.go | 5 +---- swarm/bmt/bmt_test.go | 19 +++++++++++++------ swarm/chunk/chunk.go | 5 +++++ swarm/network/stream/delivery.go | 7 +++++++ swarm/storage/chunker.go | 9 +++------ swarm/storage/hasherstore.go | 9 +++++---- swarm/storage/ldbstore_test.go | 11 ++++++----- swarm/storage/localstore.go | 10 +++------- swarm/storage/localstore_test.go | 8 +++++--- swarm/storage/mru/handler.go | 25 ++++++++----------------- swarm/storage/mru/resource_test.go | 8 +++----- swarm/storage/mru/testutil.go | 5 +---- swarm/storage/mru/update.go | 3 ++- swarm/storage/pyramid.go | 5 +++-- swarm/storage/types.go | 13 ++++++++++--- swarm/swarm.go | 13 ++++--------- 16 files changed, 79 insertions(+), 76 deletions(-) create mode 100644 swarm/chunk/chunk.go diff --git a/swarm/bmt/bmt.go b/swarm/bmt/bmt.go index 97e0e141ed..a85d4369e5 100644 --- a/swarm/bmt/bmt.go +++ b/swarm/bmt/bmt.go @@ -55,9 +55,6 @@ Two implementations are provided: */ const ( - // SegmentCount is the maximum number of segments of the underlying chunk - // Should be equal to max-chunk-data-size / hash-size - SegmentCount = 128 // PoolSize is the maximum number of bmt trees used by the hashers, i.e, // the maximum number of concurrent BMT hashing operations performed by the same hasher PoolSize = 8 @@ -318,7 +315,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) { // with every full segment calls writeSection in a go routine func (h *Hasher) Write(b []byte) (int, error) { l := len(b) - if l == 0 || l > 4096 { + if l == 0 || l > h.pool.Size { return 0, nil } t := h.getTree() diff --git a/swarm/bmt/bmt_test.go b/swarm/bmt/bmt_test.go index 891d8cbb29..760aa11d8b 100644 --- a/swarm/bmt/bmt_test.go +++ b/swarm/bmt/bmt_test.go @@ -34,6 +34,13 @@ import ( // the actual data length generated (could be longer than max datalength of the BMT) const BufferSize = 4128 +const ( + // segmentCount is the maximum number of segments of the underlying chunk + // Should be equal to max-chunk-data-size / hash-size + // Currently set to 128 == 4096 (default chunk size) / 32 (sha3.keccak256 size) + segmentCount = 128 +) + var counts = []int{1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 32, 37, 42, 53, 63, 64, 65, 111, 127, 128} // calculates the Keccak256 SHA3 hash of the data @@ -224,14 +231,14 @@ func TestHasherReuse(t *testing.T) { // tests if bmt reuse is not corrupting result func testHasherReuse(poolsize int, t *testing.T) { hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, poolsize) + pool := NewTreePool(hasher, segmentCount, poolsize) defer pool.Drain(0) bmt := New(pool) for i := 0; i < 100; i++ { data := newData(BufferSize) n := rand.Intn(bmt.Size()) - err := testHasherCorrectness(bmt, hasher, data, n, SegmentCount) + err := testHasherCorrectness(bmt, hasher, data, n, segmentCount) if err != nil { t.Fatal(err) } @@ -241,7 +248,7 @@ func testHasherReuse(poolsize int, t *testing.T) { // Tests if pool can be cleanly reused even in concurrent use by several hasher func TestBMTConcurrentUse(t *testing.T) { hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, PoolSize) + pool := NewTreePool(hasher, segmentCount, PoolSize) defer pool.Drain(0) cycles := 100 errc := make(chan error) @@ -451,7 +458,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) { func benchmarkBMT(t *testing.B, n int) { data := newData(n) hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, PoolSize) + pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool) t.ReportAllocs() @@ -465,7 +472,7 @@ func benchmarkBMT(t *testing.B, n int) { func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { data := newData(n) hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, PoolSize) + pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool).NewAsyncWriter(double) idxs, segments := splitAndShuffle(bmt.SectionSize(), data) shuffle(len(idxs), func(i int, j int) { @@ -483,7 +490,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { func benchmarkPool(t *testing.B, poolsize, n int) { data := newData(n) hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, poolsize) + pool := NewTreePool(hasher, segmentCount, poolsize) cycles := 100 t.ReportAllocs() diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go new file mode 100644 index 0000000000..1449efccd0 --- /dev/null +++ b/swarm/chunk/chunk.go @@ -0,0 +1,5 @@ +package chunk + +const ( + DefaultSize = 4096 +) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index fa210e3007..36040339d3 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/discover" + cp "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/spancontext" @@ -229,6 +230,11 @@ R: for req := range d.receiveC { processReceivedChunksCount.Inc(1) + if len(req.SData) > cp.DefaultSize+8 { + log.Warn("received chunk is bigger than expected", "len", len(req.SData)) + continue R + } + // this should be has locally chunk, err := d.db.Get(context.TODO(), req.Addr) if err == nil { @@ -244,6 +250,7 @@ R: continue R default: } + chunk.SData = req.SData d.db.Put(context.TODO(), chunk) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index b9b5022734..6d805b8e29 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/spancontext" opentracing "github.com/opentracing/opentracing-go" @@ -69,10 +70,6 @@ var ( errOperationTimedOut = errors.New("operation timed out") ) -const ( - DefaultChunkSize int64 = 4096 -) - type ChunkerParams struct { chunkSize int64 hashSize int64 @@ -136,7 +133,7 @@ type TreeChunker struct { func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader { jp := &JoinerParams{ ChunkerParams: ChunkerParams{ - chunkSize: DefaultChunkSize, + chunkSize: chunk.DefaultSize, hashSize: int64(len(addr)), }, addr: addr, @@ -156,7 +153,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) ( tsp := &TreeSplitterParams{ SplitterParams: SplitterParams{ ChunkerParams: ChunkerParams{ - chunkSize: DefaultChunkSize, + chunkSize: chunk.DefaultSize, hashSize: putter.RefSize(), }, reader: data, diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 139c0ee031..bc23077c18 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/encryption" ) @@ -57,7 +58,7 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) refSize := int64(hashSize) if toEncrypt { refSize += encryption.KeyLength - chunkEncryption = newChunkEncryption(DefaultChunkSize, refSize) + chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize) } return &hasherStore{ @@ -190,9 +191,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp // removing extra bytes which were just added for padding length := ChunkData(decryptedSpan).Size() - for length > DefaultChunkSize { - length = length + (DefaultChunkSize - 1) - length = length / DefaultChunkSize + for length > chunk.DefaultSize { + length = length + (chunk.DefaultSize - 1) + length = length / chunk.DefaultSize length *= h.refSize } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index baf9e8c142..5ee88baa51 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" @@ -184,7 +185,7 @@ func testIterator(t *testing.T, mock bool) { t.Fatalf("init dbStore failed: %v", err) } - chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount) + chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount) wg := &sync.WaitGroup{} wg.Add(len(chunks)) @@ -294,7 +295,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { chunks := []*Chunk{} for i := 0; i < n; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -344,7 +345,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) { chunks := []*Chunk{} for i := 0; i < n; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -398,7 +399,7 @@ func TestLDBStoreAddRemove(t *testing.T) { chunks := []*Chunk{} for i := 0; i < n; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -460,7 +461,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { chunks := []*Chunk{} for i := 0; i < capacity; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 096d150ae3..9e34749797 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -98,20 +98,16 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { // After the LDBStore.Put, it is ensured that the MemStore // contains the chunk with the same data, but nil ReqC channel. func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk) { - if l := len(chunk.SData); l < 9 { - log.Debug("incomplete chunk data", "addr", chunk.Addr, "length", l) - chunk.SetErrored(ErrChunkInvalid) - chunk.markAsStored() - return - } valid := true + // ls.Validators contains a list of one validator per chunk type. + // if one validator succeeds, then the chunk is valid for _, v := range ls.Validators { if valid = v.Validate(chunk.Addr, chunk.SData); valid { break } } if !valid { - log.Trace("invalid content address", "addr", chunk.Addr) + log.Trace("invalid chunk", "addr", chunk.Addr, "len", len(chunk.SData)) chunk.SetErrored(ErrChunkInvalid) chunk.markAsStored() return diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 2bb81efa3a..ae62218fe8 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -20,6 +20,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/ethereum/go-ethereum/swarm/chunk" ) var ( @@ -61,7 +63,7 @@ func TestValidator(t *testing.T) { // add content address validator and check puts // bad should fail, good should pass store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc)) - chunks = GenerateRandomChunks(DefaultChunkSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.SData, goodChunk.SData) @@ -79,7 +81,7 @@ func TestValidator(t *testing.T) { var negV boolTestValidator store.Validators = append(store.Validators, negV) - chunks = GenerateRandomChunks(DefaultChunkSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.SData, goodChunk.SData) @@ -97,7 +99,7 @@ func TestValidator(t *testing.T) { var posV boolTestValidator = true store.Validators = append(store.Validators, posV) - chunks = GenerateRandomChunks(DefaultChunkSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.SData, goodChunk.SData) diff --git a/swarm/storage/mru/handler.go b/swarm/storage/mru/handler.go index 32f43d5029..57561fd14b 100644 --- a/swarm/storage/mru/handler.go +++ b/swarm/storage/mru/handler.go @@ -21,17 +21,15 @@ package mru import ( "bytes" "context" - "fmt" "sync" "time" "unsafe" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" ) -const chunkSize = 4096 // temporary until we implement FileStore in the resourcehandler - type Handler struct { chunkStore *storage.NetStore HashSize int @@ -66,8 +64,7 @@ func init() { } // NewHandler creates a new Mutable Resource API -func NewHandler(params *HandlerParams) (*Handler, error) { - +func NewHandler(params *HandlerParams) *Handler { rh := &Handler{ resources: make(map[uint64]*resource), storeTimeout: defaultStoreTimeout, @@ -82,7 +79,7 @@ func NewHandler(params *HandlerParams) (*Handler, error) { hashPool.Put(hashfunc) } - return rh, nil + return rh } // SetStore sets the store backend for the Mutable Resource API @@ -94,9 +91,8 @@ func (h *Handler) SetStore(store *storage.NetStore) { // If it looks like a resource update, the chunk address is checked against the ownerAddr of the update's signature // It implements the storage.ChunkValidator interface func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { - dataLength := len(data) - if dataLength < minimumChunkLength { + if dataLength < minimumChunkLength || dataLength > chunk.DefaultSize+8 { return false } @@ -106,7 +102,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { rootAddr, _ := metadataHash(data) valid := bytes.Equal(chunkAddr, rootAddr) if !valid { - log.Debug(fmt.Sprintf("Invalid root metadata chunk with address: %s", chunkAddr.Hex())) + log.Debug("Invalid root metadata chunk with address", "addr", chunkAddr.Hex()) } return valid } @@ -118,7 +114,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // First, deserialize the chunk var r SignedResourceUpdate if err := r.fromChunk(chunkAddr, data); err != nil { - log.Debug("Invalid resource chunk with address %s: %s ", chunkAddr.Hex(), err.Error()) + log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error()) return false } @@ -126,7 +122,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // that was used to retrieve this chunk // if this validation fails, someone forged a chunk. if !bytes.Equal(chunkAddr, r.updateHeader.UpdateAddr()) { - log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr %s", chunkAddr.Hex()) + log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr", "addr", chunkAddr.Hex()) return false } @@ -134,7 +130,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // If it fails, it means either the signature is not valid, data is corrupted // or someone is trying to update someone else's resource. if err := r.Verify(); err != nil { - log.Debug("Invalid signature: %v", err) + log.Debug("Invalid signature", "err", err) return false } @@ -172,11 +168,6 @@ func (h *Handler) GetVersion(rootAddr storage.Address) (uint32, error) { return rsrc.version, nil } -// \TODO should be hashsize * branches from the chosen chunker, implement with FileStore -func (h *Handler) chunkSize() int64 { - return chunkSize -} - // New creates a new metadata chunk out of the request passed in. func (h *Handler) New(ctx context.Context, request *Request) error { diff --git a/swarm/storage/mru/resource_test.go b/swarm/storage/mru/resource_test.go index 95c9eccdfc..76d7c58a1e 100644 --- a/swarm/storage/mru/resource_test.go +++ b/swarm/storage/mru/resource_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -776,14 +777,11 @@ func TestValidatorInStore(t *testing.T) { // set up resource handler and add is as a validator to the localstore rhParams := &HandlerParams{} - rh, err := NewHandler(rhParams) - if err != nil { - t.Fatal(err) - } + rh := NewHandler(rhParams) store.Validators = append(store.Validators, rh) // create content addressed chunks, one good, one faulty - chunks := storage.GenerateRandomChunks(storage.DefaultChunkSize, 2) + chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk := chunks[0] badChunk := chunks[1] badChunk.SData = goodChunk.SData diff --git a/swarm/storage/mru/testutil.go b/swarm/storage/mru/testutil.go index 751f51af32..6efcba9aba 100644 --- a/swarm/storage/mru/testutil.go +++ b/swarm/storage/mru/testutil.go @@ -38,10 +38,7 @@ func (t *TestHandler) Close() { // NewTestHandler creates Handler object to be used for testing purposes. func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) { path := filepath.Join(datadir, testDbDirName) - rh, err := NewHandler(params) - if err != nil { - return nil, fmt.Errorf("resource handler create fail: %v", err) - } + rh := NewHandler(params) localstoreparams := storage.NewDefaultLocalStoreParams() localstoreparams.Init(path) localStore, err := storage.NewLocalStore(localstoreparams, nil) diff --git a/swarm/storage/mru/update.go b/swarm/storage/mru/update.go index 88c4ac4e59..d1bd37ddff 100644 --- a/swarm/storage/mru/update.go +++ b/swarm/storage/mru/update.go @@ -20,6 +20,7 @@ import ( "encoding/binary" "errors" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/multihash" ) @@ -42,7 +43,7 @@ const chunkPrefixLength = 2 + 2 // // Minimum size is Header + 1 (minimum data length, enforced) const minimumUpdateDataLength = updateHeaderLength + 1 -const maxUpdateDataLength = chunkSize - signatureLength - updateHeaderLength - chunkPrefixLength +const maxUpdateDataLength = chunk.DefaultSize - signatureLength - updateHeaderLength - chunkPrefixLength // binaryPut serializes the resource update information into the given slice func (r *resourceUpdate) binaryPut(serializedData []byte) error { diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 2923c81c5a..36ff66d045 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" ) @@ -101,11 +102,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get New chunks to store are store using the putter which the caller provides. */ func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx) } func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx) } // Entry to create a tree node diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 3114ef5767..53e3af485a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/swarm/bmt" + "github.com/ethereum/go-ethereum/swarm/chunk" ) const MaxPO = 16 @@ -114,7 +115,9 @@ func MakeHashFunc(hash string) SwarmHasher { case "BMT": return func() SwarmHash { hasher := sha3.NewKeccak256 - pool := bmt.NewTreePool(hasher, bmt.SegmentCount, bmt.PoolSize) + hasherSize := hasher().Size() + segmentCount := chunk.DefaultSize / hasherSize + pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize) return bmt.New(pool) } } @@ -230,8 +233,8 @@ func GenerateRandomChunk(dataSize int64) *Chunk { func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) { var i int hasher := MakeHashFunc(DefaultHash)() - if dataSize > DefaultChunkSize { - dataSize = DefaultChunkSize + if dataSize > chunk.DefaultSize { + dataSize = chunk.DefaultSize } for i = 0; i < count; i++ { @@ -345,6 +348,10 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { // Validate that the given key is a valid content address for the given data func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { + if l := len(data); l < 9 || l > chunk.DefaultSize+8 { + return false + } + hasher := v.Hasher() hasher.ResetWithLength(data[:8]) hasher.Write(data[8:]) diff --git a/swarm/swarm.go b/swarm/swarm.go index c380a376f6..f731ff33d7 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -195,18 +195,13 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e var resourceHandler *mru.Handler rhparams := &mru.HandlerParams{} - resourceHandler, err = mru.NewHandler(rhparams) - if err != nil { - return nil, err - } + resourceHandler = mru.NewHandler(rhparams) resourceHandler.SetStore(netStore) - var validators []storage.ChunkValidator - validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash))) - if resourceHandler != nil { - validators = append(validators, resourceHandler) + self.lstore.Validators = []storage.ChunkValidator{ + storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)), + resourceHandler, } - self.lstore.Validators = validators // setup local store log.Debug(fmt.Sprintf("Set up local storage")) From e0e0e53401e93733d921338b6d794162c40a7883 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 14 Aug 2018 23:30:42 +0800 Subject: [PATCH 017/126] crypto: change formula for create2 (#17393) --- core/vm/evm.go | 2 +- crypto/crypto.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index a2722537dd..a24f6f3865 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -427,7 +427,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I // Create2 creates a new contract using code as deployment code. // -// The different between Create2 with Create is Create2 uses sha3(msg.sender ++ salt ++ init_code)[12:] +// The different between Create2 with Create is Create2 uses sha3(0xff ++ msg.sender ++ salt ++ sha3(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), code) diff --git a/crypto/crypto.go b/crypto/crypto.go index dec6e3c19e..3211957e0a 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -78,8 +78,8 @@ func CreateAddress(b common.Address, nonce uint64) common.Address { // CreateAddress2 creates an ethereum address given the address bytes, initial // contract code and a salt. -func CreateAddress2(b common.Address, salt common.Hash, code []byte) common.Address { - return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt.Bytes(), code)[12:]) +func CreateAddress2(b common.Address, salt [32]byte, code []byte) common.Address { + return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], Keccak256(code))[12:]) } // ToECDSA creates a private key with the given D value. From a1783d169732dd34aa8c7d68f411ce741c1a5015 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 14 Aug 2018 23:34:33 +0800 Subject: [PATCH 018/126] miner: move agent logic to worker (#17351) * miner: move agent logic to worker * miner: polish * core: persist block before reorg --- core/blockchain.go | 7 +- miner/agent.go | 116 ----- miner/miner.go | 64 +-- miner/worker.go | 1023 +++++++++++++++++++++++------------------- miner/worker_test.go | 212 +++++++++ 5 files changed, 797 insertions(+), 625 deletions(-) delete mode 100644 miner/agent.go create mode 100644 miner/worker_test.go diff --git a/core/blockchain.go b/core/blockchain.go index 62dc261250..0461da7fd9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -899,9 +899,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { return NonStatTy, err } - // Write other block data using a batch. - batch := bc.db.NewBatch() - rawdb.WriteBlock(batch, block) + rawdb.WriteBlock(bc.db, block) root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { @@ -955,6 +953,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } } } + + // Write other block data using a batch. + batch := bc.db.NewBatch() rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) // If the total difficulty is higher than our known, add it to the canonical chain diff --git a/miner/agent.go b/miner/agent.go deleted file mode 100644 index e922ea153c..0000000000 --- a/miner/agent.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package miner - -import ( - "sync" - "sync/atomic" - - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/log" -) - -type CpuAgent struct { - mu sync.Mutex - - taskCh chan *Package - returnCh chan<- *Package - stop chan struct{} - quitCurrentOp chan struct{} - - chain consensus.ChainReader - engine consensus.Engine - - started int32 // started indicates whether the agent is currently started -} - -func NewCpuAgent(chain consensus.ChainReader, engine consensus.Engine) *CpuAgent { - agent := &CpuAgent{ - chain: chain, - engine: engine, - stop: make(chan struct{}, 1), - taskCh: make(chan *Package, 1), - } - return agent -} - -func (self *CpuAgent) AssignTask(p *Package) { - if atomic.LoadInt32(&self.started) == 1 { - self.taskCh <- p - } -} -func (self *CpuAgent) DeliverTo(ch chan<- *Package) { self.returnCh = ch } - -func (self *CpuAgent) Start() { - if !atomic.CompareAndSwapInt32(&self.started, 0, 1) { - return // agent already started - } - go self.update() -} - -func (self *CpuAgent) Stop() { - if !atomic.CompareAndSwapInt32(&self.started, 1, 0) { - return // agent already stopped - } - self.stop <- struct{}{} -done: - // Empty work channel - for { - select { - case <-self.taskCh: - default: - break done - } - } -} - -func (self *CpuAgent) update() { -out: - for { - select { - case p := <-self.taskCh: - self.mu.Lock() - if self.quitCurrentOp != nil { - close(self.quitCurrentOp) - } - self.quitCurrentOp = make(chan struct{}) - go self.mine(p, self.quitCurrentOp) - self.mu.Unlock() - case <-self.stop: - self.mu.Lock() - if self.quitCurrentOp != nil { - close(self.quitCurrentOp) - self.quitCurrentOp = nil - } - self.mu.Unlock() - break out - } - } -} - -func (self *CpuAgent) mine(p *Package, stop <-chan struct{}) { - var err error - if p.Block, err = self.engine.Seal(self.chain, p.Block, stop); p.Block != nil { - log.Info("Successfully sealed new block", "number", p.Block.Number(), "hash", p.Block.Hash()) - self.returnCh <- p - } else { - if err != nil { - log.Warn("Block sealing failed", "err", err) - } - self.returnCh <- nil - } -} diff --git a/miner/miner.go b/miner/miner.go index 4c5717c8ad..e350e456e9 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -21,14 +21,12 @@ import ( "fmt" "sync/atomic" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -36,10 +34,8 @@ import ( // Backend wraps all methods required for mining. type Backend interface { - AccountManager() *accounts.Manager BlockChain() *core.BlockChain TxPool() *core.TxPool - ChainDb() ethdb.Database } // Miner creates blocks and searches for proof-of-work values. @@ -49,6 +45,7 @@ type Miner struct { coinbase common.Address eth Backend engine consensus.Engine + exitCh chan struct{} canStart int32 // can start indicates whether we can start the mining operation shouldStart int32 // should start indicates whether we should start after sync @@ -59,10 +56,10 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con eth: eth, mux: mux, engine: engine, + exitCh: make(chan struct{}), worker: newWorker(config, engine, eth, mux), canStart: 1, } - miner.Register(NewCpuAgent(eth.BlockChain(), engine)) go miner.update() return miner @@ -74,28 +71,35 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con // and halt your mining operation for as long as the DOS continues. func (self *Miner) update() { events := self.mux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{}) -out: - for ev := range events.Chan() { - switch ev.Data.(type) { - case downloader.StartEvent: - atomic.StoreInt32(&self.canStart, 0) - if self.Mining() { - self.Stop() - atomic.StoreInt32(&self.shouldStart, 1) - log.Info("Mining aborted due to sync") - } - case downloader.DoneEvent, downloader.FailedEvent: - shouldStart := atomic.LoadInt32(&self.shouldStart) == 1 + defer events.Unsubscribe() - atomic.StoreInt32(&self.canStart, 1) - atomic.StoreInt32(&self.shouldStart, 0) - if shouldStart { - self.Start(self.coinbase) + for { + select { + case ev := <-events.Chan(): + if ev == nil { + return } - // unsubscribe. we're only interested in this event once - events.Unsubscribe() - // stop immediately and ignore all further pending events - break out + switch ev.Data.(type) { + case downloader.StartEvent: + atomic.StoreInt32(&self.canStart, 0) + if self.Mining() { + self.Stop() + atomic.StoreInt32(&self.shouldStart, 1) + log.Info("Mining aborted due to sync") + } + case downloader.DoneEvent, downloader.FailedEvent: + shouldStart := atomic.LoadInt32(&self.shouldStart) == 1 + + atomic.StoreInt32(&self.canStart, 1) + atomic.StoreInt32(&self.shouldStart, 0) + if shouldStart { + self.Start(self.coinbase) + } + // stop immediately and ignore all further pending events + return + } + case <-self.exitCh: + return } } } @@ -109,7 +113,6 @@ func (self *Miner) Start(coinbase common.Address) { return } self.worker.start() - self.worker.commitNewWork() } func (self *Miner) Stop() { @@ -117,12 +120,9 @@ func (self *Miner) Stop() { atomic.StoreInt32(&self.shouldStart, 0) } -func (self *Miner) Register(agent Agent) { - self.worker.register(agent) -} - -func (self *Miner) Unregister(agent Agent) { - self.worker.unregister(agent) +func (self *Miner) Close() { + self.worker.close() + close(self.exitCh) } func (self *Miner) Mining() bool { diff --git a/miner/worker.go b/miner/worker.go index ae695f0198..81a63c29a4 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -32,16 +32,14 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) const ( - resultQueueSize = 10 - miningLogAtDepth = 5 - + // resultQueueSize is the size of channel listening to sealing result. + resultQueueSize = 10 // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 @@ -49,17 +47,10 @@ const ( chainHeadChanSize = 10 // chainSideChanSize is the size of channel listening to ChainSideEvent. chainSideChanSize = 10 + miningLogAtDepth = 5 ) -// Agent can register themselves with the worker -type Agent interface { - AssignTask(*Package) - DeliverTo(chan<- *Package) - Start() - Stop() -} - -// Env is the workers current environment and holds all of the current state information. +// Env is the worker's current environment and holds all of the current state information. type Env struct { config *params.ChainConfig signer types.Signer @@ -74,461 +65,20 @@ type Env struct { header *types.Header txs []*types.Transaction receipts []*types.Receipt - - createdAt time.Time } -// Package contains all information for consensus engine sealing and result submitting. -type Package struct { - Receipts []*types.Receipt - State *state.StateDB - Block *types.Block -} +func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { + snap := env.state.Snapshot() -// worker is the main object which takes care of applying messages to the new state -type worker struct { - config *params.ChainConfig - engine consensus.Engine - - mu sync.Mutex - - // update loop - mux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - chainSideCh chan core.ChainSideEvent - chainSideSub event.Subscription - - agents map[Agent]struct{} - recv chan *Package - - eth Backend - chain *core.BlockChain - proc core.Validator - chainDb ethdb.Database - - coinbase common.Address - extra []byte - - currentMu sync.Mutex - current *Env - - snapshotMu sync.RWMutex - snapshotBlock *types.Block - snapshotState *state.StateDB - - uncleMu sync.Mutex - possibleUncles map[common.Hash]*types.Block - - unconfirmed *unconfirmedBlocks // set of locally mined blocks pending canonicalness confirmations - - // atomic status counters - running int32 // The indicator whether the consensus engine is running or not. -} - -func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { - worker := &worker{ - config: config, - engine: engine, - eth: eth, - mux: mux, - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), - chainDb: eth.ChainDb(), - recv: make(chan *Package, resultQueueSize), - chain: eth.BlockChain(), - proc: eth.BlockChain().Validator(), - possibleUncles: make(map[common.Hash]*types.Block), - agents: make(map[Agent]struct{}), - unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - } - // Subscribe NewTxsEvent for tx pool - worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) - // Subscribe events for blockchain - worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) - worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) - go worker.update() - - go worker.wait() - worker.commitNewWork() - - return worker -} - -func (self *worker) setEtherbase(addr common.Address) { - self.mu.Lock() - defer self.mu.Unlock() - self.coinbase = addr -} - -func (self *worker) setExtra(extra []byte) { - self.mu.Lock() - defer self.mu.Unlock() - self.extra = extra -} - -func (self *worker) pending() (*types.Block, *state.StateDB) { - // return a snapshot to avoid contention on currentMu mutex - self.snapshotMu.RLock() - defer self.snapshotMu.RUnlock() - return self.snapshotBlock, self.snapshotState.Copy() -} - -func (self *worker) pendingBlock() *types.Block { - // return a snapshot to avoid contention on currentMu mutex - self.snapshotMu.RLock() - defer self.snapshotMu.RUnlock() - return self.snapshotBlock -} - -func (self *worker) start() { - self.mu.Lock() - defer self.mu.Unlock() - atomic.StoreInt32(&self.running, 1) - for agent := range self.agents { - agent.Start() - } -} - -func (self *worker) stop() { - self.mu.Lock() - defer self.mu.Unlock() - - atomic.StoreInt32(&self.running, 0) - for agent := range self.agents { - agent.Stop() - } -} - -func (self *worker) isRunning() bool { - return atomic.LoadInt32(&self.running) == 1 -} - -func (self *worker) register(agent Agent) { - self.mu.Lock() - defer self.mu.Unlock() - self.agents[agent] = struct{}{} - agent.DeliverTo(self.recv) - if self.isRunning() { - agent.Start() - } -} - -func (self *worker) unregister(agent Agent) { - self.mu.Lock() - defer self.mu.Unlock() - delete(self.agents, agent) - agent.Stop() -} - -func (self *worker) update() { - defer self.txsSub.Unsubscribe() - defer self.chainHeadSub.Unsubscribe() - defer self.chainSideSub.Unsubscribe() - - for { - // A real event arrived, process interesting content - select { - // Handle ChainHeadEvent - case <-self.chainHeadCh: - self.commitNewWork() - - // Handle ChainSideEvent - case ev := <-self.chainSideCh: - self.uncleMu.Lock() - self.possibleUncles[ev.Block.Hash()] = ev.Block - self.uncleMu.Unlock() - - // Handle NewTxsEvent - case ev := <-self.txsCh: - // Apply transactions to the pending state if we're not mining. - // - // Note all transactions received may not be continuous with transactions - // already included in the current mining block. These transactions will - // be automatically eliminated. - if !self.isRunning() { - self.currentMu.Lock() - txs := make(map[common.Address]types.Transactions) - for _, tx := range ev.Txs { - acc, _ := types.Sender(self.current.signer, tx) - txs[acc] = append(txs[acc], tx) - } - txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) - self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) - self.updateSnapshot() - self.currentMu.Unlock() - } else { - // If we're mining, but nothing is being processed, wake on new transactions - if self.config.Clique != nil && self.config.Clique.Period == 0 { - self.commitNewWork() - } - } - - // System stopped - case <-self.txsSub.Err(): - return - case <-self.chainHeadSub.Err(): - return - case <-self.chainSideSub.Err(): - return - } - } -} - -func (self *worker) wait() { - for { - for result := range self.recv { - - if result == nil { - continue - } - block := result.Block - - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - for _, r := range result.Receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } - } - for _, log := range result.State.Logs() { - log.BlockHash = block.Hash() - } - self.currentMu.Lock() - stat, err := self.chain.WriteBlockWithState(block, result.Receipts, result.State) - self.currentMu.Unlock() - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - // Broadcast the block and announce chain insertion event - self.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - logs = result.State.Logs() - ) - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - if stat == core.CanonStatTy { - events = append(events, core.ChainHeadEvent{Block: block}) - } - self.chain.PostChainEvents(events, logs) - - // Insert the block into the set of pending ones to wait for confirmations - self.unconfirmed.Insert(block.NumberU64(), block.Hash()) - } - } -} - -// push sends a new work task to currently live miner agents. -func (self *worker) push(p *Package) { - for agent := range self.agents { - agent.AssignTask(p) - } -} - -// makeCurrent creates a new environment for the current cycle. -func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error { - state, err := self.chain.StateAt(parent.Root()) + receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) if err != nil { - return err - } - env := &Env{ - config: self.config, - signer: types.NewEIP155Signer(self.config.ChainID), - state: state, - ancestors: mapset.NewSet(), - family: mapset.NewSet(), - uncles: mapset.NewSet(), - header: header, - createdAt: time.Now(), + env.state.RevertToSnapshot(snap) + return err, nil } + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) - // when 08 is processed ancestors contain 07 (quick block) - for _, ancestor := range self.chain.GetBlocksFromHash(parent.Hash(), 7) { - for _, uncle := range ancestor.Uncles() { - env.family.Add(uncle.Hash()) - } - env.family.Add(ancestor.Hash()) - env.ancestors.Add(ancestor.Hash()) - } - - // Keep track of transactions which return errors so they can be removed - env.tcount = 0 - self.current = env - return nil -} - -func (self *worker) commitNewWork() { - self.mu.Lock() - defer self.mu.Unlock() - self.uncleMu.Lock() - defer self.uncleMu.Unlock() - self.currentMu.Lock() - defer self.currentMu.Unlock() - - tstart := time.Now() - parent := self.chain.CurrentBlock() - - tstamp := tstart.Unix() - if parent.Time().Cmp(new(big.Int).SetInt64(tstamp)) >= 0 { - tstamp = parent.Time().Int64() + 1 - } - // this will ensure we're not going off too far in the future - if now := time.Now().Unix(); tstamp > now+1 { - wait := time.Duration(tstamp-now) * time.Second - log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) - time.Sleep(wait) - } - - num := parent.Number() - header := &types.Header{ - ParentHash: parent.Hash(), - Number: num.Add(num, common.Big1), - GasLimit: core.CalcGasLimit(parent), - Extra: self.extra, - Time: big.NewInt(tstamp), - } - // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) - if self.isRunning() { - if self.coinbase == (common.Address{}) { - log.Error("Refusing to mine without etherbase") - return - } - header.Coinbase = self.coinbase - } - if err := self.engine.Prepare(self.chain, header); err != nil { - log.Error("Failed to prepare header for mining", "err", err) - return - } - // If we are care about TheDAO hard-fork check whether to override the extra-data or not - if daoBlock := self.config.DAOForkBlock; daoBlock != nil { - // Check whether the block is among the fork extra-override range - limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange) - if header.Number.Cmp(daoBlock) >= 0 && header.Number.Cmp(limit) < 0 { - // Depending whether we support or oppose the fork, override differently - if self.config.DAOForkSupport { - header.Extra = common.CopyBytes(params.DAOForkBlockExtra) - } else if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - header.Extra = []byte{} // If miner opposes, don't let it use the reserved extra-data - } - } - } - // Could potentially happen if starting to mine in an odd state. - err := self.makeCurrent(parent, header) - if err != nil { - log.Error("Failed to create mining context", "err", err) - return - } - // Create the current work task and check any fork transitions needed - env := self.current - if self.config.DAOForkSupport && self.config.DAOForkBlock != nil && self.config.DAOForkBlock.Cmp(header.Number) == 0 { - misc.ApplyDAOHardFork(env.state) - } - - // compute uncles for the new block. - var ( - uncles []*types.Header - badUncles []common.Hash - ) - for hash, uncle := range self.possibleUncles { - if len(uncles) == 2 { - break - } - if err := self.commitUncle(env, uncle.Header()); err != nil { - log.Trace("Bad uncle found and will be removed", "hash", hash) - log.Trace(fmt.Sprint(uncle)) - - badUncles = append(badUncles, hash) - } else { - log.Debug("Committing new uncle to block", "hash", hash) - uncles = append(uncles, uncle.Header()) - } - } - for _, hash := range badUncles { - delete(self.possibleUncles, hash) - } - - var ( - emptyBlock *types.Block - fullBlock *types.Block - ) - - // Create an empty block based on temporary copied state for sealing in advance without waiting block - // execution finished. - emptyState := env.state.Copy() - if emptyBlock, err = self.engine.Finalize(self.chain, header, emptyState, nil, uncles, nil); err != nil { - log.Error("Failed to finalize block for temporary sealing", "err", err) - } else { - // Push empty work in advance without applying pending transaction. - // The reason is transactions execution can cost a lot and sealer need to - // take advantage of this part time. - if self.isRunning() { - log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) - self.push(&Package{nil, emptyState, emptyBlock}) - } - } - - // Fill the block with all available pending transactions. - pending, err := self.eth.TxPool().Pending() - if err != nil { - log.Error("Failed to fetch pending transactions", "err", err) - return - } - txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) - env.commitTransactions(self.mux, txs, self.chain, self.coinbase) - - // Create the full block to seal with the consensus engine - if fullBlock, err = self.engine.Finalize(self.chain, header, env.state, env.txs, uncles, env.receipts); err != nil { - log.Error("Failed to finalize block for sealing", "err", err) - return - } - // We only care about logging if we're actually mining. - if self.isRunning() { - log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) - self.unconfirmed.Shift(fullBlock.NumberU64() - 1) - self.push(&Package{env.receipts, env.state, fullBlock}) - } - self.updateSnapshot() -} - -func (self *worker) commitUncle(env *Env, uncle *types.Header) error { - hash := uncle.Hash() - if env.uncles.Contains(hash) { - return fmt.Errorf("uncle not unique") - } - if !env.ancestors.Contains(uncle.ParentHash) { - return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) - } - if env.family.Contains(hash) { - return fmt.Errorf("uncle already in family (%x)", hash) - } - env.uncles.Add(uncle.Hash()) - return nil -} - -func (self *worker) updateSnapshot() { - self.snapshotMu.Lock() - defer self.snapshotMu.Unlock() - - var uncles []*types.Header - self.current.uncles.Each(func(item interface{}) bool { - if header, ok := item.(*types.Header); ok { - uncles = append(uncles, header) - return true - } - return false - }) - - self.snapshotBlock = types.NewBlock( - self.current.header, - self.current.txs, - uncles, - self.current.receipts, - ) - self.snapshotState = self.current.state.Copy() + return nil, receipt.Logs } func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { @@ -616,16 +166,541 @@ func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsBy } } -func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { - snap := env.state.Snapshot() - - receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) - if err != nil { - env.state.RevertToSnapshot(snap) - return err, nil - } - env.txs = append(env.txs, tx) - env.receipts = append(env.receipts, receipt) - - return nil, receipt.Logs +// task contains all information for consensus engine sealing and result submitting. +type task struct { + receipts []*types.Receipt + state *state.StateDB + block *types.Block + createdAt time.Time +} + +// worker is the main object which takes care of submitting new work to consensus engine +// and gathering the sealing result. +type worker struct { + config *params.ChainConfig + engine consensus.Engine + eth Backend + chain *core.BlockChain + + // Subscriptions + mux *event.TypeMux + txsCh chan core.NewTxsEvent + txsSub event.Subscription + chainHeadCh chan core.ChainHeadEvent + chainHeadSub event.Subscription + chainSideCh chan core.ChainSideEvent + chainSideSub event.Subscription + + // Channels + newWork chan struct{} + taskCh chan *task + resultCh chan *task + exitCh chan struct{} + + current *Env // An environment for current running cycle. + possibleUncles map[common.Hash]*types.Block // A set of side blocks as the possible uncle blocks. + unconfirmed *unconfirmedBlocks // A set of locally mined blocks pending canonicalness confirmations. + + mu sync.RWMutex // The lock used to protect the coinbase and extra fields + coinbase common.Address + extra []byte + + snapshotMu sync.RWMutex // The lock used to protect the block snapshot and state snapshot + snapshotBlock *types.Block + snapshotState *state.StateDB + + // atomic status counters + running int32 // The indicator whether the consensus engine is running or not. + + // Test hooks + newTaskHook func(*task) // Method to call upon receiving a new sealing task + fullTaskInterval func() // Method to call before pushing the full sealing task +} + +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { + worker := &worker{ + config: config, + engine: engine, + eth: eth, + mux: mux, + chain: eth.BlockChain(), + possibleUncles: make(map[common.Hash]*types.Block), + unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), + txsCh: make(chan core.NewTxsEvent, txChanSize), + chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), + chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), + newWork: make(chan struct{}, 1), + taskCh: make(chan *task), + resultCh: make(chan *task, resultQueueSize), + exitCh: make(chan struct{}), + } + // Subscribe NewTxsEvent for tx pool + worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) + // Subscribe events for blockchain + worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) + worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) + + go worker.mainLoop() + go worker.resultLoop() + go worker.taskLoop() + + // Submit first work to initialize pending state. + worker.newWork <- struct{}{} + return worker +} + +// setEtherbase sets the etherbase used to initialize the block coinbase field. +func (w *worker) setEtherbase(addr common.Address) { + w.mu.Lock() + defer w.mu.Unlock() + w.coinbase = addr +} + +// setExtra sets the content used to initialize the block extra field. +func (w *worker) setExtra(extra []byte) { + w.mu.Lock() + defer w.mu.Unlock() + w.extra = extra +} + +// pending returns the pending state and corresponding block. +func (w *worker) pending() (*types.Block, *state.StateDB) { + // return a snapshot to avoid contention on currentMu mutex + w.snapshotMu.RLock() + defer w.snapshotMu.RUnlock() + if w.snapshotState == nil { + return nil, nil + } + return w.snapshotBlock, w.snapshotState.Copy() +} + +// pendingBlock returns pending block. +func (w *worker) pendingBlock() *types.Block { + // return a snapshot to avoid contention on currentMu mutex + w.snapshotMu.RLock() + defer w.snapshotMu.RUnlock() + return w.snapshotBlock +} + +// start sets the running status as 1 and triggers new work submitting. +func (w *worker) start() { + atomic.StoreInt32(&w.running, 1) + w.newWork <- struct{}{} +} + +// stop sets the running status as 0. +func (w *worker) stop() { + atomic.StoreInt32(&w.running, 0) +} + +// isRunning returns an indicator whether worker is running or not. +func (w *worker) isRunning() bool { + return atomic.LoadInt32(&w.running) == 1 +} + +// close terminates all background threads maintained by the worker and cleans up buffered channels. +// Note the worker does not support being closed multiple times. +func (w *worker) close() { + close(w.exitCh) + // Clean up buffered channels + for empty := false; !empty; { + select { + case <-w.resultCh: + default: + empty = true + } + } +} + +// mainLoop is a standalone goroutine to regenerate the sealing task based on the received event. +func (w *worker) mainLoop() { + defer w.txsSub.Unsubscribe() + defer w.chainHeadSub.Unsubscribe() + defer w.chainSideSub.Unsubscribe() + + for { + select { + case <-w.newWork: + // Submit a work when the worker is created or started. + w.commitNewWork() + + case <-w.chainHeadCh: + // Resubmit a work for new cycle once worker receives chain head event. + w.commitNewWork() + + case ev := <-w.chainSideCh: + // Add side block to possible uncle block set. + w.possibleUncles[ev.Block.Hash()] = ev.Block + + case ev := <-w.txsCh: + // Apply transactions to the pending state if we're not mining. + // + // Note all transactions received may not be continuous with transactions + // already included in the current mining block. These transactions will + // be automatically eliminated. + if !w.isRunning() && w.current != nil { + w.mu.Lock() + coinbase := w.coinbase + w.mu.Unlock() + + txs := make(map[common.Address]types.Transactions) + for _, tx := range ev.Txs { + acc, _ := types.Sender(w.current.signer, tx) + txs[acc] = append(txs[acc], tx) + } + txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs) + w.current.commitTransactions(w.mux, txset, w.chain, coinbase) + w.updateSnapshot() + } else { + // If we're mining, but nothing is being processed, wake on new transactions + if w.config.Clique != nil && w.config.Clique.Period == 0 { + w.commitNewWork() + } + } + + // System stopped + case <-w.exitCh: + return + case <-w.txsSub.Err(): + return + case <-w.chainHeadSub.Err(): + return + case <-w.chainSideSub.Err(): + return + } + } +} + +// seal pushes a sealing task to consensus engine and submits the result. +func (w *worker) seal(t *task, stop <-chan struct{}) { + var ( + err error + res *task + ) + + if t.block, err = w.engine.Seal(w.chain, t.block, stop); t.block != nil { + log.Info("Successfully sealed new block", "number", t.block.Number(), "hash", t.block.Hash(), + "elapsed", common.PrettyDuration(time.Since(t.createdAt))) + res = t + } else { + if err != nil { + log.Warn("Block sealing failed", "err", err) + } + res = nil + } + select { + case w.resultCh <- res: + case <-w.exitCh: + } +} + +// taskLoop is a standalone goroutine to fetch sealing task from the generator and +// push them to consensus engine. +func (w *worker) taskLoop() { + var stopCh chan struct{} + + // interrupt aborts the in-flight sealing task. + interrupt := func() { + if stopCh != nil { + close(stopCh) + stopCh = nil + } + } + for { + select { + case task := <-w.taskCh: + if w.newTaskHook != nil { + w.newTaskHook(task) + } + interrupt() + stopCh = make(chan struct{}) + go w.seal(task, stopCh) + case <-w.exitCh: + interrupt() + return + } + } +} + +// resultLoop is a standalone goroutine to handle sealing result submitting +// and flush relative data to the database. +func (w *worker) resultLoop() { + for { + select { + case result := <-w.resultCh: + if result == nil { + continue + } + block := result.block + + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, r := range result.receipts { + for _, l := range r.Logs { + l.BlockHash = block.Hash() + } + } + for _, log := range result.state.Logs() { + log.BlockHash = block.Hash() + } + // Commit block and state to database. + stat, err := w.chain.WriteBlockWithState(block, result.receipts, result.state) + if err != nil { + log.Error("Failed writing block to chain", "err", err) + continue + } + // Broadcast the block and announce chain insertion event + w.mux.Post(core.NewMinedBlockEvent{Block: block}) + var ( + events []interface{} + logs = result.state.Logs() + ) + switch stat { + case core.CanonStatTy: + events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) + events = append(events, core.ChainHeadEvent{Block: block}) + case core.SideStatTy: + events = append(events, core.ChainSideEvent{Block: block}) + } + w.chain.PostChainEvents(events, logs) + + // Insert the block into the set of pending ones to resultLoop for confirmations + w.unconfirmed.Insert(block.NumberU64(), block.Hash()) + + case <-w.exitCh: + return + } + } +} + +// makeCurrent creates a new environment for the current cycle. +func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { + state, err := w.chain.StateAt(parent.Root()) + if err != nil { + return err + } + env := &Env{ + config: w.config, + signer: types.NewEIP155Signer(w.config.ChainID), + state: state, + ancestors: mapset.NewSet(), + family: mapset.NewSet(), + uncles: mapset.NewSet(), + header: header, + } + + // when 08 is processed ancestors contain 07 (quick block) + for _, ancestor := range w.chain.GetBlocksFromHash(parent.Hash(), 7) { + for _, uncle := range ancestor.Uncles() { + env.family.Add(uncle.Hash()) + } + env.family.Add(ancestor.Hash()) + env.ancestors.Add(ancestor.Hash()) + } + + // Keep track of transactions which return errors so they can be removed + env.tcount = 0 + w.current = env + return nil +} + +// commitUncle adds the given block to uncle block set, returns error if failed to add. +func (w *worker) commitUncle(env *Env, uncle *types.Header) error { + hash := uncle.Hash() + if env.uncles.Contains(hash) { + return fmt.Errorf("uncle not unique") + } + if !env.ancestors.Contains(uncle.ParentHash) { + return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) + } + if env.family.Contains(hash) { + return fmt.Errorf("uncle already in family (%x)", hash) + } + env.uncles.Add(uncle.Hash()) + return nil +} + +// updateSnapshot updates pending snapshot block and state. +// Note this function assumes the current variable is thread safe. +func (w *worker) updateSnapshot() { + w.snapshotMu.Lock() + defer w.snapshotMu.Unlock() + + var uncles []*types.Header + w.current.uncles.Each(func(item interface{}) bool { + hash, ok := item.(common.Hash) + if !ok { + return false + } + uncle, exist := w.possibleUncles[hash] + if !exist { + return false + } + uncles = append(uncles, uncle.Header()) + return true + }) + + w.snapshotBlock = types.NewBlock( + w.current.header, + w.current.txs, + uncles, + w.current.receipts, + ) + + w.snapshotState = w.current.state.Copy() +} + +// commitNewWork generates several new sealing tasks based on the parent block. +func (w *worker) commitNewWork() { + w.mu.RLock() + defer w.mu.RUnlock() + + tstart := time.Now() + parent := w.chain.CurrentBlock() + + tstamp := tstart.Unix() + if parent.Time().Cmp(new(big.Int).SetInt64(tstamp)) >= 0 { + tstamp = parent.Time().Int64() + 1 + } + // this will ensure we're not going off too far in the future + if now := time.Now().Unix(); tstamp > now+1 { + wait := time.Duration(tstamp-now) * time.Second + log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) + time.Sleep(wait) + } + + num := parent.Number() + header := &types.Header{ + ParentHash: parent.Hash(), + Number: num.Add(num, common.Big1), + GasLimit: core.CalcGasLimit(parent), + Extra: w.extra, + Time: big.NewInt(tstamp), + } + // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) + if w.isRunning() { + if w.coinbase == (common.Address{}) { + log.Error("Refusing to mine without etherbase") + return + } + header.Coinbase = w.coinbase + } + if err := w.engine.Prepare(w.chain, header); err != nil { + log.Error("Failed to prepare header for mining", "err", err) + return + } + // If we are care about TheDAO hard-fork check whether to override the extra-data or not + if daoBlock := w.config.DAOForkBlock; daoBlock != nil { + // Check whether the block is among the fork extra-override range + limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange) + if header.Number.Cmp(daoBlock) >= 0 && header.Number.Cmp(limit) < 0 { + // Depending whether we support or oppose the fork, override differently + if w.config.DAOForkSupport { + header.Extra = common.CopyBytes(params.DAOForkBlockExtra) + } else if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { + header.Extra = []byte{} // If miner opposes, don't let it use the reserved extra-data + } + } + } + // Could potentially happen if starting to mine in an odd state. + err := w.makeCurrent(parent, header) + if err != nil { + log.Error("Failed to create mining context", "err", err) + return + } + // Create the current work task and check any fork transitions needed + env := w.current + if w.config.DAOForkSupport && w.config.DAOForkBlock != nil && w.config.DAOForkBlock.Cmp(header.Number) == 0 { + misc.ApplyDAOHardFork(env.state) + } + + // compute uncles for the new block. + var ( + uncles []*types.Header + badUncles []common.Hash + ) + for hash, uncle := range w.possibleUncles { + if len(uncles) == 2 { + break + } + if err := w.commitUncle(env, uncle.Header()); err != nil { + log.Trace("Bad uncle found and will be removed", "hash", hash) + log.Trace(fmt.Sprint(uncle)) + + badUncles = append(badUncles, hash) + } else { + log.Debug("Committing new uncle to block", "hash", hash) + uncles = append(uncles, uncle.Header()) + } + } + for _, hash := range badUncles { + delete(w.possibleUncles, hash) + } + + var ( + emptyBlock, fullBlock *types.Block + emptyState, fullState *state.StateDB + ) + + // Create an empty block based on temporary copied state for sealing in advance without waiting block + // execution finished. + emptyState = env.state.Copy() + if emptyBlock, err = w.engine.Finalize(w.chain, header, emptyState, nil, uncles, nil); err != nil { + log.Error("Failed to finalize block for temporary sealing", "err", err) + } else { + // Push empty work in advance without applying pending transaction. + // The reason is transactions execution can cost a lot and sealer need to + // take advantage of this part time. + if w.isRunning() { + select { + case w.taskCh <- &task{receipts: nil, state: emptyState, block: emptyBlock, createdAt: time.Now()}: + log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) + case <-w.exitCh: + log.Info("Worker has exited") + return + } + } + } + + // Fill the block with all available pending transactions. + pending, err := w.eth.TxPool().Pending() + if err != nil { + log.Error("Failed to fetch pending transactions", "err", err) + return + } + // Short circuit if there is no available pending transactions + if len(pending) == 0 { + w.updateSnapshot() + return + } + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) + env.commitTransactions(w.mux, txs, w.chain, w.coinbase) + + // Create the full block to seal with the consensus engine + fullState = env.state.Copy() + if fullBlock, err = w.engine.Finalize(w.chain, header, fullState, env.txs, uncles, env.receipts); err != nil { + log.Error("Failed to finalize block for sealing", "err", err) + return + } + // Deep copy receipts here to avoid interaction between different tasks. + cpy := make([]*types.Receipt, len(env.receipts)) + for i, l := range env.receipts { + cpy[i] = new(types.Receipt) + *cpy[i] = *l + } + // We only care about logging if we're actually mining. + if w.isRunning() { + if w.fullTaskInterval != nil { + w.fullTaskInterval() + } + + select { + case w.taskCh <- &task{receipts: cpy, state: fullState, block: fullBlock, createdAt: time.Now()}: + w.unconfirmed.Shift(fullBlock.NumberU64() - 1) + log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) + case <-w.exitCh: + log.Info("Worker has exited") + } + } + w.updateSnapshot() } diff --git a/miner/worker_test.go b/miner/worker_test.go new file mode 100644 index 0000000000..5823a608ef --- /dev/null +++ b/miner/worker_test.go @@ -0,0 +1,212 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package miner + +import ( + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" +) + +var ( + // Test chain configurations + testTxPoolConfig core.TxPoolConfig + ethashChainConfig *params.ChainConfig + cliqueChainConfig *params.ChainConfig + + // Test accounts + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + acc1Key, _ = crypto.GenerateKey() + acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) + + // Test transactions + pendingTxs []*types.Transaction + newTxs []*types.Transaction +) + +func init() { + testTxPoolConfig = core.DefaultTxPoolConfig + testTxPoolConfig.Journal = "" + ethashChainConfig = params.TestChainConfig + cliqueChainConfig = params.TestChainConfig + cliqueChainConfig.Clique = ¶ms.CliqueConfig{ + Period: 1, + Epoch: 30000, + } + tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + pendingTxs = append(pendingTxs, tx1) + tx2, _ := types.SignTx(types.NewTransaction(1, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + newTxs = append(newTxs, tx2) +} + +// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. +type testWorkerBackend struct { + db ethdb.Database + txPool *core.TxPool + chain *core.BlockChain + testTxFeed event.Feed +} + +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend { + var ( + db = ethdb.NewMemDatabase() + gspec = core.Genesis{ + Config: chainConfig, + Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + } + ) + + switch engine.(type) { + case *clique.Clique: + gspec.ExtraData = make([]byte, 32+common.AddressLength+65) + copy(gspec.ExtraData[32:], testBankAddress[:]) + case *ethash.Ethash: + default: + t.Fatal("unexpect consensus engine type") + } + gspec.MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) + txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) + + return &testWorkerBackend{ + db: db, + chain: chain, + txPool: txpool, + } +} + +func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } +func (b *testWorkerBackend) TxPool() *core.TxPool { return b.txPool } +func (b *testWorkerBackend) PostChainEvents(events []interface{}) { + b.chain.PostChainEvents(events, nil) +} + +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine) + backend.txPool.AddLocals(pendingTxs) + w := newWorker(chainConfig, engine, backend, new(event.TypeMux)) + w.setEtherbase(testBankAddress) + return w, backend +} + +func TestPendingStateAndBlockEthash(t *testing.T) { + testPendingStateAndBlock(t, ethashChainConfig, ethash.NewFaker()) +} +func TestPendingStateAndBlockClique(t *testing.T) { + testPendingStateAndBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, b := newTestWorker(t, chainConfig, engine) + defer w.close() + + // Ensure snapshot has been updated. + time.Sleep(100 * time.Millisecond) + block, state := w.pending() + if block.NumberU64() != 1 { + t.Errorf("block number mismatch, has %d, want %d", block.NumberU64(), 1) + } + if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(1000)) != 0 { + t.Errorf("account balance mismatch, has %d, want %d", balance, 1000) + } + b.txPool.AddLocals(newTxs) + // Ensure the new tx events has been processed + time.Sleep(100 * time.Millisecond) + block, state = w.pending() + if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(2000)) != 0 { + t.Errorf("account balance mismatch, has %d, want %d", balance, 2000) + } +} + +func TestEmptyWorkEthash(t *testing.T) { + testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) +} +func TestEmptyWorkClique(t *testing.T) { + testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, _ := newTestWorker(t, chainConfig, engine) + defer w.close() + + var ( + taskCh = make(chan struct{}, 2) + taskIndex int + ) + + checkEqual := func(t *testing.T, task *task, index int) { + receiptLen, balance := 0, big.NewInt(0) + if index == 1 { + receiptLen, balance = 1, big.NewInt(1000) + } + if len(task.receipts) != receiptLen { + t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + } + if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { + t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + } + } + + w.newTaskHook = func(task *task) { + if task.block.NumberU64() == 1 { + checkEqual(t, task, taskIndex) + taskIndex += 1 + taskCh <- struct{}{} + } + } + w.fullTaskInterval = func() { + time.Sleep(100 * time.Millisecond) + } + + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + for i := 0; i < 2; i += 1 { + to := time.NewTimer(time.Second) + select { + case <-taskCh: + case <-to.C: + t.Error("new task timeout") + } + } +} From b2ddb1fcbf77771d0693ee5a00f8ae1cd4c0f87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 14 Aug 2018 22:44:46 +0200 Subject: [PATCH 019/126] les: implement client connection logic (#16899) This PR implements les.freeClientPool. It also adds a simulated clock in common/mclock, which enables time-sensitive tests to run quickly and still produce accurate results, and package common/prque which is a generalised variant of prque that enables removing elements other than the top one from the queue. les.freeClientPool implements a client database that limits the connection time of each client and manages accepting/rejecting incoming connections and even kicking out some connected clients. The pool calculates recent usage time for each known client (a value that increases linearly when the client is connected and decreases exponentially when not connected). Clients with lower recent usage are preferred, unknown nodes have the highest priority. Already connected nodes receive a small bias in their favor in order to avoid accepting and instantly kicking out clients. Note: the pool can use any string for client identification. Using signature keys for that purpose would not make sense when being known has a negative value for the client. Currently the LES protocol manager uses IP addresses (without port address) to identify clients. --- common/mclock/mclock.go | 31 +++++ common/mclock/simclock.go | 129 ++++++++++++++++++ common/prque/prque.go | 57 ++++++++ common/prque/sstack.go | 106 +++++++++++++++ les/freeclient.go | 278 ++++++++++++++++++++++++++++++++++++++ les/freeclient_test.go | 139 +++++++++++++++++++ les/handler.go | 22 ++- 7 files changed, 761 insertions(+), 1 deletion(-) create mode 100644 common/mclock/simclock.go create mode 100755 common/prque/prque.go create mode 100755 common/prque/sstack.go create mode 100644 les/freeclient.go create mode 100644 les/freeclient_test.go diff --git a/common/mclock/mclock.go b/common/mclock/mclock.go index 02608d17b0..dcac59c6ce 100644 --- a/common/mclock/mclock.go +++ b/common/mclock/mclock.go @@ -30,3 +30,34 @@ type AbsTime time.Duration func Now() AbsTime { return AbsTime(monotime.Now()) } + +// Add returns t + d. +func (t AbsTime) Add(d time.Duration) AbsTime { + return t + AbsTime(d) +} + +// Clock interface makes it possible to replace the monotonic system clock with +// a simulated clock. +type Clock interface { + Now() AbsTime + Sleep(time.Duration) + After(time.Duration) <-chan time.Time +} + +// System implements Clock using the system clock. +type System struct{} + +// Now implements Clock. +func (System) Now() AbsTime { + return AbsTime(monotime.Now()) +} + +// Sleep implements Clock. +func (System) Sleep(d time.Duration) { + time.Sleep(d) +} + +// After implements Clock. +func (System) After(d time.Duration) <-chan time.Time { + return time.After(d) +} diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go new file mode 100644 index 0000000000..e014f56150 --- /dev/null +++ b/common/mclock/simclock.go @@ -0,0 +1,129 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mclock + +import ( + "sync" + "time" +) + +// Simulated implements a virtual Clock for reproducible time-sensitive tests. It +// simulates a scheduler on a virtual timescale where actual processing takes zero time. +// +// The virtual clock doesn't advance on its own, call Run to advance it and execute timers. +// Since there is no way to influence the Go scheduler, testing timeout behaviour involving +// goroutines needs special care. A good way to test such timeouts is as follows: First +// perform the action that is supposed to time out. Ensure that the timer you want to test +// is created. Then run the clock until after the timeout. Finally observe the effect of +// the timeout using a channel or semaphore. +type Simulated struct { + now AbsTime + scheduled []event + mu sync.RWMutex + cond *sync.Cond +} + +type event struct { + do func() + at AbsTime +} + +// Run moves the clock by the given duration, executing all timers before that duration. +func (s *Simulated) Run(d time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + end := s.now + AbsTime(d) + for len(s.scheduled) > 0 { + ev := s.scheduled[0] + if ev.at > end { + break + } + s.now = ev.at + ev.do() + s.scheduled = s.scheduled[1:] + } + s.now = end +} + +func (s *Simulated) ActiveTimers() int { + s.mu.RLock() + defer s.mu.RUnlock() + + return len(s.scheduled) +} + +func (s *Simulated) WaitForTimers(n int) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + for len(s.scheduled) < n { + s.cond.Wait() + } +} + +// Now implements Clock. +func (s *Simulated) Now() AbsTime { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.now +} + +// Sleep implements Clock. +func (s *Simulated) Sleep(d time.Duration) { + <-s.After(d) +} + +// After implements Clock. +func (s *Simulated) After(d time.Duration) <-chan time.Time { + after := make(chan time.Time, 1) + s.insert(d, func() { + after <- (time.Time{}).Add(time.Duration(s.now)) + }) + return after +} + +func (s *Simulated) insert(d time.Duration, do func()) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + at := s.now + AbsTime(d) + l, h := 0, len(s.scheduled) + ll := h + for l != h { + m := (l + h) / 2 + if at < s.scheduled[m].at { + h = m + } else { + l = m + 1 + } + } + s.scheduled = append(s.scheduled, event{}) + copy(s.scheduled[l+1:], s.scheduled[l:ll]) + s.scheduled[l] = event{do: do, at: at} + s.cond.Broadcast() +} + +func (s *Simulated) init() { + if s.cond == nil { + s.cond = sync.NewCond(&s.mu) + } +} diff --git a/common/prque/prque.go b/common/prque/prque.go new file mode 100755 index 0000000000..9fd31a2e5d --- /dev/null +++ b/common/prque/prque.go @@ -0,0 +1,57 @@ +// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque". + +package prque + +import ( + "container/heap" +) + +// Priority queue data structure. +type Prque struct { + cont *sstack +} + +// Creates a new priority queue. +func New(setIndex setIndexCallback) *Prque { + return &Prque{newSstack(setIndex)} +} + +// Pushes a value with a given priority into the queue, expanding if necessary. +func (p *Prque) Push(data interface{}, priority int64) { + heap.Push(p.cont, &item{data, priority}) +} + +// Pops the value with the greates priority off the stack and returns it. +// Currently no shrinking is done. +func (p *Prque) Pop() (interface{}, int64) { + item := heap.Pop(p.cont).(*item) + return item.value, item.priority +} + +// Pops only the item from the queue, dropping the associated priority value. +func (p *Prque) PopItem() interface{} { + return heap.Pop(p.cont).(*item).value +} + +// Remove removes the element with the given index. +func (p *Prque) Remove(i int) interface{} { + if i < 0 { + return nil + } + return heap.Remove(p.cont, i) +} + +// Checks whether the priority queue is empty. +func (p *Prque) Empty() bool { + return p.cont.Len() == 0 +} + +// Returns the number of element in the priority queue. +func (p *Prque) Size() int { + return p.cont.Len() +} + +// Clears the contents of the priority queue. +func (p *Prque) Reset() { + *p = *New(p.cont.setIndex) +} diff --git a/common/prque/sstack.go b/common/prque/sstack.go new file mode 100755 index 0000000000..4875dae99d --- /dev/null +++ b/common/prque/sstack.go @@ -0,0 +1,106 @@ +// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque". + +package prque + +// The size of a block of data +const blockSize = 4096 + +// A prioritized item in the sorted stack. +// +// Note: priorities can "wrap around" the int64 range, a comes before b if (a.priority - b.priority) > 0. +// The difference between the lowest and highest priorities in the queue at any point should be less than 2^63. +type item struct { + value interface{} + priority int64 +} + +// setIndexCallback is called when the element is moved to a new index. +// Providing setIndexCallback is optional, it is needed only if the application needs +// to delete elements other than the top one. +type setIndexCallback func(a interface{}, i int) + +// Internal sortable stack data structure. Implements the Push and Pop ops for +// the stack (heap) functionality and the Len, Less and Swap methods for the +// sortability requirements of the heaps. +type sstack struct { + setIndex setIndexCallback + size int + capacity int + offset int + + blocks [][]*item + active []*item +} + +// Creates a new, empty stack. +func newSstack(setIndex setIndexCallback) *sstack { + result := new(sstack) + result.setIndex = setIndex + result.active = make([]*item, blockSize) + result.blocks = [][]*item{result.active} + result.capacity = blockSize + return result +} + +// Pushes a value onto the stack, expanding it if necessary. Required by +// heap.Interface. +func (s *sstack) Push(data interface{}) { + if s.size == s.capacity { + s.active = make([]*item, blockSize) + s.blocks = append(s.blocks, s.active) + s.capacity += blockSize + s.offset = 0 + } else if s.offset == blockSize { + s.active = s.blocks[s.size/blockSize] + s.offset = 0 + } + if s.setIndex != nil { + s.setIndex(data.(*item).value, s.size) + } + s.active[s.offset] = data.(*item) + s.offset++ + s.size++ +} + +// Pops a value off the stack and returns it. Currently no shrinking is done. +// Required by heap.Interface. +func (s *sstack) Pop() (res interface{}) { + s.size-- + s.offset-- + if s.offset < 0 { + s.offset = blockSize - 1 + s.active = s.blocks[s.size/blockSize] + } + res, s.active[s.offset] = s.active[s.offset], nil + if s.setIndex != nil { + s.setIndex(res.(*item).value, -1) + } + return +} + +// Returns the length of the stack. Required by sort.Interface. +func (s *sstack) Len() int { + return s.size +} + +// Compares the priority of two elements of the stack (higher is first). +// Required by sort.Interface. +func (s *sstack) Less(i, j int) bool { + return (s.blocks[i/blockSize][i%blockSize].priority - s.blocks[j/blockSize][j%blockSize].priority) > 0 +} + +// Swaps two elements in the stack. Required by sort.Interface. +func (s *sstack) Swap(i, j int) { + ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize + a, b := s.blocks[jb][jo], s.blocks[ib][io] + if s.setIndex != nil { + s.setIndex(a.value, i) + s.setIndex(b.value, j) + } + s.blocks[ib][io], s.blocks[jb][jo] = a, b +} + +// Resets the stack, effectively clearing its contents. +func (s *sstack) Reset() { + *s = *newSstack(s.setIndex) +} diff --git a/les/freeclient.go b/les/freeclient.go new file mode 100644 index 0000000000..5ee607be8f --- /dev/null +++ b/les/freeclient.go @@ -0,0 +1,278 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package les implements the Light Ethereum Subprotocol. +package les + +import ( + "io" + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// freeClientPool implements a client database that limits the connection time +// of each client and manages accepting/rejecting incoming connections and even +// kicking out some connected clients. The pool calculates recent usage time +// for each known client (a value that increases linearly when the client is +// connected and decreases exponentially when not connected). Clients with lower +// recent usage are preferred, unknown nodes have the highest priority. Already +// connected nodes receive a small bias in their favor in order to avoid accepting +// and instantly kicking out clients. +// +// Note: the pool can use any string for client identification. Using signature +// keys for that purpose would not make sense when being known has a negative +// value for the client. Currently the LES protocol manager uses IP addresses +// (without port address) to identify clients. +type freeClientPool struct { + db ethdb.Database + lock sync.Mutex + clock mclock.Clock + closed bool + + connectedLimit, totalLimit int + + addressMap map[string]*freeClientPoolEntry + connPool, disconnPool *prque.Prque + startupTime mclock.AbsTime + logOffsetAtStartup int64 +} + +const ( + recentUsageExpTC = time.Hour // time constant of the exponential weighting window for "recent" server usage + fixedPointMultiplier = 0x1000000 // constant to convert logarithms to fixed point format + connectedBias = time.Minute // this bias is applied in favor of already connected clients in order to avoid kicking them out very soon +) + +// newFreeClientPool creates a new free client pool +func newFreeClientPool(db ethdb.Database, connectedLimit, totalLimit int, clock mclock.Clock) *freeClientPool { + pool := &freeClientPool{ + db: db, + clock: clock, + addressMap: make(map[string]*freeClientPoolEntry), + connPool: prque.New(poolSetIndex), + disconnPool: prque.New(poolSetIndex), + connectedLimit: connectedLimit, + totalLimit: totalLimit, + } + pool.loadFromDb() + return pool +} + +func (f *freeClientPool) stop() { + f.lock.Lock() + f.closed = true + f.saveToDb() + f.lock.Unlock() +} + +// connect should be called after a successful handshake. If the connection was +// rejected, there is no need to call disconnect. +// +// Note: the disconnectFn callback should not block. +func (f *freeClientPool) connect(address string, disconnectFn func()) bool { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return false + } + e := f.addressMap[address] + now := f.clock.Now() + var recentUsage int64 + if e == nil { + e = &freeClientPoolEntry{address: address, index: -1} + f.addressMap[address] = e + } else { + if e.connected { + log.Debug("Client already connected", "address", address) + return false + } + recentUsage = int64(math.Exp(float64(e.logUsage-f.logOffset(now)) / fixedPointMultiplier)) + } + e.linUsage = recentUsage - int64(now) + // check whether (linUsage+connectedBias) is smaller than the highest entry in the connected pool + if f.connPool.Size() == f.connectedLimit { + i := f.connPool.PopItem().(*freeClientPoolEntry) + if e.linUsage+int64(connectedBias)-i.linUsage < 0 { + // kick it out and accept the new client + f.connPool.Remove(i.index) + f.calcLogUsage(i, now) + i.connected = false + f.disconnPool.Push(i, -i.logUsage) + log.Debug("Client kicked out", "address", i.address) + i.disconnectFn() + } else { + // keep the old client and reject the new one + f.connPool.Push(i, i.linUsage) + log.Debug("Client rejected", "address", address) + return false + } + } + f.disconnPool.Remove(e.index) + e.connected = true + e.disconnectFn = disconnectFn + f.connPool.Push(e, e.linUsage) + if f.connPool.Size()+f.disconnPool.Size() > f.totalLimit { + f.disconnPool.Pop() + } + log.Debug("Client accepted", "address", address) + return true +} + +// disconnect should be called when a connection is terminated. If the disconnection +// was initiated by the pool itself using disconnectFn then calling disconnect is +// not necessary but permitted. +func (f *freeClientPool) disconnect(address string) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return + } + e := f.addressMap[address] + now := f.clock.Now() + if !e.connected { + log.Debug("Client already disconnected", "address", address) + return + } + + f.connPool.Remove(e.index) + f.calcLogUsage(e, now) + e.connected = false + f.disconnPool.Push(e, -e.logUsage) + log.Debug("Client disconnected", "address", address) +} + +// logOffset calculates the time-dependent offset for the logarithmic +// representation of recent usage +func (f *freeClientPool) logOffset(now mclock.AbsTime) int64 { + // Note: fixedPointMultiplier acts as a multiplier here; the reason for dividing the divisor + // is to avoid int64 overflow. We assume that int64(recentUsageExpTC) >> fixedPointMultiplier. + logDecay := int64((time.Duration(now - f.startupTime)) / (recentUsageExpTC / fixedPointMultiplier)) + return f.logOffsetAtStartup + logDecay +} + +// calcLogUsage converts recent usage from linear to logarithmic representation +// when disconnecting a peer or closing the client pool +func (f *freeClientPool) calcLogUsage(e *freeClientPoolEntry, now mclock.AbsTime) { + dt := e.linUsage + int64(now) + if dt < 1 { + dt = 1 + } + e.logUsage = int64(math.Log(float64(dt))*fixedPointMultiplier) + f.logOffset(now) +} + +// freeClientPoolStorage is the RLP representation of the pool's database storage +type freeClientPoolStorage struct { + LogOffset uint64 + List []*freeClientPoolEntry +} + +// loadFromDb restores pool status from the database storage +// (automatically called at initialization) +func (f *freeClientPool) loadFromDb() { + enc, err := f.db.Get([]byte("freeClientPool")) + if err != nil { + return + } + var storage freeClientPoolStorage + err = rlp.DecodeBytes(enc, &storage) + if err != nil { + log.Error("Failed to decode client list", "err", err) + return + } + f.logOffsetAtStartup = int64(storage.LogOffset) + f.startupTime = f.clock.Now() + for _, e := range storage.List { + log.Debug("Loaded free client record", "address", e.address, "logUsage", e.logUsage) + f.addressMap[e.address] = e + f.disconnPool.Push(e, -e.logUsage) + } +} + +// saveToDb saves pool status to the database storage +// (automatically called during shutdown) +func (f *freeClientPool) saveToDb() { + now := f.clock.Now() + storage := freeClientPoolStorage{ + LogOffset: uint64(f.logOffset(now)), + List: make([]*freeClientPoolEntry, len(f.addressMap)), + } + i := 0 + for _, e := range f.addressMap { + if e.connected { + f.calcLogUsage(e, now) + } + storage.List[i] = e + i++ + } + enc, err := rlp.EncodeToBytes(storage) + if err != nil { + log.Error("Failed to encode client list", "err", err) + } else { + f.db.Put([]byte("freeClientPool"), enc) + } +} + +// freeClientPoolEntry represents a client address known by the pool. +// When connected, recent usage is calculated as linUsage + int64(clock.Now()) +// When disconnected, it is calculated as exp(logUsage - logOffset) where logOffset +// also grows linearly with time while the server is running. +// Conversion between linear and logarithmic representation happens when connecting +// or disconnecting the node. +// +// Note: linUsage and logUsage are values used with constantly growing offsets so +// even though they are close to each other at any time they may wrap around int64 +// limits over time. Comparison should be performed accordingly. +type freeClientPoolEntry struct { + address string + connected bool + disconnectFn func() + linUsage, logUsage int64 + index int +} + +func (e *freeClientPoolEntry) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, []interface{}{e.address, uint64(e.logUsage)}) +} + +func (e *freeClientPoolEntry) DecodeRLP(s *rlp.Stream) error { + var entry struct { + Address string + LogUsage uint64 + } + if err := s.Decode(&entry); err != nil { + return err + } + e.address = entry.Address + e.logUsage = int64(entry.LogUsage) + e.connected = false + e.index = -1 + return nil +} + +// poolSetIndex callback is used by both priority queues to set/update the index of +// the element in the queue. Index is needed to remove elements other than the top one. +func poolSetIndex(a interface{}, i int) { + a.(*freeClientPoolEntry).index = i +} diff --git a/les/freeclient_test.go b/les/freeclient_test.go new file mode 100644 index 0000000000..e95abc7aad --- /dev/null +++ b/les/freeclient_test.go @@ -0,0 +1,139 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package light implements on-demand retrieval capable state and chain objects +// for the Ethereum Light Client. +package les + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb" +) + +func TestFreeClientPoolL10C100(t *testing.T) { + testFreeClientPool(t, 10, 100) +} + +func TestFreeClientPoolL40C200(t *testing.T) { + testFreeClientPool(t, 40, 200) +} + +func TestFreeClientPoolL100C300(t *testing.T) { + testFreeClientPool(t, 100, 300) +} + +const testFreeClientPoolTicks = 500000 + +func testFreeClientPool(t *testing.T, connLimit, clientCount int) { + var ( + clock mclock.Simulated + db = ethdb.NewMemDatabase() + pool = newFreeClientPool(db, connLimit, 10000, &clock) + connected = make([]bool, clientCount) + connTicks = make([]int, clientCount) + disconnCh = make(chan int, clientCount) + ) + peerId := func(i int) string { + return fmt.Sprintf("test peer #%d", i) + } + disconnFn := func(i int) func() { + return func() { + disconnCh <- i + } + } + + // pool should accept new peers up to its connected limit + for i := 0; i < connLimit; i++ { + if pool.connect(peerId(i), disconnFn(i)) { + connected[i] = true + } else { + t.Fatalf("Test peer #%d rejected", i) + } + } + // since all accepted peers are new and should not be kicked out, the next one should be rejected + if pool.connect(peerId(connLimit), disconnFn(connLimit)) { + connected[connLimit] = true + t.Fatalf("Peer accepted over connected limit") + } + + // randomly connect and disconnect peers, expect to have a similar total connection time at the end + for tickCounter := 0; tickCounter < testFreeClientPoolTicks; tickCounter++ { + clock.Run(1 * time.Second) + + i := rand.Intn(clientCount) + if connected[i] { + pool.disconnect(peerId(i)) + connected[i] = false + connTicks[i] += tickCounter + } else { + if pool.connect(peerId(i), disconnFn(i)) { + connected[i] = true + connTicks[i] -= tickCounter + } + } + pollDisconnects: + for { + select { + case i := <-disconnCh: + pool.disconnect(peerId(i)) + if connected[i] { + connTicks[i] += tickCounter + connected[i] = false + } + default: + break pollDisconnects + } + } + } + + expTicks := testFreeClientPoolTicks * connLimit / clientCount + expMin := expTicks - expTicks/10 + expMax := expTicks + expTicks/10 + + // check if the total connected time of peers are all in the expected range + for i, c := range connected { + if c { + connTicks[i] += testFreeClientPoolTicks + } + if connTicks[i] < expMin || connTicks[i] > expMax { + t.Errorf("Total connected time of test node #%d (%d) outside expected range (%d to %d)", i, connTicks[i], expMin, expMax) + } + } + + // a previously unknown peer should be accepted now + if !pool.connect("newPeer", func() {}) { + t.Fatalf("Previously unknown peer rejected") + } + + // close and restart pool + pool.stop() + pool = newFreeClientPool(db, connLimit, 10000, &clock) + + // try connecting all known peers (connLimit should be filled up) + for i := 0; i < clientCount; i++ { + pool.connect(peerId(i), func() {}) + } + // expect pool to remember known nodes and kick out one of them to accept a new one + if !pool.connect("newPeer2", func() {}) { + t.Errorf("Previously unknown peer rejected after restarting pool") + } + pool.stop() +} diff --git a/les/handler.go b/les/handler.go index 2fc4cde349..91a235bf0a 100644 --- a/les/handler.go +++ b/les/handler.go @@ -28,6 +28,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -104,6 +105,7 @@ type ProtocolManager struct { odr *LesOdr server *LesServer serverPool *serverPool + clientPool *freeClientPool lesTopic discv5.Topic reqDist *requestDistributor retriever *retrieveManager @@ -226,6 +228,7 @@ func (pm *ProtocolManager) Start(maxPeers int) { if pm.lightSync { go pm.syncer() } else { + pm.clientPool = newFreeClientPool(pm.chainDb, maxPeers, 10000, mclock.System{}) go func() { for range pm.newPeerCh { } @@ -243,6 +246,9 @@ func (pm *ProtocolManager) Stop() { pm.noMorePeers <- struct{}{} close(pm.quitSync) // quits syncer, fetcher + if pm.clientPool != nil { + pm.clientPool.stop() + } // Disconnect existing sessions. // This also closes the gate for any new registrations on the peer set. @@ -264,7 +270,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea // this function terminates, the peer is disconnected. func (pm *ProtocolManager) handle(p *peer) error { // Ignore maxPeers if this is a trusted peer - if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { + // In server mode we try to check into the client pool after handshake + if pm.lightSync && pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { return p2p.DiscTooManyPeers } @@ -282,6 +289,19 @@ func (pm *ProtocolManager) handle(p *peer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } + + if !pm.lightSync && !p.Peer.Info().Network.Trusted { + addr, ok := p.RemoteAddr().(*net.TCPAddr) + // test peer address is not a tcp address, don't use client pool if can not typecast + if ok { + id := addr.IP.String() + if !pm.clientPool.connect(id, func() { go pm.removePeer(p.id) }) { + return p2p.DiscTooManyPeers + } + defer pm.clientPool.disconnect(id) + } + } + if rw, ok := p.rw.(*meteredMsgReadWriter); ok { rw.Init(p.version) } From b52bb31b767a3e31266e6138e24f7226ebbd8882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 14 Aug 2018 22:59:18 +0200 Subject: [PATCH 020/126] p2p/discv5: add delay to refresh cycle when no seed nodes are found (#16994) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index 4c39c05533..b93c93d648 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -678,7 +678,7 @@ func (net *Network) refresh(done chan<- struct{}) { } if len(seeds) == 0 { log.Trace("no seed nodes found") - close(done) + time.AfterFunc(time.Second*10, func() { close(done) }) return } for _, n := range seeds { From 212bba47ff13812ddabb642da463e58cda4ff20f Mon Sep 17 00:00:00 2001 From: Jeff Prestes Date: Wed, 15 Aug 2018 04:15:42 -0300 Subject: [PATCH 021/126] backends: configurable gas limit to allow testing large contracts (#17358) * backends: increase gaslimit in order to allow tests of large contracts * backends: increase gaslimit in order to allow tests of large contracts * backends: increase gaslimit in order to allow tests of large contracts --- accounts/abi/bind/backends/simulated.go | 4 ++-- accounts/abi/bind/bind_test.go | 24 ++++++++++++------------ accounts/abi/bind/util_test.go | 8 +++++--- contracts/chequebook/cheque_test.go | 2 +- contracts/chequebook/gencode.go | 2 +- contracts/ens/ens_test.go | 2 +- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index fa8828f61b..1d14f8c6f1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -65,9 +65,9 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. -func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { +func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { database := ethdb.NewMemDatabase() - genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc} + genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 2a5a886487..0e5b1c1616 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -229,7 +229,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy an interaction tester contract and call a transaction on it _, _, interactor, err := DeployInteractor(auth, sim, "Deploy string") @@ -270,7 +270,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a tuple tester contract and execute a structured call on it _, _, getter, err := DeployGetter(auth, sim) @@ -302,7 +302,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a tuple tester contract and execute a structured call on it _, _, tupler, err := DeployTupler(auth, sim) @@ -344,7 +344,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a slice tester contract and execute a n array call on it _, _, slicer, err := DeploySlicer(auth, sim) @@ -378,7 +378,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a default method invoker contract and execute its default method _, _, defaulter, err := DeployDefaulter(auth, sim) @@ -411,7 +411,7 @@ var bindTests = []struct { `[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`, ` // Create a simulator and wrap a non-deployed contract - sim := backends.NewSimulatedBackend(nil) + sim := backends.NewSimulatedBackend(nil, uint64(10000000000)) nonexistent, err := NewNonExistent(common.Address{}, sim) if err != nil { @@ -447,7 +447,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a funky gas pattern contract _, _, limiter, err := DeployFunkyGasPattern(auth, sim) @@ -482,7 +482,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a sender tester contract and execute a structured call on it _, _, callfrom, err := DeployCallFrom(auth, sim) @@ -542,7 +542,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a underscorer tester contract and execute a structured call on it _, _, underscorer, err := DeployUnderscorer(auth, sim) @@ -612,7 +612,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy an eventer contract _, _, eventer, err := DeployEventer(auth, sim) @@ -761,7 +761,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) //deploy the test contract _, _, testContract, err := DeployDeeplyNestedArray(auth, sim) @@ -820,7 +820,7 @@ func TestBindings(t *testing.T) { t.Skip("go sdk not found for testing") } // Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845) - linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}") + linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil,uint64(10000000000)))\n}") linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil) if err != nil { t.Fatalf("failed check for goimports symlink bug: %v", err) diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 49e6dc813a..8f4092971f 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -53,9 +53,11 @@ var waitDeployedTests = map[string]struct { func TestWaitDeployed(t *testing.T) { for name, test := range waitDeployedTests { - backend := backends.NewSimulatedBackend(core.GenesisAlloc{ - crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, - }) + backend := backends.NewSimulatedBackend( + core.GenesisAlloc{ + crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) // Create the transaction. tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code)) diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 6b6b28e657..4bd2e176b1 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -46,7 +46,7 @@ func newTestBackend() *backends.SimulatedBackend { addr0: {Balance: big.NewInt(1000000000)}, addr1: {Balance: big.NewInt(1000000000)}, addr2: {Balance: big.NewInt(1000000000)}, - }) + }, 10000000) } func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) { diff --git a/contracts/chequebook/gencode.go b/contracts/chequebook/gencode.go index 45f6d68f3e..ddfe8d1512 100644 --- a/contracts/chequebook/gencode.go +++ b/contracts/chequebook/gencode.go @@ -40,7 +40,7 @@ var ( ) func main() { - backend := backends.NewSimulatedBackend(testAlloc) + backend := backends.NewSimulatedBackend(testAlloc, uint64(100000000)) auth := bind.NewKeyedTransactor(testKey) // Deploy the contract, get the code. diff --git a/contracts/ens/ens_test.go b/contracts/ens/ens_test.go index 6ad8447082..411b04197c 100644 --- a/contracts/ens/ens_test.go +++ b/contracts/ens/ens_test.go @@ -35,7 +35,7 @@ var ( ) func TestENS(t *testing.T) { - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, 10000000) transactOpts := bind.NewKeyedTransactor(key) ensAddr, ens, err := DeployENS(transactOpts, contractBackend) From 2a17fe25612b57d943862459dba88666685ffd69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 15 Aug 2018 11:01:49 +0300 Subject: [PATCH 022/126] cmd: polish miner flags, deprecate olds, add upgrade path --- cmd/geth/chaincmd.go | 12 ++--- cmd/geth/main.go | 29 ++++++---- cmd/geth/usage.go | 15 +++--- cmd/puppeth/module_node.go | 2 +- cmd/utils/flags.go | 108 +++++++++++++++++++++++-------------- 5 files changed, 104 insertions(+), 62 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ff27a9dfb4..87548865be 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -48,7 +48,6 @@ var ( ArgsUsage: "", Flags: []cli.Flag{ utils.DataDirFlag, - utils.LightModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -66,7 +65,7 @@ It expects the genesis file as argument.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, utils.GCModeFlag, utils.CacheDatabaseFlag, utils.CacheGCFlag, @@ -87,7 +86,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -105,7 +104,7 @@ be gzipped.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -119,7 +118,7 @@ be gzipped.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -149,7 +148,6 @@ The first argument must be the directory containing the blockchain to download f ArgsUsage: " ", Flags: []cli.Flag{ utils.DataDirFlag, - utils.LightModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -163,7 +161,7 @@ Remove blockchain and state databases`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d556ad92c3..a063860513 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -82,8 +82,6 @@ var ( utils.TxPoolAccountQueueFlag, utils.TxPoolGlobalQueueFlag, utils.TxPoolLifetimeFlag, - utils.FastSyncFlag, - utils.LightModeFlag, utils.SyncModeFlag, utils.GCModeFlag, utils.LightServFlag, @@ -96,12 +94,18 @@ var ( utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, - utils.EtherbaseFlag, - utils.GasPriceFlag, utils.MiningEnabledFlag, utils.MinerThreadsFlag, + utils.MinerLegacyThreadsFlag, utils.MinerNotifyFlag, - utils.TargetGasLimitFlag, + utils.MinerGasTargetFlag, + utils.MinerLegacyGasTargetFlag, + utils.MinerGasPriceFlag, + utils.MinerLegacyGasPriceFlag, + utils.MinerEtherbaseFlag, + utils.MinerLegacyEtherbaseFlag, + utils.MinerExtraDataFlag, + utils.MinerLegacyExtraDataFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, @@ -122,7 +126,6 @@ var ( utils.NoCompactionFlag, utils.GpoBlocksFlag, utils.GpoPercentileFlag, - utils.ExtraDataFlag, configFileFlag, } @@ -324,7 +327,7 @@ func startNode(ctx *cli.Context, stack *node.Node) { // Start auxiliary services if enabled if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) { // Mining only makes sense if a full Ethereum node is running - if ctx.GlobalBool(utils.LightModeFlag.Name) || ctx.GlobalString(utils.SyncModeFlag.Name) == "light" { + if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" { utils.Fatalf("Light clients do not support mining") } var ethereum *eth.Ethereum @@ -332,7 +335,11 @@ func startNode(ctx *cli.Context, stack *node.Node) { utils.Fatalf("Ethereum service not running: %v", err) } // Use a reduced number of threads if requested - if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 { + threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name) + if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { + threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name) + } + if threads > 0 { type threaded interface { SetThreads(threads int) } @@ -341,7 +348,11 @@ func startNode(ctx *cli.Context, stack *node.Node) { } } // Set the gas price to the limits from the CLI and start mining - ethereum.TxPool().SetGasPrice(utils.GlobalBig(ctx, utils.GasPriceFlag.Name)) + gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name) + if ctx.IsSet(utils.MinerGasPriceFlag.Name) { + gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) + } + ethereum.TxPool().SetGasPrice(gasprice) if err := ethereum.StartMining(true); err != nil { utils.Fatalf("Failed to start mining: %v", err) } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 9d63c68f7f..9e18f70472 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -186,10 +186,10 @@ var AppHelpFlagGroups = []flagGroup{ utils.MiningEnabledFlag, utils.MinerThreadsFlag, utils.MinerNotifyFlag, - utils.EtherbaseFlag, - utils.TargetGasLimitFlag, - utils.GasPriceFlag, - utils.ExtraDataFlag, + utils.MinerGasPriceFlag, + utils.MinerGasTargetFlag, + utils.MinerEtherbaseFlag, + utils.MinerExtraDataFlag, }, }, { @@ -231,8 +231,11 @@ var AppHelpFlagGroups = []flagGroup{ { Name: "DEPRECATED", Flags: []cli.Flag{ - utils.FastSyncFlag, - utils.LightModeFlag, + utils.MinerLegacyThreadsFlag, + utils.MinerLegacyGasTargetFlag, + utils.MinerLegacyGasPriceFlag, + utils.MinerLegacyEtherbaseFlag, + utils.MinerLegacyExtraDataFlag, }, }, { diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index a480a894eb..8ad41555e1 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d6142f246c..e3a8cc2eac 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -157,14 +157,6 @@ var ( Usage: "Document Root for HTTPClient file scheme", Value: DirectoryString{homeDir()}, } - FastSyncFlag = cli.BoolFlag{ - Name: "fast", - Usage: "Enable fast syncing through state downloads (replaced by --syncmode)", - } - LightModeFlag = cli.BoolFlag{ - Name: "light", - Usage: "Enable light client mode (replaced by --syncmode)", - } defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", @@ -321,29 +313,53 @@ var ( Usage: "Number of CPU threads to use for mining", Value: 0, } + MinerLegacyThreadsFlag = cli.IntFlag{ + Name: "minerthreads", + Usage: "Number of CPU threads to use for mining (deprecated, use --miner.threads)", + Value: 0, + } MinerNotifyFlag = cli.StringFlag{ Name: "miner.notify", Usage: "Comma separated HTTP URL list to notify of new work packages", } - TargetGasLimitFlag = cli.Uint64Flag{ - Name: "targetgaslimit", - Usage: "Target gas limit sets the artificial target gas floor for the blocks to mine", + MinerGasTargetFlag = cli.Uint64Flag{ + Name: "miner.gastarget", + Usage: "Target gas floor for mined blocks", Value: params.GenesisGasLimit, } - EtherbaseFlag = cli.StringFlag{ - Name: "etherbase", - Usage: "Public address for block mining rewards (default = first account created)", - Value: "0", + MinerLegacyGasTargetFlag = cli.Uint64Flag{ + Name: "targetgaslimit", + Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)", + Value: params.GenesisGasLimit, } - GasPriceFlag = BigFlag{ - Name: "gasprice", - Usage: "Minimal gas price to accept for mining a transactions", + MinerGasPriceFlag = BigFlag{ + Name: "miner.gasprice", + Usage: "Minimal gas price for mining a transactions", Value: eth.DefaultConfig.GasPrice, } - ExtraDataFlag = cli.StringFlag{ - Name: "extradata", + MinerLegacyGasPriceFlag = BigFlag{ + Name: "gasprice", + Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)", + Value: eth.DefaultConfig.GasPrice, + } + MinerEtherbaseFlag = cli.StringFlag{ + Name: "miner.etherbase", + Usage: "Public address for block mining rewards (default = first account)", + Value: "0", + } + MinerLegacyEtherbaseFlag = cli.StringFlag{ + Name: "etherbase", + Usage: "Public address for block mining rewards (default = first account, deprecated, use --miner.etherbase)", + Value: "0", + } + MinerExtraDataFlag = cli.StringFlag{ + Name: "miner.extradata", Usage: "Block extra data set by the miner (default = client version)", } + MinerLegacyExtraDataFlag = cli.StringFlag{ + Name: "extradata", + Usage: "Block extra data set by the miner (default = client version, deprecated, use --miner.extradata)", + } // Account settings UnlockedAccountFlag = cli.StringFlag{ Name: "unlock", @@ -813,10 +829,19 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase either from the directly specified // command line flags or from the keystore if CLI indexed. func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) { - if ctx.GlobalIsSet(EtherbaseFlag.Name) { - account, err := MakeAddress(ks, ctx.GlobalString(EtherbaseFlag.Name)) + // Extract the current etherbase, new flag overriding legacy one + var etherbase string + if ctx.GlobalIsSet(MinerLegacyEtherbaseFlag.Name) { + etherbase = ctx.GlobalString(MinerLegacyEtherbaseFlag.Name) + } + if ctx.GlobalIsSet(MinerEtherbaseFlag.Name) { + etherbase = ctx.GlobalString(MinerEtherbaseFlag.Name) + } + // Convert the etherbase into an address and configure it + if etherbase != "" { + account, err := MakeAddress(ks, etherbase) if err != nil { - Fatalf("Option %q: %v", EtherbaseFlag.Name, err) + Fatalf("Invalid miner etherbase: %v", err) } cfg.Etherbase = account.Address } @@ -847,7 +872,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setBootstrapNodes(ctx, cfg) setBootstrapNodesV5(ctx, cfg) - lightClient := ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalString(SyncModeFlag.Name) == "light" + lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light" lightServer := ctx.GlobalInt(LightServFlag.Name) != 0 lightPeers := ctx.GlobalInt(LightPeersFlag.Name) @@ -1052,8 +1077,6 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag) - checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag) - checkExclusive(ctx, LightServFlag, LightModeFlag) checkExclusive(ctx, LightServFlag, SyncModeFlag, "light") ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -1062,13 +1085,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) - switch { - case ctx.GlobalIsSet(SyncModeFlag.Name): + if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) - case ctx.GlobalBool(FastSyncFlag.Name): - cfg.SyncMode = downloader.FastSync - case ctx.GlobalBool(LightModeFlag.Name): - cfg.SyncMode = downloader.LightSync } if ctx.GlobalIsSet(LightServFlag.Name) { cfg.LightServ = ctx.GlobalInt(LightServFlag.Name) @@ -1093,6 +1111,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } + if ctx.GlobalIsSet(MinerLegacyThreadsFlag.Name) { + cfg.MinerThreads = ctx.GlobalInt(MinerLegacyThreadsFlag.Name) + } if ctx.GlobalIsSet(MinerThreadsFlag.Name) { cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name) } @@ -1102,11 +1123,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } - if ctx.GlobalIsSet(ExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(ExtraDataFlag.Name)) + if ctx.GlobalIsSet(MinerLegacyExtraDataFlag.Name) { + cfg.ExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name)) } - if ctx.GlobalIsSet(GasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, GasPriceFlag.Name) + if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { + cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) + } + if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { + cfg.GasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) + } + if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { + cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) } if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode @@ -1148,7 +1175,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { log.Info("Using developer account", "address", developer.Address) cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address) - if !ctx.GlobalIsSet(GasPriceFlag.Name) { + if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { cfg.GasPrice = big.NewInt(1) } } @@ -1223,7 +1250,10 @@ func RegisterEthStatsService(stack *node.Node, url string) { // SetupNetwork configures the system for either the main net or some test network. func SetupNetwork(ctx *cli.Context) { // TODO(fjl): move target gas limit into config - params.TargetGasLimit = ctx.GlobalUint64(TargetGasLimitFlag.Name) + params.TargetGasLimit = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name) + if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { + params.TargetGasLimit = ctx.GlobalUint64(MinerGasTargetFlag.Name) + } } func SetupMetrics(ctx *cli.Context) { @@ -1254,7 +1284,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { handles = makeDatabaseHandles() ) name := "chaindata" - if ctx.GlobalBool(LightModeFlag.Name) { + if ctx.GlobalString(SyncModeFlag.Name) == "light" { name = "lightchaindata" } chainDb, err := stack.OpenDatabase(name, cache, handles) From 040aa2bb101e5e602308b24812bfbf2451b21174 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 15 Aug 2018 19:09:17 +0800 Subject: [PATCH 023/126] miner: streaming uncle blocks (#17320) * miner: stream uncle block * miner: polish --- miner/worker.go | 105 ++++++++++++++++++++++++------------------- miner/worker_test.go | 76 +++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 53 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 81a63c29a4..fae480c84c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -213,8 +213,9 @@ type worker struct { running int32 // The indicator whether the consensus engine is running or not. // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task - fullTaskInterval func() // Method to call before pushing the full sealing task + newTaskHook func(*task) // Method to call upon receiving a new sealing task + skipSealHook func(*task) bool // Method to decide whether skipping the sealing. + fullTaskHook func() // Method to call before pushing the full sealing task } func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { @@ -329,8 +330,32 @@ func (w *worker) mainLoop() { w.commitNewWork() case ev := <-w.chainSideCh: + if _, exist := w.possibleUncles[ev.Block.Hash()]; exist { + continue + } // Add side block to possible uncle block set. w.possibleUncles[ev.Block.Hash()] = ev.Block + // If our mining block contains less than 2 uncle blocks, + // add the new uncle block if valid and regenerate a mining block. + if w.isRunning() && w.current != nil && w.current.uncles.Cardinality() < 2 { + start := time.Now() + if err := w.commitUncle(w.current, ev.Block.Header()); err == nil { + var uncles []*types.Header + w.current.uncles.Each(func(item interface{}) bool { + hash, ok := item.(common.Hash) + if !ok { + return false + } + uncle, exist := w.possibleUncles[hash] + if !exist { + return false + } + uncles = append(uncles, uncle.Header()) + return true + }) + w.commit(uncles, nil, true, start) + } + } case ev := <-w.txsCh: // Apply transactions to the pending state if we're not mining. @@ -378,6 +403,10 @@ func (w *worker) seal(t *task, stop <-chan struct{}) { res *task ) + if w.skipSealHook != nil && w.skipSealHook(t) { + return + } + if t.block, err = w.engine.Seal(w.chain, t.block, stop); t.block != nil { log.Info("Successfully sealed new block", "number", t.block.Number(), "hash", t.block.Hash(), "elapsed", common.PrettyDuration(time.Since(t.createdAt))) @@ -637,30 +666,9 @@ func (w *worker) commitNewWork() { delete(w.possibleUncles, hash) } - var ( - emptyBlock, fullBlock *types.Block - emptyState, fullState *state.StateDB - ) - // Create an empty block based on temporary copied state for sealing in advance without waiting block // execution finished. - emptyState = env.state.Copy() - if emptyBlock, err = w.engine.Finalize(w.chain, header, emptyState, nil, uncles, nil); err != nil { - log.Error("Failed to finalize block for temporary sealing", "err", err) - } else { - // Push empty work in advance without applying pending transaction. - // The reason is transactions execution can cost a lot and sealer need to - // take advantage of this part time. - if w.isRunning() { - select { - case w.taskCh <- &task{receipts: nil, state: emptyState, block: emptyBlock, createdAt: time.Now()}: - log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) - case <-w.exitCh: - log.Info("Worker has exited") - return - } - } - } + w.commit(uncles, nil, false, tstart) // Fill the block with all available pending transactions. pending, err := w.eth.TxPool().Pending() @@ -676,31 +684,38 @@ func (w *worker) commitNewWork() { txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) env.commitTransactions(w.mux, txs, w.chain, w.coinbase) - // Create the full block to seal with the consensus engine - fullState = env.state.Copy() - if fullBlock, err = w.engine.Finalize(w.chain, header, fullState, env.txs, uncles, env.receipts); err != nil { - log.Error("Failed to finalize block for sealing", "err", err) - return - } - // Deep copy receipts here to avoid interaction between different tasks. - cpy := make([]*types.Receipt, len(env.receipts)) - for i, l := range env.receipts { - cpy[i] = new(types.Receipt) - *cpy[i] = *l - } - // We only care about logging if we're actually mining. - if w.isRunning() { - if w.fullTaskInterval != nil { - w.fullTaskInterval() - } + w.commit(uncles, w.fullTaskHook, true, tstart) +} +// commit runs any post-transaction state modifications, assembles the final block +// and commits new work if consensus engine is running. +func (w *worker) commit(uncles []*types.Header, interval func(), update bool, start time.Time) error { + // Deep copy receipts here to avoid interaction between different tasks. + receipts := make([]*types.Receipt, len(w.current.receipts)) + for i, l := range w.current.receipts { + receipts[i] = new(types.Receipt) + *receipts[i] = *l + } + s := w.current.state.Copy() + block, err := w.engine.Finalize(w.chain, w.current.header, s, w.current.txs, uncles, w.current.receipts) + if err != nil { + return err + } + if w.isRunning() { + if interval != nil { + interval() + } select { - case w.taskCh <- &task{receipts: cpy, state: fullState, block: fullBlock, createdAt: time.Now()}: - w.unconfirmed.Shift(fullBlock.NumberU64() - 1) - log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) + case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now()}: + w.unconfirmed.Shift(block.NumberU64() - 1) + log.Info("Commit new mining work", "number", block.Number(), "txs", w.current.tcount, "uncles", len(uncles), + "elapsed", common.PrettyDuration(time.Since(start))) case <-w.exitCh: log.Info("Worker has exited") } } - w.updateSnapshot() + if update { + w.updateSnapshot() + } + return nil } diff --git a/miner/worker_test.go b/miner/worker_test.go index 5823a608ef..408c47e3b3 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -59,7 +59,7 @@ func init() { ethashChainConfig = params.TestChainConfig cliqueChainConfig = params.TestChainConfig cliqueChainConfig.Clique = ¶ms.CliqueConfig{ - Period: 1, + Period: 10, Epoch: 30000, } tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) @@ -74,6 +74,7 @@ type testWorkerBackend struct { txPool *core.TxPool chain *core.BlockChain testTxFeed event.Feed + uncleBlock *types.Block } func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend { @@ -93,15 +94,19 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine default: t.Fatal("unexpect consensus engine type") } - gspec.MustCommit(db) + genesis := gspec.MustCommit(db) chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) + blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(acc1Addr) + }) return &testWorkerBackend{ - db: db, - chain: chain, - txPool: txpool, + db: db, + chain: chain, + txPool: txpool, + uncleBlock: blocks[0], } } @@ -188,7 +193,7 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens taskCh <- struct{}{} } } - w.fullTaskInterval = func() { + w.fullTaskHook = func() { time.Sleep(100 * time.Millisecond) } @@ -202,11 +207,66 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens w.start() for i := 0; i < 2; i += 1 { - to := time.NewTimer(time.Second) select { case <-taskCh: - case <-to.C: + case <-time.NewTimer(time.Second).C: t.Error("new task timeout") } } } + +func TestStreamUncleBlock(t *testing.T) { + ethash := ethash.NewFaker() + defer ethash.Close() + + w, b := newTestWorker(t, ethashChainConfig, ethash) + defer w.close() + + var taskCh = make(chan struct{}) + + taskIndex := 0 + w.newTaskHook = func(task *task) { + if task.block.NumberU64() == 1 { + if taskIndex == 2 { + has := task.block.Header().UncleHash + want := types.CalcUncleHash([]*types.Header{b.uncleBlock.Header()}) + if has != want { + t.Errorf("uncle hash mismatch, has %s, want %s", has.Hex(), want.Hex()) + } + } + taskCh <- struct{}{} + taskIndex += 1 + } + } + w.skipSealHook = func(task *task) bool { + return true + } + w.fullTaskHook = func() { + time.Sleep(100 * time.Millisecond) + } + + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + // Ignore the first two works + for i := 0; i < 2; i += 1 { + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } + } + b.PostChainEvents([]interface{}{core.ChainSideEvent{Block: b.uncleBlock}}) + + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } +} From d8541a9f99c58d97ba4908c3a768e518f28d2441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 15 Aug 2018 13:50:16 +0300 Subject: [PATCH 024/126] consensus/ethash: use DAGs for remote mining, generate async --- consensus/ethash/consensus.go | 52 +++++++++++++++++++++++++++-------- consensus/ethash/ethash.go | 44 +++++++++++++++++++++++------ consensus/ethash/sealer.go | 11 ++++---- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index e18a06d52a..86fd997ae4 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -461,6 +461,13 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int { // VerifySeal implements consensus.Engine, checking whether the given block satisfies // the PoW difficulty requirements. func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Header) error { + return ethash.verifySeal(chain, header, false) +} + +// verifySeal checks whether a block satisfies the PoW difficulty requirements, +// either using the usual ethash cache for it, or alternatively using a full DAG +// to make remote mining fast. +func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Header, fulldag bool) error { // If we're running a fake PoW, accept any seal as valid if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake { time.Sleep(ethash.fakeDelay) @@ -471,25 +478,48 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head } // If we're running a shared PoW, delegate verification to it if ethash.shared != nil { - return ethash.shared.VerifySeal(chain, header) + return ethash.shared.verifySeal(chain, header, fulldag) } // Ensure that we have a valid difficulty for the block if header.Difficulty.Sign() <= 0 { return errInvalidDifficulty } - // Recompute the digest and PoW value and verify against the header + // Recompute the digest and PoW values number := header.Number.Uint64() - cache := ethash.cache(number) - size := datasetSize(number) - if ethash.config.PowMode == ModeTest { - size = 32 * 1024 - } - digest, result := hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) - // Caches are unmapped in a finalizer. Ensure that the cache stays live - // until after the call to hashimotoLight so it's not unmapped while being used. - runtime.KeepAlive(cache) + var ( + digest []byte + result []byte + ) + // If fast-but-heavy PoW verification was requested, use an ethash dataset + if fulldag { + dataset := ethash.dataset(number, true) + if dataset.generated() { + digest, result = hashimotoFull(dataset.dataset, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + // Datasets are unmapped in a finalizer. Ensure that the dataset stays alive + // until after the call to hashimotoFull so it's not unmapped while being used. + runtime.KeepAlive(dataset) + } else { + // Dataset not yet generated, don't hang, use a cache instead + fulldag = false + } + } + // If slow-but-light PoW verification was requested (or DAG not yet ready), use an ethash cache + if !fulldag { + cache := ethash.cache(number) + + size := datasetSize(number) + if ethash.config.PowMode == ModeTest { + size = 32 * 1024 + } + digest, result = hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + + // Caches are unmapped in a finalizer. Ensure that the cache stays alive + // until after the call to hashimotoLight so it's not unmapped while being used. + runtime.KeepAlive(cache) + } + // Verify the calculated values against the ones provided in the header if !bytes.Equal(header.MixDigest[:], digest) { return errInvalidMixDigest } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 19c94deb6b..d98c3371c5 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -29,6 +29,7 @@ import ( "runtime" "strconv" "sync" + "sync/atomic" "time" "unsafe" @@ -281,6 +282,7 @@ type dataset struct { mmap mmap.MMap // Memory map itself to unmap before releasing dataset []uint32 // The actual cache data content once sync.Once // Ensures the cache is generated only once + done uint32 // Atomic flag to determine generation status } // newDataset creates a new ethash mining dataset and returns it as a plain Go @@ -292,6 +294,9 @@ func newDataset(epoch uint64) interface{} { // generate ensures that the dataset content is generated before use. func (d *dataset) generate(dir string, limit int, test bool) { d.once.Do(func() { + // Mark the dataset generated after we're done. This is needed for remote + defer atomic.StoreUint32(&d.done, 1) + csize := cacheSize(d.epoch*epochLength + 1) dsize := datasetSize(d.epoch*epochLength + 1) seed := seedHash(d.epoch*epochLength + 1) @@ -306,6 +311,8 @@ func (d *dataset) generate(dir string, limit int, test bool) { d.dataset = make([]uint32, dsize/4) generateDataset(d.dataset, d.epoch, cache) + + return } // Disk storage is needed, this will get fancy var endian string @@ -348,6 +355,13 @@ func (d *dataset) generate(dir string, limit int, test bool) { }) } +// generated returns whether this particular dataset finished generating already +// or not (it may not have been started at all). This is useful for remote miners +// to default to verification caches instead of blocking on DAG generations. +func (d *dataset) generated() bool { + return atomic.LoadUint32(&d.done) == 1 +} + // finalizer closes any file handlers and memory maps open. func (d *dataset) finalizer() { if d.mmap != nil { @@ -589,20 +603,34 @@ func (ethash *Ethash) cache(block uint64) *cache { // dataset tries to retrieve a mining dataset for the specified block number // by first checking against a list of in-memory datasets, then against DAGs // stored on disk, and finally generating one if none can be found. -func (ethash *Ethash) dataset(block uint64) *dataset { +// +// If async is specified, not only the future but the current DAG is also +// generates on a background thread. +func (ethash *Ethash) dataset(block uint64, async bool) *dataset { + // Retrieve the requested ethash dataset epoch := block / epochLength currentI, futureI := ethash.datasets.get(epoch) current := currentI.(*dataset) - // Wait for generation finish. - current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + // If async is specified, generate everything in a background thread + if async && !current.generated() { + go func() { + current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) - // If we need a new future dataset, now's a good time to regenerate it. - if futureI != nil { - future := futureI.(*dataset) - go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + if futureI != nil { + future := futureI.(*dataset) + future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + } + }() + } else { + // Either blocking generation was requested, or already done + current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + + if futureI != nil { + future := futureI.(*dataset) + go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + } } - return current } diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 03d8484739..c3b2c86d12 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -114,7 +114,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s hash = header.HashNoNonce().Bytes() target = new(big.Int).Div(two256, header.Difficulty) number = header.Number.Uint64() - dataset = ethash.dataset(number) + dataset = ethash.dataset(number, false) ) // Start generating random nonces until we abort or find a good one var ( @@ -233,21 +233,22 @@ func (ethash *Ethash) remote(notify []string) { log.Info("Work submitted but none pending", "hash", hash) return false } - // Verify the correctness of submitted result. header := block.Header() header.Nonce = nonce header.MixDigest = mixDigest - if err := ethash.VerifySeal(nil, header); err != nil { - log.Warn("Invalid proof-of-work submitted", "hash", hash, "err", err) + + start := time.Now() + if err := ethash.verifySeal(nil, header, true); err != nil { + log.Warn("Invalid proof-of-work submitted", "hash", hash, "elapsed", time.Since(start), "err", err) return false } - // Make sure the result channel is created. if ethash.resultCh == nil { log.Warn("Ethash result channel is empty, submitted mining result is rejected") return false } + log.Trace("Verified correct proof-of-work", "hash", hash, "elapsed", time.Since(start)) // Solutions seems to be valid, return to the miner and notify acceptance. select { From e8752f4e9f9be3d2932cd4835a5d72d17ac2338b Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 15 Aug 2018 17:41:52 +0200 Subject: [PATCH 025/126] cmd/swarm, swarm: added access control functionality (#17404) Co-authored-by: Janos Guljas Co-authored-by: Anton Evangelatov Co-authored-by: Balint Gabor --- cmd/swarm/access.go | 219 ++++++++++++ cmd/swarm/access_test.go | 581 ++++++++++++++++++++++++++++++++ cmd/swarm/config.go | 1 + cmd/swarm/download.go | 40 ++- cmd/swarm/list.go | 2 +- cmd/swarm/main.go | 95 +++++- cmd/swarm/run_test.go | 25 +- swarm/api/act.go | 468 +++++++++++++++++++++++++ swarm/api/api.go | 135 +++++--- swarm/api/api_test.go | 68 +++- swarm/api/client/client.go | 48 ++- swarm/api/client/client_test.go | 4 +- swarm/api/encrypt.go | 76 +++++ swarm/api/filesystem.go | 4 +- swarm/api/filesystem_test.go | 4 +- swarm/api/http/middleware.go | 12 +- swarm/api/http/response.go | 2 +- swarm/api/http/server.go | 111 +++--- swarm/api/manifest.go | 69 ++-- swarm/api/manifest_test.go | 8 +- swarm/api/storage.go | 6 +- swarm/api/uri.go | 13 + swarm/fuse/swarmfs_test.go | 2 +- swarm/network_test.go | 2 +- swarm/sctx/sctx.go | 15 + swarm/swarm.go | 4 +- swarm/testutil/http.go | 2 +- 27 files changed, 1829 insertions(+), 187 deletions(-) create mode 100644 cmd/swarm/access.go create mode 100644 cmd/swarm/access_test.go create mode 100644 swarm/api/act.go create mode 100644 swarm/api/encrypt.go diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go new file mode 100644 index 0000000000..12cfbfc1a4 --- /dev/null +++ b/cmd/swarm/access.go @@ -0,0 +1,219 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package main + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/api/client" + "gopkg.in/urfave/cli.v1" +) + +var salt = make([]byte, 32) + +func init() { + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + panic("reading from crypto/rand failed: " + err.Error()) + } +} + +func accessNewPass(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + utils.Fatalf("Expected 1 argument - the ref") + } + + var ( + ae *api.AccessEntry + accessKey []byte + err error + ref = args[0] + password = getPassPhrase("", 0, makePasswordList(ctx)) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ) + accessKey, ae, err = api.DoPasswordNew(ctx, password, salt) + if err != nil { + utils.Fatalf("error getting session key: %v", err) + } + m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if dryRun { + err = printManifests(m, nil) + if err != nil { + utils.Fatalf("had an error printing the manifests: %v", err) + } + } else { + utils.Fatalf("uploading manifests") + err = uploadManifests(ctx, m, nil) + if err != nil { + utils.Fatalf("had an error uploading the manifests: %v", err) + } + } +} + +func accessNewPK(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + utils.Fatalf("Expected 1 argument - the ref") + } + + var ( + ae *api.AccessEntry + sessionKey []byte + err error + ref = args[0] + privateKey = getPrivKey(ctx) + granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ) + sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt) + if err != nil { + utils.Fatalf("error getting session key: %v", err) + } + m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae) + if dryRun { + err = printManifests(m, nil) + if err != nil { + utils.Fatalf("had an error printing the manifests: %v", err) + } + } else { + err = uploadManifests(ctx, m, nil) + if err != nil { + utils.Fatalf("had an error uploading the manifests: %v", err) + } + } +} + +func accessNewACT(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + utils.Fatalf("Expected 1 argument - the ref") + } + + var ( + ae *api.AccessEntry + actManifest *api.Manifest + accessKey []byte + err error + ref = args[0] + grantees = []string{} + actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) + privateKey = getPrivKey(ctx) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ) + + bytes, err := ioutil.ReadFile(actFilename) + if err != nil { + utils.Fatalf("had an error reading the grantee public key list") + } + grantees = strings.Split(string(bytes), "\n") + accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees) + if err != nil { + utils.Fatalf("error generating ACT manifest: %v", err) + } + + if err != nil { + utils.Fatalf("error getting session key: %v", err) + } + m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if err != nil { + utils.Fatalf("error generating root access manifest: %v", err) + } + + if dryRun { + err = printManifests(m, actManifest) + if err != nil { + utils.Fatalf("had an error printing the manifests: %v", err) + } + } else { + err = uploadManifests(ctx, m, actManifest) + if err != nil { + utils.Fatalf("had an error uploading the manifests: %v", err) + } + } +} + +func printManifests(rootAccessManifest, actManifest *api.Manifest) error { + js, err := json.Marshal(rootAccessManifest) + if err != nil { + return err + } + fmt.Println(string(js)) + + if actManifest != nil { + js, err := json.Marshal(actManifest) + if err != nil { + return err + } + fmt.Println(string(js)) + } + return nil +} + +func uploadManifests(ctx *cli.Context, rootAccessManifest, actManifest *api.Manifest) error { + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := client.NewClient(bzzapi) + + var ( + key string + err error + ) + if actManifest != nil { + key, err = client.UploadManifest(actManifest, false) + if err != nil { + return err + } + + rootAccessManifest.Entries[0].Access.Act = key + } + key, err = client.UploadManifest(rootAccessManifest, false) + if err != nil { + return err + } + fmt.Println(key) + return nil +} + +// makePasswordList reads password lines from the file specified by the global --password flag +// and also by the same subcommand --password flag. +// This function ia a fork of utils.MakePasswordList to lookup cli context for subcommand. +// Function ctx.SetGlobal is not setting the global flag value that can be accessed +// by ctx.GlobalString using the current version of cli package. +func makePasswordList(ctx *cli.Context) []string { + path := ctx.GlobalString(utils.PasswordFileFlag.Name) + if path == "" { + path = ctx.String(utils.PasswordFileFlag.Name) + if path == "" { + return nil + } + } + text, err := ioutil.ReadFile(path) + if err != nil { + utils.Fatalf("Failed to read password file: %v", err) + } + lines := strings.Split(string(text), "\n") + // Sanitise DOS line endings. + for i := range lines { + lines[i] = strings.TrimRight(lines[i], "\r") + } + return lines +} diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go new file mode 100644 index 0000000000..163eb2b4d6 --- /dev/null +++ b/cmd/swarm/access_test.go @@ -0,0 +1,581 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package main + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "io" + "io/ioutil" + gorand "math/rand" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" +) + +// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. +// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry +// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded +// is then fetched through 2nd node. since the tested code is not key-aware - we can just +// fetch from the 2nd node using HTTP BasicAuth +func TestAccessPassword(t *testing.T) { + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + proxyNode := cluster.Nodes[0] + + // create a tmp file + tmp, err := ioutil.TempDir("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + // write data to file + data := "notsorandomdata" + dataFilename := filepath.Join(tmp, "data.txt") + + err = ioutil.WriteFile(dataFilename, []byte(data), 0666) + if err != nil { + t.Fatal(err) + } + + hashRegexp := `[a-f\d]{128}` + + // upload the file with 'swarm up' and expect a hash + up := runSwarm(t, + "--bzzapi", + proxyNode.URL, //it doesn't matter through which node we upload content + "up", + "--encrypt", + dataFilename) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + + ref := matches[0] + + password := "smth" + passwordFilename := filepath.Join(tmp, "password.txt") + + err = ioutil.WriteFile(passwordFilename, []byte(password), 0666) + if err != nil { + t.Fatal(err) + } + + up = runSwarm(t, + "access", + "new", + "pass", + "--dry-run", + "--password", + passwordFilename, + ref, + ) + + _, matches = up.ExpectRegexp(".+") + up.ExpectExit() + + if len(matches) == 0 { + t.Fatalf("stdout not matched") + } + + var m api.Manifest + + err = json.Unmarshal([]byte(matches[0]), &m) + if err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + + if len(m.Entries) != 1 { + t.Fatalf("expected one manifest entry, got %v", len(m.Entries)) + } + + e := m.Entries[0] + + ct := "application/bzz-manifest+json" + if e.ContentType != ct { + t.Errorf("expected %q content type, got %q", ct, e.ContentType) + } + + if e.Access == nil { + t.Fatal("manifest access is nil") + } + + a := e.Access + + if a.Type != "pass" { + t.Errorf(`got access type %q, expected "pass"`, a.Type) + } + if len(a.Salt) < 32 { + t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) + } + if a.KdfParams == nil { + t.Fatal("manifest access kdf params is nil") + } + + client := swarm.NewClient(cluster.Nodes[0].URL) + + hash, err := client.UploadManifest(&m, false) + if err != nil { + t.Fatal(err) + } + + httpClient := &http.Client{} + + url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash + response, err := httpClient.Get(url) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusUnauthorized { + t.Fatal("should be a 401") + } + authHeader := response.Header.Get("WWW-Authenticate") + if authHeader == "" { + t.Fatal("should be something here") + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth("", password) + + response, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Errorf("expected status %v, got %v", http.StatusOK, response.StatusCode) + } + d, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(d) != data { + t.Errorf("expected decrypted data %q, got %q", data, string(d)) + } + + wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt") + + err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666) + if err != nil { + t.Fatal(err) + } + + //download file with 'swarm down' with wrong password + up = runSwarm(t, + "--bzzapi", + proxyNode.URL, + "down", + "bzz:/"+hash, + tmp, + "--password", + wrongPasswordFilename) + + _, matches = up.ExpectRegexp("unauthorized") + if len(matches) != 1 && matches[0] != "unauthorized" { + t.Fatal(`"unauthorized" not found in output"`) + } + up.ExpectExit() +} + +// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). +// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry +// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears. +// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware, +// the test will fail if the proxy's given private key is not granted on the ACT. +func TestAccessPK(t *testing.T) { + // Setup Swarm and upload a test file to it + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + // create a tmp file + tmp, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer tmp.Close() + defer os.Remove(tmp.Name()) + + // write data to file + data := "notsorandomdata" + _, err = io.WriteString(tmp, data) + if err != nil { + t.Fatal(err) + } + + hashRegexp := `[a-f\d]{128}` + + // upload the file with 'swarm up' and expect a hash + up := runSwarm(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + "--encrypt", + tmp.Name()) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + + ref := matches[0] + + pk := cluster.Nodes[0].PrivateKey + granteePubKey := crypto.CompressPubkey(&pk.PublicKey) + + publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") + if err != nil { + t.Fatal(err) + } + + passFile, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer passFile.Close() + defer os.Remove(passFile.Name()) + _, err = io.WriteString(passFile, testPassphrase) + if err != nil { + t.Fatal(err) + } + _, publisherAccount := getTestAccount(t, publisherDir) + up = runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "--bzzapi", + cluster.Nodes[0].URL, + "access", + "new", + "pk", + "--dry-run", + "--grant-key", + hex.EncodeToString(granteePubKey), + ref, + ) + + _, matches = up.ExpectRegexp(".+") + up.ExpectExit() + + if len(matches) == 0 { + t.Fatalf("stdout not matched") + } + + var m api.Manifest + + err = json.Unmarshal([]byte(matches[0]), &m) + if err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + + if len(m.Entries) != 1 { + t.Fatalf("expected one manifest entry, got %v", len(m.Entries)) + } + + e := m.Entries[0] + + ct := "application/bzz-manifest+json" + if e.ContentType != ct { + t.Errorf("expected %q content type, got %q", ct, e.ContentType) + } + + if e.Access == nil { + t.Fatal("manifest access is nil") + } + + a := e.Access + + if a.Type != "pk" { + t.Errorf(`got access type %q, expected "pk"`, a.Type) + } + if len(a.Salt) < 32 { + t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) + } + if a.KdfParams != nil { + t.Fatal("manifest access kdf params should be nil") + } + + client := swarm.NewClient(cluster.Nodes[0].URL) + + hash, err := client.UploadManifest(&m, false) + if err != nil { + t.Fatal(err) + } + + httpClient := &http.Client{} + + url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash + response, err := httpClient.Get(url) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatal("should be a 200") + } + d, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(d) != data { + t.Errorf("expected decrypted data %q, got %q", data, string(d)) + } +} + +// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control +// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data +// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through +// one of the nodes then disappears. +func TestAccessACT(t *testing.T) { + // Setup Swarm and upload a test file to it + cluster := newTestCluster(t, 3) + defer cluster.Shutdown() + + var uploadThroughNode = cluster.Nodes[0] + client := swarm.NewClient(uploadThroughNode.URL) + + r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) + nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`) + // create a tmp file + tmp, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer tmp.Close() + defer os.Remove(tmp.Name()) + + // write data to file + data := "notsorandomdata" + _, err = io.WriteString(tmp, data) + if err != nil { + t.Fatal(err) + } + + hashRegexp := `[a-f\d]{128}` + + // upload the file with 'swarm up' and expect a hash + up := runSwarm(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + "--encrypt", + tmp.Name()) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + + ref := matches[0] + grantees := []string{} + for i, v := range cluster.Nodes { + if i == nodeToSkip { + continue + } + pk := v.PrivateKey + granteePubKey := crypto.CompressPubkey(&pk.PublicKey) + grantees = append(grantees, hex.EncodeToString(granteePubKey)) + } + + granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv") + if err != nil { + t.Fatal(err) + } + + _, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) + if err != nil { + t.Fatal(err) + } + + defer granteesPubkeyListFile.Close() + defer os.Remove(granteesPubkeyListFile.Name()) + + publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") + if err != nil { + t.Fatal(err) + } + + passFile, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer passFile.Close() + defer os.Remove(passFile.Name()) + _, err = io.WriteString(passFile, testPassphrase) + if err != nil { + t.Fatal(err) + } + + _, publisherAccount := getTestAccount(t, publisherDir) + up = runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "--bzzapi", + cluster.Nodes[0].URL, + "access", + "new", + "act", + "--grant-keys", + granteesPubkeyListFile.Name(), + ref, + ) + + _, matches = up.ExpectRegexp(`[a-f\d]{64}`) + up.ExpectExit() + + if len(matches) == 0 { + t.Fatalf("stdout not matched") + } + hash := matches[0] + m, _, err := client.DownloadManifest(hash) + if err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + + if len(m.Entries) != 1 { + t.Fatalf("expected one manifest entry, got %v", len(m.Entries)) + } + + e := m.Entries[0] + + ct := "application/bzz-manifest+json" + if e.ContentType != ct { + t.Errorf("expected %q content type, got %q", ct, e.ContentType) + } + + if e.Access == nil { + t.Fatal("manifest access is nil") + } + + a := e.Access + + if a.Type != "act" { + t.Fatalf(`got access type %q, expected "act"`, a.Type) + } + if len(a.Salt) < 32 { + t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) + } + if a.KdfParams != nil { + t.Fatal("manifest access kdf params should be nil") + } + + httpClient := &http.Client{} + + // all nodes except the skipped node should be able to decrypt the content + for i, node := range cluster.Nodes { + log.Debug("trying to fetch from node", "node index", i) + + url := node.URL + "/" + "bzz:/" + hash + response, err := httpClient.Get(url) + if err != nil { + t.Fatal(err) + } + log.Debug("got response from node", "response code", response.StatusCode) + + if i == nodeToSkip { + log.Debug("reached node to skip", "status code", response.StatusCode) + + if response.StatusCode != http.StatusUnauthorized { + t.Fatalf("should be a 401") + } + + continue + } + + if response.StatusCode != http.StatusOK { + t.Fatal("should be a 200") + } + d, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(d) != data { + t.Errorf("expected decrypted data %q, got %q", data, string(d)) + } + } +} + +// TestKeypairSanity is a sanity test for the crypto scheme for ACT. it asserts the correct shared secret according to +// the specs at https://github.com/ethersphere/swarm-docs/blob/eb857afda906c6e7bb90d37f3f334ccce5eef230/act.md +func TestKeypairSanity(t *testing.T) { + salt := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + t.Fatalf("reading from crypto/rand failed: %v", err.Error()) + } + sharedSecret := "a85586744a1ddd56a7ed9f33fa24f40dd745b3a941be296a0d60e329dbdb896d" + + for i, v := range []struct { + publisherPriv string + granteePub string + }{ + { + publisherPriv: "ec5541555f3bc6376788425e9d1a62f55a82901683fd7062c5eddcc373a73459", + granteePub: "0226f213613e843a413ad35b40f193910d26eb35f00154afcde9ded57479a6224a", + }, + { + publisherPriv: "70c7a73011aa56584a0009ab874794ee7e5652fd0c6911cd02f8b6267dd82d2d", + granteePub: "02e6f8d5e28faaa899744972bb847b6eb805a160494690c9ee7197ae9f619181db", + }, + } { + b, _ := hex.DecodeString(v.granteePub) + granteePub, _ := crypto.DecompressPubkey(b) + publisherPrivate, _ := crypto.HexToECDSA(v.publisherPriv) + + ssKey, err := api.NewSessionKeyPK(publisherPrivate, granteePub, salt) + if err != nil { + t.Fatal(err) + } + + hasher := sha3.NewKeccak256() + hasher.Write(salt) + shared, err := hex.DecodeString(sharedSecret) + if err != nil { + t.Fatal(err) + } + hasher.Write(shared) + sum := hasher.Sum(nil) + + if !bytes.Equal(ssKey, sum) { + t.Fatalf("%d: got a session key mismatch", i) + } + } +} diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index cda8c41c32..1183f8bc81 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -78,6 +78,7 @@ const ( SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" GETH_ENV_DATADIR = "GETH_DATADIR" ) diff --git a/cmd/swarm/download.go b/cmd/swarm/download.go index c2418f744c..91bc2c93ab 100644 --- a/cmd/swarm/download.go +++ b/cmd/swarm/download.go @@ -68,18 +68,36 @@ func download(ctx *cli.Context) { utils.Fatalf("could not parse uri argument: %v", err) } - // assume behaviour according to --recursive switch - if isRecursive { - if err := client.DownloadDirectory(uri.Addr, uri.Path, dest); err != nil { - utils.Fatalf("encoutered an error while downloading directory: %v", err) - } - } else { - // we are downloading a file - log.Debug(fmt.Sprintf("downloading file/path from a manifest. hash: %s, path:%s", uri.Addr, uri.Path)) + dl := func(credentials string) error { + // assume behaviour according to --recursive switch + if isRecursive { + if err := client.DownloadDirectory(uri.Addr, uri.Path, dest, credentials); err != nil { + if err == swarm.ErrUnauthorized { + return err + } + return fmt.Errorf("directory %s: %v", uri.Path, err) + } + } else { + // we are downloading a file + log.Debug("downloading file/path from a manifest", "uri.Addr", uri.Addr, "uri.Path", uri.Path) - err := client.DownloadFile(uri.Addr, uri.Path, dest) - if err != nil { - utils.Fatalf("could not download %s from given address: %s. error: %v", uri.Path, uri.Addr, err) + err := client.DownloadFile(uri.Addr, uri.Path, dest, credentials) + if err != nil { + if err == swarm.ErrUnauthorized { + return err + } + return fmt.Errorf("file %s from address: %s: %v", uri.Path, uri.Addr, err) + } } + return nil + } + if passwords := makePasswordList(ctx); passwords != nil { + password := getPassPhrase(fmt.Sprintf("Downloading %s is restricted", uri), 0, passwords) + err = dl(password) + } else { + err = dl("") + } + if err != nil { + utils.Fatalf("download: %v", err) } } diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go index 57b5517c6e..01b3f4ab6c 100644 --- a/cmd/swarm/list.go +++ b/cmd/swarm/list.go @@ -44,7 +44,7 @@ func list(ctx *cli.Context) { bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") client := swarm.NewClient(bzzapi) - list, err := client.List(manifest, prefix) + list, err := client.List(manifest, prefix, "") if err != nil { utils.Fatalf("Failed to generate file and directory list: %s", err) } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index ac09ae9981..76be60cb68 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -155,6 +155,14 @@ var ( Name: "defaultpath", Usage: "path to file served for empty url path (none)", } + SwarmAccessGrantKeyFlag = cli.StringFlag{ + Name: "grant-key", + Usage: "grants a given public key access to an ACT", + } + SwarmAccessGrantKeysFlag = cli.StringFlag{ + Name: "grant-keys", + Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT", + } SwarmUpFromStdinFlag = cli.BoolFlag{ Name: "stdin", Usage: "reads data to be uploaded from stdin", @@ -167,6 +175,15 @@ var ( Name: "encrypt", Usage: "use encrypted upload", } + SwarmAccessPasswordFlag = cli.StringFlag{ + Name: "password", + Usage: "Password", + EnvVar: SWARM_ACCESS_PASSWORD, + } + SwarmDryRunFlag = cli.BoolFlag{ + Name: "dry-run", + Usage: "dry-run", + } CorsStringFlag = cli.StringFlag{ Name: "corsdomain", Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", @@ -252,6 +269,61 @@ func init() { Flags: []cli.Flag{SwarmEncryptedFlag}, Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash", }, + { + CustomHelpTemplate: helpTemplate, + Name: "access", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root manifest", + Subcommands: []cli.Command{ + { + CustomHelpTemplate: helpTemplate, + Name: "new", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + Subcommands: []cli.Command{ + { + Action: accessNewPass, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + }, + Name: "pass", + Usage: "encrypts a reference with a password and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewPK, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + SwarmAccessGrantKeyFlag, + }, + Name: "pk", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewACT, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + SwarmAccessGrantKeysFlag, + SwarmDryRunFlag, + }, + Name: "act", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + }, + }, + }, + }, { CustomHelpTemplate: helpTemplate, Name: "resource", @@ -304,16 +376,13 @@ func init() { Description: "Prints the swarm hash of file or directory", }, { - Action: download, - Name: "down", - Flags: []cli.Flag{SwarmRecursiveFlag}, - Usage: "downloads a swarm manifest or a file inside a manifest", - ArgsUsage: " []", - Description: ` -Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries. -`, + Action: download, + Name: "down", + Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag}, + Usage: "downloads a swarm manifest or a file inside a manifest", + ArgsUsage: " []", + Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`, }, - { Name: "manifest", CustomHelpTemplate: helpTemplate, @@ -413,16 +482,14 @@ pv(1) tool to get a progress bar: Name: "import", Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)", ArgsUsage: " ", - Description: ` -Import chunks from a tar archive into a local chunk database (use - to read from stdin). + Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin). swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar The import may be quite large, consider piping the input through the Unix pv(1) tool to get a progress bar: - pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks - -`, + pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, }, { Action: dbClean, @@ -535,6 +602,7 @@ func version(ctx *cli.Context) error { func bzzd(ctx *cli.Context) error { //build a valid bzzapi.Config from all available sources: //default config, file config, command line and env vars + bzzconfig, err := buildConfig(ctx) if err != nil { utils.Fatalf("unable to configure swarm: %v", err) @@ -557,6 +625,7 @@ func bzzd(ctx *cli.Context) error { if err != nil { utils.Fatalf("can't create node: %v", err) } + //a few steps need to be done after the config phase is completed, //due to overriding behavior initSwarmNode(bzzconfig, stack, ctx) diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 90d3c98ba6..3e766dc10d 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -18,10 +18,12 @@ package main import ( "context" + "crypto/ecdsa" "fmt" "io/ioutil" "net" "os" + "path" "path/filepath" "runtime" "sync" @@ -175,14 +177,15 @@ func (c *testCluster) Cleanup() { } type testNode struct { - Name string - Addr string - URL string - Enode string - Dir string - IpcPath string - Client *rpc.Client - Cmd *cmdtest.TestCmd + Name string + Addr string + URL string + Enode string + Dir string + IpcPath string + PrivateKey *ecdsa.PrivateKey + Client *rpc.Client + Cmd *cmdtest.TestCmd } const testPassphrase = "swarm-test-passphrase" @@ -289,7 +292,11 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { func newTestNode(t *testing.T, dir string) *testNode { conf, account := getTestAccount(t, dir) - node := &testNode{Dir: dir} + ks := keystore.NewKeyStore(path.Join(dir, "keystore"), 1<<18, 1) + + pk := decryptStoreAccount(ks, account.Address.Hex(), []string{testPassphrase}) + + node := &testNode{Dir: dir, PrivateKey: pk} // assign ports ports, err := getAvailableTCPPorts(2) diff --git a/swarm/api/act.go b/swarm/api/act.go new file mode 100644 index 0000000000..b1a5947831 --- /dev/null +++ b/swarm/api/act.go @@ -0,0 +1,468 @@ +package api + +import ( + "context" + "crypto/ecdsa" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/ecies" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/sctx" + "github.com/ethereum/go-ethereum/swarm/storage" + "golang.org/x/crypto/scrypt" + cli "gopkg.in/urfave/cli.v1" +) + +var ( + ErrDecrypt = errors.New("cant decrypt - forbidden") + ErrUnknownAccessType = errors.New("unknown access type (or not implemented)") + ErrDecryptDomainForbidden = errors.New("decryption request domain forbidden - can only decrypt on localhost") + AllowedDecryptDomains = []string{ + "localhost", + "127.0.0.1", + } +) + +const EMPTY_CREDENTIALS = "" + +type AccessEntry struct { + Type AccessType + Publisher string + Salt []byte + Act string + KdfParams *KdfParams +} + +type DecryptFunc func(*ManifestEntry) error + +func (a *AccessEntry) MarshalJSON() (out []byte, err error) { + + return json.Marshal(struct { + Type AccessType `json:"type,omitempty"` + Publisher string `json:"publisher,omitempty"` + Salt string `json:"salt,omitempty"` + Act string `json:"act,omitempty"` + KdfParams *KdfParams `json:"kdf_params,omitempty"` + }{ + Type: a.Type, + Publisher: a.Publisher, + Salt: hex.EncodeToString(a.Salt), + Act: a.Act, + KdfParams: a.KdfParams, + }) + +} + +func (a *AccessEntry) UnmarshalJSON(value []byte) error { + v := struct { + Type AccessType `json:"type,omitempty"` + Publisher string `json:"publisher,omitempty"` + Salt string `json:"salt,omitempty"` + Act string `json:"act,omitempty"` + KdfParams *KdfParams `json:"kdf_params,omitempty"` + }{} + + err := json.Unmarshal(value, &v) + if err != nil { + return err + } + a.Act = v.Act + a.KdfParams = v.KdfParams + a.Publisher = v.Publisher + a.Salt, err = hex.DecodeString(v.Salt) + if err != nil { + return err + } + if len(a.Salt) != 32 { + return errors.New("salt should be 32 bytes long") + } + a.Type = v.Type + return nil +} + +type KdfParams struct { + N int `json:"n"` + P int `json:"p"` + R int `json:"r"` +} + +type AccessType string + +const AccessTypePass = AccessType("pass") +const AccessTypePK = AccessType("pk") +const AccessTypeACT = AccessType("act") + +func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) { + if len(salt) != 32 { + return nil, fmt.Errorf("salt should be 32 bytes long") + } + return &AccessEntry{ + Type: AccessTypePass, + Salt: salt, + KdfParams: kdfParams, + }, nil +} + +func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) { + if len(publisher) != 66 { + return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher)) + } + if len(salt) != 32 { + return nil, fmt.Errorf("salt should be 32 bytes long") + } + return &AccessEntry{ + Type: AccessTypePK, + Publisher: publisher, + Salt: salt, + }, nil +} + +func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) { + if len(salt) != 32 { + return nil, fmt.Errorf("salt should be 32 bytes long") + } + if len(publisher) != 66 { + return nil, fmt.Errorf("publisher should be 66 characters long") + } + + return &AccessEntry{ + Type: AccessTypeACT, + Publisher: publisher, + Salt: salt, + Act: act, + }, nil +} + +func NOOPDecrypt(*ManifestEntry) error { + return nil +} + +var DefaultKdfParams = NewKdfParams(262144, 1, 8) + +func NewKdfParams(n, p, r int) *KdfParams { + + return &KdfParams{ + N: n, + P: p, + R: r, + } +} + +// NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt +// and kdf parameters in the access entry +func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) { + if accessEntry.Type != AccessTypePass { + return nil, errors.New("incorrect access entry type") + } + return scrypt.Key( + []byte(password), + accessEntry.Salt, + accessEntry.KdfParams.N, + accessEntry.KdfParams.R, + accessEntry.KdfParams.P, + 32, + ) +} + +// NewSessionKeyPK creates a new ACT Session Key using an ECDH shared secret for the given key pair and the given salt value +func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt []byte) ([]byte, error) { + granteePubEcies := ecies.ImportECDSAPublic(public) + privateKey := ecies.ImportECDSA(private) + + bytes, err := privateKey.GenerateShared(granteePubEcies, 16, 16) + if err != nil { + return nil, err + } + bytes = append(salt, bytes...) + sessionKey := crypto.Keccak256(bytes) + return sessionKey, nil +} + +func (a *API) NodeSessionKey(privateKey *ecdsa.PrivateKey, publicKey *ecdsa.PublicKey, salt []byte) ([]byte, error) { + return NewSessionKeyPK(privateKey, publicKey, salt) +} +func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc { + return func(m *ManifestEntry) error { + if m.Access == nil { + return nil + } + + allowed := false + requestDomain := sctx.GetHost(ctx) + for _, v := range AllowedDecryptDomains { + if strings.Contains(requestDomain, v) { + allowed = true + } + } + + if !allowed { + return ErrDecryptDomainForbidden + } + + switch m.Access.Type { + case "pass": + if credentials != "" { + key, err := NewSessionKeyPassword(credentials, m.Access) + if err != nil { + return err + } + + ref, err := hex.DecodeString(m.Hash) + if err != nil { + return err + } + + enc := NewRefEncryption(len(ref) - 8) + decodedRef, err := enc.Decrypt(ref, key) + if err != nil { + return ErrDecrypt + } + + m.Hash = hex.EncodeToString(decodedRef) + m.Access = nil + return nil + } + return ErrDecrypt + case "pk": + publisherBytes, err := hex.DecodeString(m.Access.Publisher) + if err != nil { + return ErrDecrypt + } + publisher, err := crypto.DecompressPubkey(publisherBytes) + if err != nil { + return ErrDecrypt + } + key, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + if err != nil { + return ErrDecrypt + } + ref, err := hex.DecodeString(m.Hash) + if err != nil { + return err + } + + enc := NewRefEncryption(len(ref) - 8) + decodedRef, err := enc.Decrypt(ref, key) + if err != nil { + return ErrDecrypt + } + + m.Hash = hex.EncodeToString(decodedRef) + m.Access = nil + return nil + case "act": + publisherBytes, err := hex.DecodeString(m.Access.Publisher) + if err != nil { + return ErrDecrypt + } + publisher, err := crypto.DecompressPubkey(publisherBytes) + if err != nil { + return ErrDecrypt + } + + sessionKey, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + if err != nil { + return ErrDecrypt + } + + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + + hasher.Reset() + + hasher.Write(append(sessionKey, 1)) + accessKeyDecryptionKey := hasher.Sum(nil) + + lk := hex.EncodeToString(lookupKey) + list, err := a.GetManifestList(ctx, NOOPDecrypt, storage.Address(common.Hex2Bytes(m.Access.Act)), lk) + + found := "" + for _, v := range list.Entries { + if v.Path == lk { + found = v.Hash + } + } + + if found == "" { + return ErrDecrypt + } + + v, err := hex.DecodeString(found) + if err != nil { + return err + } + enc := NewRefEncryption(len(v) - 8) + decodedRef, err := enc.Decrypt(v, accessKeyDecryptionKey) + if err != nil { + return ErrDecrypt + } + + ref, err := hex.DecodeString(m.Hash) + if err != nil { + return err + } + + enc = NewRefEncryption(len(ref) - 8) + decodedMainRef, err := enc.Decrypt(ref, decodedRef) + if err != nil { + return ErrDecrypt + } + m.Hash = hex.EncodeToString(decodedMainRef) + m.Access = nil + return nil + } + return ErrUnknownAccessType + } +} + +func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) { + refBytes, err := hex.DecodeString(ref) + if err != nil { + return nil, err + } + // encrypt ref with accessKey + enc := NewRefEncryption(len(refBytes)) + encrypted, err := enc.Encrypt(refBytes, accessKey) + if err != nil { + return nil, err + } + + m := &Manifest{ + Entries: []ManifestEntry{ + { + Hash: hex.EncodeToString(encrypted), + ContentType: ManifestType, + ModTime: time.Now(), + Access: ae, + }, + }, + } + + return m, nil +} + +func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { + if granteePublicKey == "" { + return nil, nil, errors.New("need a grantee Public Key") + } + b, err := hex.DecodeString(granteePublicKey) + if err != nil { + log.Error("error decoding grantee public key", "err", err) + return nil, nil, err + } + + granteePub, err := crypto.DecompressPubkey(b) + if err != nil { + log.Error("error decompressing grantee public key", "err", err) + return nil, nil, err + } + + sessionKey, err = NewSessionKeyPK(privateKey, granteePub, salt) + if err != nil { + log.Error("error getting session key", "err", err) + return nil, nil, err + } + + ae, err = NewAccessEntryPK(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt) + if err != nil { + log.Error("error generating access entry", "err", err) + return nil, nil, err + } + + return sessionKey, ae, nil +} + +func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) { + if len(grantees) == 0 { + return nil, nil, nil, errors.New("did not get any grantee public keys") + } + + publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + grantees = append(grantees, publisherPub) + + accessKey = make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + panic("reading from crypto/rand failed: " + err.Error()) + } + if _, err := io.ReadFull(rand.Reader, accessKey); err != nil { + panic("reading from crypto/rand failed: " + err.Error()) + } + + lookupPathEncryptedAccessKeyMap := make(map[string]string) + i := 0 + for _, v := range grantees { + i++ + if v == "" { + return nil, nil, nil, errors.New("need a grantee Public Key") + } + b, err := hex.DecodeString(v) + if err != nil { + log.Error("error decoding grantee public key", "err", err) + return nil, nil, nil, err + } + + granteePub, err := crypto.DecompressPubkey(b) + if err != nil { + log.Error("error decompressing grantee public key", "err", err) + return nil, nil, nil, err + } + sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt) + + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + + hasher.Reset() + hasher.Write(append(sessionKey, 1)) + + accessKeyEncryptionKey := hasher.Sum(nil) + + enc := NewRefEncryption(len(accessKey)) + encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey) + + lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey) + } + + m := &Manifest{ + Entries: []ManifestEntry{}, + } + + for k, v := range lookupPathEncryptedAccessKeyMap { + m.Entries = append(m.Entries, ManifestEntry{ + Path: k, + Hash: v, + ContentType: "text/plain", + }) + } + + ae, err = NewAccessEntryACT(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt, "") + if err != nil { + return nil, nil, nil, err + } + + return accessKey, ae, m, nil +} + +func DoPasswordNew(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { + ae, err = NewAccessEntryPassword(salt, DefaultKdfParams) + if err != nil { + return nil, nil, err + } + + sessionKey, err = NewSessionKeyPassword(password, ae) + if err != nil { + return nil, nil, err + } + return sessionKey, ae, nil +} diff --git a/swarm/api/api.go b/swarm/api/api.go index 99d971b105..adf469cfaa 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -19,6 +19,9 @@ package api import ( "archive/tar" "context" + "crypto/ecdsa" + "encoding/hex" + "errors" "fmt" "io" "math/big" @@ -43,6 +46,10 @@ import ( opentracing "github.com/opentracing/opentracing-go" ) +var ( + ErrNotFound = errors.New("not found") +) + var ( apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) @@ -227,14 +234,18 @@ type API struct { resource *mru.Handler fileStore *storage.FileStore dns Resolver + Decryptor func(context.Context, string) DecryptFunc } // NewAPI the api constructor initialises a new API instance. -func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler) (self *API) { +func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler, pk *ecdsa.PrivateKey) (self *API) { self = &API{ fileStore: fileStore, dns: dns, resource: resourceHandler, + Decryptor: func(ctx context.Context, credentials string) DecryptFunc { + return self.doDecrypt(ctx, credentials, pk) + }, } return } @@ -260,8 +271,30 @@ func (a *API) Store(ctx context.Context, data io.Reader, size int64, toEncrypt b // ErrResolve is returned when an URI cannot be resolved from ENS. type ErrResolve error +// Resolve a name into a content-addressed hash +// where address could be an ENS name, or a content addressed hash +func (a *API) Resolve(ctx context.Context, address string) (storage.Address, error) { + // if DNS is not configured, return an error + if a.dns == nil { + if hashMatcher.MatchString(address) { + return common.Hex2Bytes(address), nil + } + apiResolveFail.Inc(1) + return nil, fmt.Errorf("no DNS to resolve name: %q", address) + } + // try and resolve the address + resolved, err := a.dns.Resolve(address) + if err != nil { + if hashMatcher.MatchString(address) { + return common.Hex2Bytes(address), nil + } + return nil, err + } + return resolved[:], nil +} + // Resolve resolves a URI to an Address using the MultiResolver. -func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) { +func (a *API) ResolveURI(ctx context.Context, uri *URI, credentials string) (storage.Address, error) { apiResolveCount.Inc(1) log.Trace("resolving", "uri", uri.Addr) @@ -280,28 +313,44 @@ func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) { return key, nil } - // if DNS is not configured, check if the address is a hash - if a.dns == nil { - key := uri.Address() - if key == nil { - apiResolveFail.Inc(1) - return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) - } - return key, nil - } - - // try and resolve the address - resolved, err := a.dns.Resolve(uri.Addr) - if err == nil { - return resolved[:], nil - } - - key := uri.Address() - if key == nil { - apiResolveFail.Inc(1) + addr, err := a.Resolve(ctx, uri.Addr) + if err != nil { return nil, err } - return key, nil + + if uri.Path == "" { + return addr, nil + } + walker, err := a.NewManifestWalker(ctx, addr, a.Decryptor(ctx, credentials), nil) + if err != nil { + return nil, err + } + var entry *ManifestEntry + walker.Walk(func(e *ManifestEntry) error { + // if the entry matches the path, set entry and stop + // the walk + if e.Path == uri.Path { + entry = e + // return an error to cancel the walk + return errors.New("found") + } + // ignore non-manifest files + if e.ContentType != ManifestType { + return nil + } + // if the manifest's path is a prefix of the + // requested path, recurse into it by returning + // nil and continuing the walk + if strings.HasPrefix(uri.Path, e.Path) { + return nil + } + return ErrSkipManifest + }) + if entry == nil { + return nil, errors.New("not found") + } + addr = storage.Address(common.Hex2Bytes(entry.Hash)) + return addr, nil } // Put provides singleton manifest creation on top of FileStore store @@ -332,10 +381,10 @@ func (a *API) Put(ctx context.Context, content string, contentType string, toEnc // Get uses iterative manifest retrieval and prefix matching // to resolve basePath to content using FileStore retrieve // it returns a section reader, mimeType, status, the key of the actual content and an error -func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) { +func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) { log.Debug("api.get", "key", manifestAddr, "path", path) apiGetCount.Inc(1) - trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil) + trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound @@ -347,6 +396,16 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string if entry != nil { log.Debug("trie got entry", "key", manifestAddr, "path", path, "entry.Hash", entry.Hash) + + if entry.ContentType == ManifestType { + log.Debug("entry is manifest", "key", manifestAddr, "new key", entry.Hash) + adr, err := hex.DecodeString(entry.Hash) + if err != nil { + return nil, "", 0, nil, err + } + return a.Get(ctx, decrypt, adr, entry.Path) + } + // we need to do some extra work if this is a mutable resource manifest if entry.ContentType == ResourceContentType { @@ -398,7 +457,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string log.Trace("resource is multihash", "key", manifestAddr) // get the manifest the multihash digest points to - trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil) + trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound @@ -451,7 +510,7 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add apiDeleteFail.Inc(1) return nil, err } - key, err := a.Resolve(ctx, uri) + key, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { return nil, err @@ -470,13 +529,13 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add // GetDirectoryTar fetches a requested directory as a tarstream // it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser -func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) { +func (a *API) GetDirectoryTar(ctx context.Context, decrypt DecryptFunc, uri *URI) (io.ReadCloser, error) { apiGetTarCount.Inc(1) - addr, err := a.Resolve(ctx, uri) + addr, err := a.Resolve(ctx, uri.Addr) if err != nil { return nil, err } - walker, err := a.NewManifestWalker(ctx, addr, nil) + walker, err := a.NewManifestWalker(ctx, addr, decrypt, nil) if err != nil { apiGetTarFail.Inc(1) return nil, err @@ -542,9 +601,9 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err // GetManifestList lists the manifest entries for the specified address and prefix // and returns it as a ManifestList -func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) { +func (a *API) GetManifestList(ctx context.Context, decryptor DecryptFunc, addr storage.Address, prefix string) (list ManifestList, err error) { apiManifestListCount.Inc(1) - walker, err := a.NewManifestWalker(ctx, addr, nil) + walker, err := a.NewManifestWalker(ctx, addr, decryptor, nil) if err != nil { apiManifestListFail.Inc(1) return ManifestList{}, err @@ -631,7 +690,7 @@ func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update f func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) { apiModifyCount.Inc(1) quitC := make(chan bool) - trie, err := loadManifest(ctx, a.fileStore, addr, quitC) + trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) if err != nil { apiModifyFail.Inc(1) return nil, err @@ -663,7 +722,7 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] apiAddFileFail.Inc(1) return nil, "", err } - mkey, err := a.Resolve(ctx, uri) + mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { apiAddFileFail.Inc(1) return nil, "", err @@ -770,7 +829,7 @@ func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname s apiRmFileFail.Inc(1) return "", err } - mkey, err := a.Resolve(ctx, uri) + mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { apiRmFileFail.Inc(1) return "", err @@ -837,7 +896,7 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin apiAppendFileFail.Inc(1) return nil, "", err } - mkey, err := a.Resolve(ctx, uri) + mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { apiAppendFileFail.Inc(1) return nil, "", err @@ -891,13 +950,13 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver if err != nil { return nil, nil, err } - addr, err = a.Resolve(ctx, uri) + addr, err = a.Resolve(ctx, uri.Addr) if err != nil { return nil, nil, err } quitC := make(chan bool) - rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC) + rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) if err != nil { return nil, nil, fmt.Errorf("can't load manifest %v: %v", addr.String(), err) } @@ -955,7 +1014,7 @@ func (a *API) ResourceHashSize() int { // ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk. func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) { - trie, err := loadManifest(ctx, a.fileStore, addr, nil) + trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt) if err != nil { return nil, fmt.Errorf("cannot load resource manifest: %v", err) } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 78fab9508a..a65bf07e2e 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -19,6 +19,7 @@ package api import ( "context" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -28,10 +29,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage" ) +func init() { + loglevel := flag.Int("loglevel", 2, "loglevel") + flag.Parse() + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + func testAPI(t *testing.T, f func(*API, bool)) { datadir, err := ioutil.TempDir("", "bzz-test") if err != nil { @@ -42,7 +50,7 @@ func testAPI(t *testing.T, f func(*API, bool)) { if err != nil { return } - api := NewAPI(fileStore, nil, nil) + api := NewAPI(fileStore, nil, nil, nil) f(api, false) f(api, true) } @@ -85,7 +93,7 @@ func expResponse(content string, mimeType string, status int) *Response { func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse { addr := storage.Address(common.Hex2Bytes(bzzhash)) - reader, mimeType, status, _, err := api.Get(context.TODO(), addr, path) + reader, mimeType, status, _, err := api.Get(context.TODO(), NOOPDecrypt, addr, path) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -229,7 +237,7 @@ func TestAPIResolve(t *testing.T) { if x.immutable { uri.Scheme = "bzz-immutable" } - res, err := api.Resolve(context.TODO(), uri) + res, err := api.ResolveURI(context.TODO(), uri, "") if err == nil { if x.expectErr != nil { t.Fatalf("expected error %q, got result %q", x.expectErr, res) @@ -373,3 +381,55 @@ func TestMultiResolver(t *testing.T) { }) } } + +func TestDecryptOriginForbidden(t *testing.T) { + ctx := context.TODO() + ctx = sctx.SetHost(ctx, "swarm-gateways.net") + + me := &ManifestEntry{ + Access: &AccessEntry{Type: AccessTypePass}, + } + + api := NewAPI(nil, nil, nil, nil) + + f := api.Decryptor(ctx, "") + err := f(me) + if err != ErrDecryptDomainForbidden { + t.Fatalf("should fail with ErrDecryptDomainForbidden, got %v", err) + } +} + +func TestDecryptOrigin(t *testing.T) { + for _, v := range []struct { + host string + expectError error + }{ + { + host: "localhost", + expectError: ErrDecrypt, + }, + { + host: "127.0.0.1", + expectError: ErrDecrypt, + }, + { + host: "swarm-gateways.net", + expectError: ErrDecryptDomainForbidden, + }, + } { + ctx := context.TODO() + ctx = sctx.SetHost(ctx, v.host) + + me := &ManifestEntry{ + Access: &AccessEntry{Type: AccessTypePass}, + } + + api := NewAPI(nil, nil, nil, nil) + + f := api.Decryptor(ctx, "") + err := f(me) + if err != v.expectError { + t.Fatalf("should fail with %v, got %v", v.expectError, err) + } + } +} diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 8a9efe3608..3d06e9e1cc 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -43,6 +43,10 @@ var ( DefaultClient = NewClient(DefaultGateway) ) +var ( + ErrUnauthorized = errors.New("unauthorized") +) + func NewClient(gateway string) *Client { return &Client{ Gateway: gateway, @@ -188,7 +192,7 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo // DownloadDirectory downloads the files contained in a swarm manifest under // the given path into a local directory (existing files will be overwritten) -func (c *Client) DownloadDirectory(hash, path, destDir string) error { +func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error { stat, err := os.Stat(destDir) if err != nil { return err @@ -201,13 +205,20 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error { if err != nil { return err } + if credentials != "" { + req.SetBasicAuth("", credentials) + } req.Header.Set("Accept", "application/x-tar") res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() - if res.StatusCode != http.StatusOK { + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + return ErrUnauthorized + default: return fmt.Errorf("unexpected HTTP status: %s", res.Status) } tr := tar.NewReader(res.Body) @@ -248,7 +259,7 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error { // DownloadFile downloads a single file into the destination directory // if the manifest entry does not specify a file name - it will fallback // to the hash of the file as a filename -func (c *Client) DownloadFile(hash, path, dest string) error { +func (c *Client) DownloadFile(hash, path, dest, credentials string) error { hasDestinationFilename := false if stat, err := os.Stat(dest); err == nil { hasDestinationFilename = !stat.IsDir() @@ -261,9 +272,9 @@ func (c *Client) DownloadFile(hash, path, dest string) error { } } - manifestList, err := c.List(hash, path) + manifestList, err := c.List(hash, path, credentials) if err != nil { - return fmt.Errorf("could not list manifest: %v", err) + return err } switch len(manifestList.Entries) { @@ -280,13 +291,19 @@ func (c *Client) DownloadFile(hash, path, dest string) error { if err != nil { return err } + if credentials != "" { + req.SetBasicAuth("", credentials) + } res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() - - if res.StatusCode != http.StatusOK { + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + return ErrUnauthorized + default: return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode) } filename := "" @@ -367,13 +384,24 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) { // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] // // where entries ending with "/" are common prefixes. -func (c *Client) List(hash, prefix string) (*api.ManifestList, error) { - res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix) +func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) { + req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil) + if err != nil { + return nil, err + } + if credentials != "" { + req.SetBasicAuth("", credentials) + } + res, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer res.Body.Close() - if res.StatusCode != http.StatusOK { + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + return nil, ErrUnauthorized + default: return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) } var list api.ManifestList diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index ae82a91d79..2212f5c4c3 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -228,7 +228,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmp) - if err := client.DownloadDirectory(hash, "", tmp); err != nil { + if err := client.DownloadDirectory(hash, "", tmp, ""); err != nil { t.Fatal(err) } for _, file := range testDirFiles { @@ -265,7 +265,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { } ls := func(prefix string) []string { - list, err := client.List(hash, prefix) + list, err := client.List(hash, prefix, "") if err != nil { t.Fatal(err) } diff --git a/swarm/api/encrypt.go b/swarm/api/encrypt.go new file mode 100644 index 0000000000..9a2e369149 --- /dev/null +++ b/swarm/api/encrypt.go @@ -0,0 +1,76 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "encoding/binary" + "errors" + + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/storage/encryption" +) + +type RefEncryption struct { + spanEncryption encryption.Encryption + dataEncryption encryption.Encryption + span []byte +} + +func NewRefEncryption(refSize int) *RefEncryption { + span := make([]byte, 8) + binary.LittleEndian.PutUint64(span, uint64(refSize)) + return &RefEncryption{ + spanEncryption: encryption.New(0, uint32(refSize/32), sha3.NewKeccak256), + dataEncryption: encryption.New(refSize, 0, sha3.NewKeccak256), + span: span, + } +} + +func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) { + encryptedSpan, err := re.spanEncryption.Encrypt(re.span, key) + if err != nil { + return nil, err + } + encryptedData, err := re.dataEncryption.Encrypt(ref, key) + if err != nil { + return nil, err + } + encryptedRef := make([]byte, len(ref)+8) + copy(encryptedRef[:8], encryptedSpan) + copy(encryptedRef[8:], encryptedData) + + return encryptedRef, nil +} + +func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) { + decryptedSpan, err := re.spanEncryption.Decrypt(ref[:8], key) + if err != nil { + return nil, err + } + + size := binary.LittleEndian.Uint64(decryptedSpan) + if size != uint64(len(ref)-8) { + return nil, errors.New("invalid span in encrypted reference") + } + + decryptedRef, err := re.dataEncryption.Decrypt(ref[8:], key) + if err != nil { + return nil, err + } + + return decryptedRef, nil +} diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index aacd266998..8251ebc4dc 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -191,7 +191,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error { if err != nil { return err } - addr, err := fs.api.Resolve(context.TODO(), uri) + addr, err := fs.api.Resolve(context.TODO(), uri.Addr) if err != nil { return err } @@ -202,7 +202,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error { } quitC := make(chan bool) - trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC) + trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC, NOOPDecrypt) if err != nil { log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err)) return err diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 84a2989d6e..fe7527b1f1 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -64,7 +64,7 @@ func TestApiDirUpload0(t *testing.T) { checkResponse(t, resp, exp) addr := storage.Address(common.Hex2Bytes(bzzhash)) - _, _, _, _, err = api.Get(context.TODO(), addr, "") + _, _, _, _, err = api.Get(context.TODO(), NOOPDecrypt, addr, "") if err == nil { t.Fatalf("expected error: %v", err) } @@ -143,7 +143,7 @@ func TestApiDirUploadModify(t *testing.T) { exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - _, _, _, _, err = api.Get(context.TODO(), addr, "") + _, _, _, _, err = api.Get(context.TODO(), nil, addr, "") if err == nil { t.Errorf("expected error: %v", err) } diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index c0d8d1a408..3b2dcc7d59 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/pborman/uuid" ) @@ -35,6 +36,15 @@ func SetRequestID(h http.Handler) http.Handler { }) } +func SetRequestHost(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(sctx.SetHost(r.Context(), r.Host)) + log.Info("setting request host", "ruid", GetRUID(r.Context()), "host", sctx.GetHost(r.Context())) + + h.ServeHTTP(w, r) + }) +} + func ParseURI(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) @@ -87,7 +97,7 @@ func RecoverPanic(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer func() { if err := recover(); err != nil { - log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header) + log.Error("panic recovery!", "stack trace", string(debug.Stack()), "url", r.URL.String(), "headers", r.Header) } }() h.ServeHTTP(w, r) diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index f050e706a8..c9fb9d2855 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -79,7 +79,7 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s } func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) { - log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) + log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) RespondTemplate(w, r, "error", msg, code) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 5a5c42adc0..b5ea0c23d4 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -23,7 +23,6 @@ import ( "bufio" "bytes" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -97,6 +96,7 @@ func NewServer(api *api.API, corsString string) *Server { defaultMiddlewares := []Adapter{ RecoverPanic, SetRequestID, + SetRequestHost, InitLoggingResponseWriter, ParseURI, InstrumentOpenTracing, @@ -169,6 +169,7 @@ func NewServer(api *api.API, corsString string) *Server { } func (s *Server) ListenAndServe(addr string) error { + s.listenAddr = addr return http.ListenAndServe(addr, s) } @@ -178,16 +179,24 @@ func (s *Server) ListenAndServe(addr string) error { // https://github.com/atom/electron/blob/master/docs/api/protocol.md type Server struct { http.Handler - api *api.API + api *api.API + listenAddr string } func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { - log.Debug("handleBzzGet", "ruid", GetRUID(r.Context())) + log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI) if r.Header.Get("Accept") == "application/x-tar" { uri := GetURI(r.Context()) - reader, err := s.api.GetDirectoryTar(r.Context(), uri) + _, credentials, _ := r.BasicAuth() + reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri) if err != nil { + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String())) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + return } defer reader.Close() @@ -287,7 +296,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { var addr storage.Address if uri.Addr != "" && uri.Addr != "encrypt" { - addr, err = s.api.Resolve(r.Context(), uri) + addr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { postFilesFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) @@ -563,7 +572,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { // resolve the content key. manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), uri) + manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) @@ -682,62 +691,21 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { uri := GetURI(r.Context()) log.Debug("handle.get", "ruid", ruid, "uri", uri) getCount.Inc(1) + _, pass, _ := r.BasicAuth() - var err error - addr := uri.Address() - if addr == nil { - addr, err = s.api.Resolve(r.Context(), uri) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) - return - } - } else { - w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. + addr, err := s.api.ResolveURI(r.Context(), uri, pass) + if err != nil { + getFail.Inc(1) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + return } + w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. log.Debug("handle.get: resolved", "ruid", ruid, "key", addr) // if path is set, interpret as a manifest and return the // raw entry at the given path - if uri.Path != "" { - walker, err := s.api.NewManifestWalker(r.Context(), addr, nil) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest) - return - } - var entry *api.ManifestEntry - walker.Walk(func(e *api.ManifestEntry) error { - // if the entry matches the path, set entry and stop - // the walk - if e.Path == uri.Path { - entry = e - // return an error to cancel the walk - return errors.New("found") - } - // ignore non-manifest files - if e.ContentType != api.ManifestType { - return nil - } - - // if the manifest's path is a prefix of the - // requested path, recurse into it by returning - // nil and continuing the walk - if strings.HasPrefix(uri.Path, e.Path) { - return nil - } - - return api.ErrSkipManifest - }) - if entry == nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound) - return - } - addr = storage.Address(common.Hex2Bytes(entry.Hash)) - } etag := common.Bytes2Hex(addr) noneMatchEtag := r.Header.Get("If-None-Match") w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key. @@ -781,6 +749,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) + _, credentials, _ := r.BasicAuth() log.Debug("handle.get.list", "ruid", ruid, "uri", uri) getListCount.Inc(1) @@ -790,7 +759,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { return } - addr, err := s.api.Resolve(r.Context(), uri) + addr, err := s.api.Resolve(r.Context(), uri.Addr) if err != nil { getListFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) @@ -798,9 +767,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { } log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr) - list, err := s.api.GetManifestList(r.Context(), addr, uri.Path) + list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), addr, uri.Path) if err != nil { getListFail.Inc(1) + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String())) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } RespondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -833,7 +807,8 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) - log.Debug("handle.get.file", "ruid", ruid) + _, credentials, _ := r.BasicAuth() + log.Debug("handle.get.file", "ruid", ruid, "uri", r.RequestURI) getFileCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work @@ -845,7 +820,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), uri) + manifestAddr, err = s.api.ResolveURI(r.Context(), uri, credentials) if err != nil { getFileFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) @@ -856,7 +831,8 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { } log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr) - reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path) + + reader, contentType, status, contentKey, err := s.api.Get(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path) etag := common.Bytes2Hex(contentKey) noneMatchEtag := r.Header.Get("If-None-Match") @@ -869,6 +845,12 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { } if err != nil { + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } + switch status { case http.StatusNotFound: getFileNotFound.Inc(1) @@ -883,9 +865,14 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { //the request results in ambiguous files //e.g. /read with readme.md and readinglist.txt available in manifest if status == http.StatusMultipleChoices { - list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path) + list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path) if err != nil { getFileFail.Inc(1) + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } RespondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -951,3 +938,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) { lrw.statusCode = code lrw.ResponseWriter.WriteHeader(code) } + +func isDecryptError(err error) bool { + return strings.Contains(err.Error(), api.ErrDecrypt.Error()) +} diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 2a163dd39c..a1329a800f 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -46,13 +46,14 @@ type Manifest struct { // ManifestEntry represents an entry in a swarm manifest type ManifestEntry struct { - Hash string `json:"hash,omitempty"` - Path string `json:"path,omitempty"` - ContentType string `json:"contentType,omitempty"` - Mode int64 `json:"mode,omitempty"` - Size int64 `json:"size,omitempty"` - ModTime time.Time `json:"mod_time,omitempty"` - Status int `json:"status,omitempty"` + Hash string `json:"hash,omitempty"` + Path string `json:"path,omitempty"` + ContentType string `json:"contentType,omitempty"` + Mode int64 `json:"mode,omitempty"` + Size int64 `json:"size,omitempty"` + ModTime time.Time `json:"mod_time,omitempty"` + Status int `json:"status,omitempty"` + Access *AccessEntry `json:"access,omitempty"` } // ManifestList represents the result of listing files in a manifest @@ -98,7 +99,7 @@ type ManifestWriter struct { } func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) { - trie, err := loadManifest(ctx, a.fileStore, addr, quitC) + trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) if err != nil { return nil, fmt.Errorf("error loading manifest %s: %s", addr, err) } @@ -141,8 +142,8 @@ type ManifestWalker struct { quitC chan bool } -func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWalker, error) { - trie, err := loadManifest(ctx, a.fileStore, addr, quitC) +func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) { + trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt) if err != nil { return nil, fmt.Errorf("error loading manifest %s: %s", addr, err) } @@ -194,6 +195,7 @@ type manifestTrie struct { entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry ref storage.Address // if ref != nil, it is stored encrypted bool + decrypt DecryptFunc } func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry { @@ -209,15 +211,15 @@ type manifestTrieEntry struct { subtrie *manifestTrie } -func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand log.Trace("manifest lookup", "key", hash) // retrieve manifest via FileStore manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash) log.Trace("reader retrieved", "key", hash) - return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC) + return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC, decrypt) } -func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests size, err := mr.Size(mr.Context(), quitC) @@ -258,26 +260,41 @@ func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore trie = &manifestTrie{ fileStore: fileStore, encrypted: isEncrypted, + decrypt: decrypt, } for _, entry := range man.Entries { - trie.addEntry(entry, quitC) + err = trie.addEntry(entry, quitC) + if err != nil { + return + } } return } -func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { +func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error { mt.ref = nil // trie modified, hash needs to be re-calculated on demand + if entry.ManifestEntry.Access != nil { + if mt.decrypt == nil { + return errors.New("dont have decryptor") + } + + err := mt.decrypt(&entry.ManifestEntry) + if err != nil { + return err + } + } + if len(entry.Path) == 0 { mt.entries[256] = entry - return + return nil } b := entry.Path[0] oldentry := mt.entries[b] if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) { mt.entries[b] = entry - return + return nil } cpl := 0 @@ -287,12 +304,12 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) { if mt.loadSubTrie(oldentry, quitC) != nil { - return + return nil } entry.Path = entry.Path[cpl:] oldentry.subtrie.addEntry(entry, quitC) oldentry.Hash = "" - return + return nil } commonPrefix := entry.Path[:cpl] @@ -310,6 +327,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { Path: commonPrefix, ContentType: ManifestType, }, subtrie) + return nil } func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { @@ -398,9 +416,20 @@ func (mt *manifestTrie) recalcAndStore() error { } func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) { + if entry.ManifestEntry.Access != nil { + if mt.decrypt == nil { + return errors.New("dont have decryptor") + } + + err := mt.decrypt(&entry.ManifestEntry) + if err != nil { + return err + } + } + if entry.subtrie == nil { hash := common.Hex2Bytes(entry.Hash) - entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC) + entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt) entry.Hash = "" // might not match, should be recalculated } return diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index d65f023f85..1c8e53c433 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -44,7 +44,7 @@ func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...stri quitC := make(chan bool) fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) ref := make([]byte, fileStore.HashSize()) - trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC) + trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC, NOOPDecrypt) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -101,7 +101,7 @@ func TestExactMatch(t *testing.T) { mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) ref := make([]byte, fileStore.HashSize()) - trie, err := readManifest(mf, ref, fileStore, false, quitC) + trie, err := readManifest(mf, ref, fileStore, false, quitC, nil) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -134,7 +134,7 @@ func TestAddFileWithManifestPath(t *testing.T) { } fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) ref := make([]byte, fileStore.HashSize()) - trie, err := readManifest(reader, ref, fileStore, false, nil) + trie, err := readManifest(reader, ref, fileStore, false, nil, NOOPDecrypt) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestReadManifestOverSizeLimit(t *testing.T) { reader := &storage.LazyTestSectionReader{ SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))), } - _, err := readManifest(reader, storage.Address{}, nil, false, nil) + _, err := readManifest(reader, storage.Address{}, nil, false, nil, NOOPDecrypt) if err == nil { t.Fatal("got no error from readManifest") } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 3b52301a01..8a48fe5bc0 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -63,11 +63,11 @@ func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) { if err != nil { return nil, err } - addr, err := s.api.Resolve(ctx, uri) + addr, err := s.api.Resolve(ctx, uri.Addr) if err != nil { return nil, err } - reader, mimeType, status, _, err := s.api.Get(ctx, addr, uri.Path) + reader, mimeType, status, _, err := s.api.Get(ctx, nil, addr, uri.Path) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func (s *Storage) Modify(ctx context.Context, rootHash, path, contentHash, conte if err != nil { return "", err } - addr, err := s.api.Resolve(ctx, uri) + addr, err := s.api.Resolve(ctx, uri.Addr) if err != nil { return "", err } diff --git a/swarm/api/uri.go b/swarm/api/uri.go index 14965e0d93..808517088a 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -53,6 +53,19 @@ type URI struct { Path string } +func (u *URI) MarshalJSON() (out []byte, err error) { + return []byte(`"` + u.String() + `"`), nil +} + +func (u *URI) UnmarshalJSON(value []byte) error { + uri, err := Parse(string(value)) + if err != nil { + return err + } + *u = *uri + return nil +} + // Parse parses rawuri into a URI struct, where rawuri is expected to have one // of the following formats: // diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index d579d15a02..6efeb78d9b 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -1650,7 +1650,7 @@ func TestFUSE(t *testing.T) { if err != nil { t.Fatal(err) } - ta := &testAPI{api: api.NewAPI(fileStore, nil, nil)} + ta := &testAPI{api: api.NewAPI(fileStore, nil, nil, nil)} //run a short suite of tests //approx time: 28s diff --git a/swarm/network_test.go b/swarm/network_test.go index d2a0309338..176c635d82 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -445,7 +445,7 @@ func retrieve( log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount)) - r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/") + r, _, _, _, err := swarm.api.Get(context.TODO(), api.NOOPDecrypt, f.addr, "/") if err != nil { errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err) return diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index 8619f6e19c..bed2b11458 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -1,7 +1,22 @@ package sctx +import "context" + type ContextKey int const ( HTTPRequestIDKey ContextKey = iota + requestHostKey ) + +func SetHost(ctx context.Context, domain string) context.Context { + return context.WithValue(ctx, requestHostKey, domain) +} + +func GetHost(ctx context.Context) string { + v, ok := ctx.Value(requestHostKey).(string) + if ok { + return v + } + return "" +} diff --git a/swarm/swarm.go b/swarm/swarm.go index f731ff33d7..a895bdfa55 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -85,14 +85,12 @@ type Swarm struct { type SwarmAPI struct { Api *api.API Backend chequebook.Backend - PrvKey *ecdsa.PrivateKey } func (self *Swarm) API() *SwarmAPI { return &SwarmAPI{ Api: self.api, Backend: self.backend, - PrvKey: self.privateKey, } } @@ -217,7 +215,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) } - self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler) + self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey) // Manifests for Smart Hosting log.Debug(fmt.Sprintf("-> Web3 virtual server API")) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 238f783088..7fd60fcc3d 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -77,7 +77,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *Tes t.Fatal(err) } - a := api.NewAPI(fileStore, nil, rh.Handler) + a := api.NewAPI(fileStore, nil, rh.Handler, nil) srv := httptest.NewServer(serverFunc(a)) return &TestSwarmServer{ Server: srv, From 2cdf6ee7e00d6127c372e7a28bb27a80ef495cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Wed, 15 Aug 2018 22:25:46 +0200 Subject: [PATCH 026/126] light: CHT and bloom trie indexers working in light mode (#16534) This PR enables the indexers to work in light client mode by downloading a part of these tries (the Merkle proofs of the last values of the last known section) in order to be able to add new values and recalculate subsequent hashes. It also adds CHT data to NodeInfo. --- core/chain_indexer.go | 30 +++++-- core/chain_indexer_test.go | 6 +- eth/backend.go | 2 +- eth/bloombits.go | 23 +++-- les/backend.go | 39 +++++---- les/distributor.go | 4 - les/handler.go | 34 ++++++-- les/helper_test.go | 6 +- les/odr.go | 18 ++-- les/odr_test.go | 3 +- les/request_test.go | 3 +- les/retrieve.go | 3 +- les/server.go | 4 +- light/lightchain.go | 14 +-- light/odr.go | 4 + light/postprocess.go | 170 ++++++++++++++++++++++++++++--------- 16 files changed, 251 insertions(+), 112 deletions(-) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 0b927116d0..11a7c96fa0 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -17,6 +17,7 @@ package core import ( + "context" "encoding/binary" "fmt" "sync" @@ -37,11 +38,11 @@ import ( type ChainIndexerBackend interface { // Reset initiates the processing of a new chain segment, potentially terminating // any partially completed operations (in case of a reorg). - Reset(section uint64, prevHead common.Hash) error + Reset(ctx context.Context, section uint64, prevHead common.Hash) error // Process crunches through the next header in the chain segment. The caller // will ensure a sequential order of headers. - Process(header *types.Header) + Process(ctx context.Context, header *types.Header) error // Commit finalizes the section metadata and stores it into the database. Commit() error @@ -71,9 +72,11 @@ type ChainIndexer struct { backend ChainIndexerBackend // Background processor generating the index data content children []*ChainIndexer // Child indexers to cascade chain updates to - active uint32 // Flag whether the event loop was started - update chan struct{} // Notification channel that headers should be processed - quit chan chan error // Quit channel to tear down running goroutines + active uint32 // Flag whether the event loop was started + update chan struct{} // Notification channel that headers should be processed + quit chan chan error // Quit channel to tear down running goroutines + ctx context.Context + ctxCancel func() sectionSize uint64 // Number of blocks in a single chain segment to process confirmsReq uint64 // Number of confirmations before processing a completed segment @@ -105,6 +108,8 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken } // Initialize database dependent fields and start the updater c.loadValidSections() + c.ctx, c.ctxCancel = context.WithCancel(context.Background()) + go c.updateLoop() return c @@ -138,6 +143,8 @@ func (c *ChainIndexer) Start(chain ChainIndexerChain) { func (c *ChainIndexer) Close() error { var errs []error + c.ctxCancel() + // Tear down the primary update loop errc := make(chan error) c.quit <- errc @@ -297,6 +304,12 @@ func (c *ChainIndexer) updateLoop() { c.lock.Unlock() newHead, err := c.processSection(section, oldHead) if err != nil { + select { + case <-c.ctx.Done(): + <-c.quit <- nil + return + default: + } c.log.Error("Section processing failed", "error", err) } c.lock.Lock() @@ -344,7 +357,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com // Reset and partial processing - if err := c.backend.Reset(section, lastHead); err != nil { + if err := c.backend.Reset(c.ctx, section, lastHead); err != nil { c.setValidSections(0) return common.Hash{}, err } @@ -360,11 +373,12 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com } else if header.ParentHash != lastHead { return common.Hash{}, fmt.Errorf("chain reorged during section processing") } - c.backend.Process(header) + if err := c.backend.Process(c.ctx, header); err != nil { + return common.Hash{}, err + } lastHead = header.Hash() } if err := c.backend.Commit(); err != nil { - c.log.Error("Section commit failed", "error", err) return common.Hash{}, err } return lastHead, nil diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 550caf5567..a029dec626 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" "math/rand" @@ -210,13 +211,13 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { return b.stored * b.indexer.sectionSize } -func (b *testChainIndexBackend) Reset(section uint64, prevHead common.Hash) error { +func (b *testChainIndexBackend) Reset(ctx context.Context, section uint64, prevHead common.Hash) error { b.section = section b.headerCnt = 0 return nil } -func (b *testChainIndexBackend) Process(header *types.Header) { +func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Header) error { b.headerCnt++ if b.headerCnt > b.indexer.sectionSize { b.t.Error("Processing too many headers") @@ -227,6 +228,7 @@ func (b *testChainIndexBackend) Process(header *types.Header) { b.t.Fatal("Unexpected call to Process") case b.processCh <- header.Number.Uint64(): } + return nil } func (b *testChainIndexBackend) Commit() error { diff --git a/eth/backend.go b/eth/backend.go index 865534b198..6549cb8a3d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -130,7 +130,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { gasPrice: config.GasPrice, etherbase: config.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks), + bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, bloomConfirms), } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) diff --git a/eth/bloombits.go b/eth/bloombits.go index 954239d141..eb18565e2c 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -17,6 +17,7 @@ package eth import ( + "context" "time" "github.com/ethereum/go-ethereum/common" @@ -92,30 +93,28 @@ const ( // BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index // for the Ethereum header bloom filters, permitting blazing fast filtering. type BloomIndexer struct { - size uint64 // section size to generate bloombits for - - db ethdb.Database // database instance to write index data and metadata into - gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index - - section uint64 // Section is the section number being processed currently - head common.Hash // Head is the hash of the last header processed + size uint64 // section size to generate bloombits for + db ethdb.Database // database instance to write index data and metadata into + gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index + section uint64 // Section is the section number being processed currently + head common.Hash // Head is the hash of the last header processed } // NewBloomIndexer returns a chain indexer that generates bloom bits data for the // canonical chain for fast logs filtering. -func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer { +func NewBloomIndexer(db ethdb.Database, size, confReq uint64) *core.ChainIndexer { backend := &BloomIndexer{ db: db, size: size, } table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) - return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits") + return core.NewChainIndexer(db, table, backend, size, confReq, bloomThrottling, "bloombits") } // Reset implements core.ChainIndexerBackend, starting a new bloombits index // section. -func (b *BloomIndexer) Reset(section uint64, lastSectionHead common.Hash) error { +func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { gen, err := bloombits.NewGenerator(uint(b.size)) b.gen, b.section, b.head = gen, section, common.Hash{} return err @@ -123,16 +122,16 @@ func (b *BloomIndexer) Reset(section uint64, lastSectionHead common.Hash) error // Process implements core.ChainIndexerBackend, adding a new header's bloom into // the index. -func (b *BloomIndexer) Process(header *types.Header) { +func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error { b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) b.head = header.Hash() + return nil } // Commit implements core.ChainIndexerBackend, finalizing the bloom section and // writing it out into the database. func (b *BloomIndexer) Commit() error { batch := b.db.NewBatch() - for i := 0; i < types.BloomBitLength; i++ { bits, err := b.gen.Bitset(uint(i)) if err != nil { diff --git a/les/backend.go b/les/backend.go index 178bc1e0e4..9b8cc1828f 100644 --- a/les/backend.go +++ b/les/backend.go @@ -95,29 +95,35 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { quitSync := make(chan struct{}) leth := &LightEthereum{ - config: config, - chainConfig: chainConfig, - chainDb: chainDb, - eventMux: ctx.EventMux, - peers: peers, - reqDist: newRequestDistributor(peers, quitSync), - accountManager: ctx.AccountManager, - engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), - shutdownChan: make(chan bool), - networkId: config.NetworkId, - bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: eth.NewBloomIndexer(chainDb, light.BloomTrieFrequency), - chtIndexer: light.NewChtIndexer(chainDb, true), - bloomTrieIndexer: light.NewBloomTrieIndexer(chainDb, true), + config: config, + chainConfig: chainConfig, + chainDb: chainDb, + eventMux: ctx.EventMux, + peers: peers, + reqDist: newRequestDistributor(peers, quitSync), + accountManager: ctx.AccountManager, + engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), + shutdownChan: make(chan bool), + networkId: config.NetworkId, + bloomRequests: make(chan chan *bloombits.Retrieval), + bloomIndexer: eth.NewBloomIndexer(chainDb, light.BloomTrieFrequency, light.HelperTrieConfirmations), } leth.relay = NewLesTxRelay(peers, leth.reqDist) leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) - leth.odr = NewLesOdr(chainDb, leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer, leth.retriever) + leth.odr = NewLesOdr(chainDb, leth.retriever) + leth.chtIndexer = light.NewChtIndexer(chainDb, true, leth.odr) + leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, true, leth.odr) + leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) + // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with + // indexers already set but not started yet if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine); err != nil { return nil, err } + // Note: AddChildIndexer starts the update process for the child + leth.bloomIndexer.AddChildIndexer(leth.bloomTrieIndexer) + leth.chtIndexer.Start(leth.blockchain) leth.bloomIndexer.Start(leth.blockchain) // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { @@ -242,9 +248,6 @@ func (s *LightEthereum) Stop() error { if s.chtIndexer != nil { s.chtIndexer.Close() } - if s.bloomTrieIndexer != nil { - s.bloomTrieIndexer.Close() - } s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() diff --git a/les/distributor.go b/les/distributor.go index 159fa4c73f..d3f6b21d18 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -20,14 +20,10 @@ package les import ( "container/list" - "errors" "sync" "time" ) -// ErrNoPeers is returned if no peers capable of serving a queued request are available -var ErrNoPeers = errors.New("no suitable peers available") - // requestDistributor implements a mechanism that distributes requests to // suitable peers, obeying flow control rules and prioritizing them in creation // order (even when a resend is necessary). diff --git a/les/handler.go b/les/handler.go index 91a235bf0a..ccb4a88448 100644 --- a/les/handler.go +++ b/les/handler.go @@ -1206,11 +1206,12 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // NodeInfo represents a short summary of the Ethereum sub-protocol metadata // known about the host peer. type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) - Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain - Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block - Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules - Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block + Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain + Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block + Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules + Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block + CHT light.TrustedCheckpoint `json:"cht"` // Trused CHT checkpoint for fast catchup } // NodeInfo retrieves some protocol metadata about the running host node. @@ -1218,12 +1219,31 @@ func (self *ProtocolManager) NodeInfo() *NodeInfo { head := self.blockchain.CurrentHeader() hash := head.Hash() + var cht light.TrustedCheckpoint + + sections, _, sectionHead := self.odr.ChtIndexer().Sections() + sections2, _, sectionHead2 := self.odr.BloomTrieIndexer().Sections() + if sections2 < sections { + sections = sections2 + sectionHead = sectionHead2 + } + if sections > 0 { + sectionIndex := sections - 1 + cht = light.TrustedCheckpoint{ + SectionIdx: sectionIndex, + SectionHead: sectionHead, + CHTRoot: light.GetChtRoot(self.chainDb, sectionIndex, sectionHead), + BloomRoot: light.GetBloomTrieRoot(self.chainDb, sectionIndex, sectionHead), + } + } + return &NodeInfo{ Network: self.networkId, Difficulty: self.blockchain.GetTd(hash, head.Number.Uint64()), Genesis: self.blockchain.Genesis().Hash(), Config: self.blockchain.Config(), Head: hash, + CHT: cht, } } @@ -1258,7 +1278,7 @@ func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, s } _, ok := <-pc.manager.reqDist.queue(rq) if !ok { - return ErrNoPeers + return light.ErrNoPeers } return nil } @@ -1282,7 +1302,7 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip } _, ok := <-pc.manager.reqDist.queue(rq) if !ok { - return ErrNoPeers + return light.ErrNoPeers } return nil } diff --git a/les/helper_test.go b/les/helper_test.go index 8fd01a39e0..50c97e06e1 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -156,12 +156,12 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor } else { blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) - chtIndexer := light.NewChtIndexer(db, false) + chtIndexer := light.NewChtIndexer(db, false, nil) chtIndexer.Start(blockchain) - bbtIndexer := light.NewBloomTrieIndexer(db, false) + bbtIndexer := light.NewBloomTrieIndexer(db, false, nil) - bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks) + bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks, light.HelperTrieProcessConfirmations) bloomIndexer.AddChildIndexer(bbtIndexer) bloomIndexer.Start(blockchain) diff --git a/les/odr.go b/les/odr.go index f8412aaad7..2ad28d5d9f 100644 --- a/les/odr.go +++ b/les/odr.go @@ -33,14 +33,11 @@ type LesOdr struct { stop chan struct{} } -func NewLesOdr(db ethdb.Database, chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer, retriever *retrieveManager) *LesOdr { +func NewLesOdr(db ethdb.Database, retriever *retrieveManager) *LesOdr { return &LesOdr{ - db: db, - chtIndexer: chtIndexer, - bloomTrieIndexer: bloomTrieIndexer, - bloomIndexer: bloomIndexer, - retriever: retriever, - stop: make(chan struct{}), + db: db, + retriever: retriever, + stop: make(chan struct{}), } } @@ -54,6 +51,13 @@ func (odr *LesOdr) Database() ethdb.Database { return odr.db } +// SetIndexers adds the necessary chain indexers to the ODR backend +func (odr *LesOdr) SetIndexers(chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer) { + odr.chtIndexer = chtIndexer + odr.bloomTrieIndexer = bloomTrieIndexer + odr.bloomIndexer = bloomIndexer +} + // ChtIndexer returns the CHT chain indexer func (odr *LesOdr) ChtIndexer() *core.ChainIndexer { return odr.chtIndexer diff --git a/les/odr_test.go b/les/odr_test.go index 983f7262b0..c7c25cbe4b 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -167,7 +167,8 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { rm := newRetrieveManager(peers, dist, nil) db := ethdb.NewMemDatabase() ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) + odr := NewLesOdr(ldb, rm) + odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) diff --git a/les/request_test.go b/les/request_test.go index ba2f603d8b..db576798b4 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -89,7 +89,8 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { rm := newRetrieveManager(peers, dist, nil) db := ethdb.NewMemDatabase() ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) + odr := NewLesOdr(ldb, rm) + odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) diff --git a/les/retrieve.go b/les/retrieve.go index a9037a38ef..8ae36d82cd 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/light" ) var ( @@ -207,7 +208,7 @@ func (r *sentReq) stateRequesting() reqStateFn { return r.stateNoMorePeers } // nothing to wait for, no more peers to ask, return with error - r.stop(ErrNoPeers) + r.stop(light.ErrNoPeers) // no need to go to stopped state because waiting() already returned false return nil } diff --git a/les/server.go b/les/server.go index fca6124c9c..a934fbf26e 100644 --- a/les/server.go +++ b/les/server.go @@ -67,8 +67,8 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { protocolManager: pm, quitSync: quitSync, lesTopics: lesTopics, - chtIndexer: light.NewChtIndexer(eth.ChainDb(), false), - bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false), + chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), + bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), } logger := log.New() diff --git a/light/lightchain.go b/light/lightchain.go index 30b9bd89a6..b7e629e88b 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -116,19 +116,19 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. } // addTrustedCheckpoint adds a trusted checkpoint to the blockchain -func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) { +func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) { if self.odr.ChtIndexer() != nil { - StoreChtRoot(self.chainDb, cp.sectionIdx, cp.sectionHead, cp.chtRoot) - self.odr.ChtIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) + StoreChtRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.CHTRoot) + self.odr.ChtIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomTrieIndexer() != nil { - StoreBloomTrieRoot(self.chainDb, cp.sectionIdx, cp.sectionHead, cp.bloomTrieRoot) - self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) + StoreBloomTrieRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.BloomRoot) + self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomIndexer() != nil { - self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) + self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } - log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*CHTFrequencyClient-1, "hash", cp.sectionHead) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*CHTFrequencyClient-1, "hash", cp.SectionHead) } func (self *LightChain) getProcInterrupt() bool { diff --git a/light/odr.go b/light/odr.go index 8f1e50b817..83c64055a2 100644 --- a/light/odr.go +++ b/light/odr.go @@ -20,6 +20,7 @@ package light import ( "context" + "errors" "math/big" "github.com/ethereum/go-ethereum/common" @@ -33,6 +34,9 @@ import ( // service is not required. var NoOdr = context.Background() +// ErrNoPeers is returned if no peers capable of serving a queued request are available +var ErrNoPeers = errors.New("no suitable peers available") + // OdrBackend is an interface to a backend service that handles ODR retrievals type type OdrBackend interface { Database() ethdb.Database diff --git a/light/postprocess.go b/light/postprocess.go index 2090a9d044..0b25e1d881 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -17,8 +17,10 @@ package light import ( + "context" "encoding/binary" "errors" + "fmt" "math/big" "time" @@ -47,35 +49,35 @@ const ( HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated ) -// trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with +// TrustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with // the appropriate section index and head hash. It is used to start light syncing from this checkpoint // and avoid downloading the entire header chain while still being able to securely access old headers/logs. -type trustedCheckpoint struct { - name string - sectionIdx uint64 - sectionHead, chtRoot, bloomTrieRoot common.Hash +type TrustedCheckpoint struct { + name string + SectionIdx uint64 + SectionHead, CHTRoot, BloomRoot common.Hash } var ( - mainnetCheckpoint = trustedCheckpoint{ - name: "mainnet", - sectionIdx: 179, - sectionHead: common.HexToHash("ae778e455492db1183e566fa0c67f954d256fdd08618f6d5a393b0e24576d0ea"), - chtRoot: common.HexToHash("646b338f9ca74d936225338916be53710ec84020b89946004a8605f04c817f16"), - bloomTrieRoot: common.HexToHash("d0f978f5dbc86e5bf931d8dd5b2ecbebbda6dc78f8896af6a27b46a3ced0ac25"), + mainnetCheckpoint = TrustedCheckpoint{ + name: "mainnet", + SectionIdx: 179, + SectionHead: common.HexToHash("ae778e455492db1183e566fa0c67f954d256fdd08618f6d5a393b0e24576d0ea"), + CHTRoot: common.HexToHash("646b338f9ca74d936225338916be53710ec84020b89946004a8605f04c817f16"), + BloomRoot: common.HexToHash("d0f978f5dbc86e5bf931d8dd5b2ecbebbda6dc78f8896af6a27b46a3ced0ac25"), } - ropstenCheckpoint = trustedCheckpoint{ - name: "ropsten", - sectionIdx: 107, - sectionHead: common.HexToHash("e1988f95399debf45b873e065e5cd61b416ef2e2e5deec5a6f87c3127086e1ce"), - chtRoot: common.HexToHash("15cba18e4de0ab1e95e202625199ba30147aec8b0b70384b66ebea31ba6a18e0"), - bloomTrieRoot: common.HexToHash("e00fa6389b2e597d9df52172cd8e936879eed0fca4fa59db99e2c8ed682562f2"), + ropstenCheckpoint = TrustedCheckpoint{ + name: "ropsten", + SectionIdx: 107, + SectionHead: common.HexToHash("e1988f95399debf45b873e065e5cd61b416ef2e2e5deec5a6f87c3127086e1ce"), + CHTRoot: common.HexToHash("15cba18e4de0ab1e95e202625199ba30147aec8b0b70384b66ebea31ba6a18e0"), + BloomRoot: common.HexToHash("e00fa6389b2e597d9df52172cd8e936879eed0fca4fa59db99e2c8ed682562f2"), } ) // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to -var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ +var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ params.MainnetGenesisHash: mainnetCheckpoint, params.TestnetGenesisHash: ropstenCheckpoint, } @@ -119,7 +121,8 @@ func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common // ChtIndexerBackend implements core.ChainIndexerBackend type ChtIndexerBackend struct { - diskdb ethdb.Database + diskdb, trieTable ethdb.Database + odr OdrBackend triedb *trie.Database section, sectionSize uint64 lastHash common.Hash @@ -127,7 +130,7 @@ type ChtIndexerBackend struct { } // NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { +func NewChtIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { var sectionSize, confirmReq uint64 if clientMode { sectionSize = CHTFrequencyClient @@ -137,28 +140,64 @@ func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { confirmReq = HelperTrieProcessConfirmations } idb := ethdb.NewTable(db, "chtIndex-") + trieTable := ethdb.NewTable(db, ChtTablePrefix) backend := &ChtIndexerBackend{ diskdb: db, - triedb: trie.NewDatabase(ethdb.NewTable(db, ChtTablePrefix)), + odr: odr, + trieTable: trieTable, + triedb: trie.NewDatabase(trieTable), sectionSize: sectionSize, } return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") } +// fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the +// ODR backend in order to be able to add new entries and calculate subsequent root hashes +func (c *ChtIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { + batch := c.trieTable.NewBatch() + r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1} + for { + err := c.odr.Retrieve(ctx, r) + switch err { + case nil: + r.Proof.Store(batch) + return batch.Write() + case ErrNoPeers: + // if there are no peers to serve, retry later + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second * 10): + // stay in the loop and try again + } + default: + return err + } + } +} + // Reset implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { +func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { var root common.Hash if section > 0 { root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error c.trie, err = trie.New(root, c.triedb) + + if err != nil && c.odr != nil { + err = c.fetchMissingNodes(ctx, section, root) + if err == nil { + c.trie, err = trie.New(root, c.triedb) + } + } + c.section = section return err } // Process implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Process(header *types.Header) { +func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) error { hash, num := header.Hash(), header.Number.Uint64() c.lastHash = hash @@ -170,6 +209,7 @@ func (c *ChtIndexerBackend) Process(header *types.Header) { binary.BigEndian.PutUint64(encNumber[:], num) data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) c.trie.Update(encNumber[:], data) + return nil } // Commit implements core.ChainIndexerBackend @@ -181,16 +221,15 @@ func (c *ChtIndexerBackend) Commit() error { c.triedb.Commit(root, false) if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { - log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) + log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil } const ( - BloomTrieFrequency = 32768 - ethBloomBitsSection = 4096 - ethBloomBitsConfirmations = 256 + BloomTrieFrequency = 32768 + ethBloomBitsSection = 4096 ) var ( @@ -215,7 +254,8 @@ func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root // BloomTrieIndexerBackend implements core.ChainIndexerBackend type BloomTrieIndexerBackend struct { - diskdb ethdb.Database + diskdb, trieTable ethdb.Database + odr OdrBackend triedb *trie.Database section, parentSectionSize, bloomTrieRatio uint64 trie *trie.Trie @@ -223,44 +263,98 @@ type BloomTrieIndexerBackend struct { } // NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewBloomTrieIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { +func NewBloomTrieIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { + trieTable := ethdb.NewTable(db, BloomTrieTablePrefix) backend := &BloomTrieIndexerBackend{ - diskdb: db, - triedb: trie.NewDatabase(ethdb.NewTable(db, BloomTrieTablePrefix)), + diskdb: db, + odr: odr, + trieTable: trieTable, + triedb: trie.NewDatabase(trieTable), } idb := ethdb.NewTable(db, "bltIndex-") - var confirmReq uint64 if clientMode { backend.parentSectionSize = BloomTrieFrequency - confirmReq = HelperTrieConfirmations } else { backend.parentSectionSize = ethBloomBitsSection - confirmReq = HelperTrieProcessConfirmations } backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) - return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") + return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, 0, time.Millisecond*100, "bloomtrie") +} + +// fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the +// ODR backend in order to be able to add new entries and calculate subsequent root hashes +func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { + indexCh := make(chan uint, types.BloomBitLength) + type res struct { + nodes *NodeSet + err error + } + resCh := make(chan res, types.BloomBitLength) + for i := 0; i < 20; i++ { + go func() { + for bitIndex := range indexCh { + r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}} + for { + if err := b.odr.Retrieve(ctx, r); err == ErrNoPeers { + // if there are no peers to serve, retry later + select { + case <-ctx.Done(): + resCh <- res{nil, ctx.Err()} + return + case <-time.After(time.Second * 10): + // stay in the loop and try again + } + } else { + resCh <- res{r.Proofs, err} + break + } + } + } + }() + } + + for i := uint(0); i < types.BloomBitLength; i++ { + indexCh <- i + } + close(indexCh) + batch := b.trieTable.NewBatch() + for i := uint(0); i < types.BloomBitLength; i++ { + res := <-resCh + if res.err != nil { + return res.err + } + res.nodes.Store(batch) + } + return batch.Write() } // Reset implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { +func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { var root common.Hash if section > 0 { root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error b.trie, err = trie.New(root, b.triedb) + if err != nil && b.odr != nil { + err = b.fetchMissingNodes(ctx, section, root) + if err == nil { + b.trie, err = trie.New(root, b.triedb) + } + } b.section = section return err } // Process implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Process(header *types.Header) { +func (b *BloomTrieIndexerBackend) Process(ctx context.Context, header *types.Header) error { num := header.Number.Uint64() - b.section*BloomTrieFrequency if (num+1)%b.parentSectionSize == 0 { b.sectionHeads[num/b.parentSectionSize] = header.Hash() } + return nil } // Commit implements core.ChainIndexerBackend @@ -300,7 +394,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.triedb.Commit(root, false) sectionHead := b.sectionHeads[b.bloomTrieRatio-1] - log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) + log.Info("Storing bloom trie", "section", b.section, "head", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression", float64(compSize)/float64(decompSize)) StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) return nil From b24fb76a3ae8b2a41bde9ed0744b4284e385e011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 16 Aug 2018 09:41:16 +0300 Subject: [PATCH 027/126] cmd/puppeth: fix nil panic on disconnected stats gathering --- cmd/puppeth/wizard_netstats.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/puppeth/wizard_netstats.go b/cmd/puppeth/wizard_netstats.go index 89b38e262b..99ca11bb17 100644 --- a/cmd/puppeth/wizard_netstats.go +++ b/cmd/puppeth/wizard_netstats.go @@ -82,7 +82,6 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s logger.Info("Starting remote server health-check") stat := &serverStat{ - address: client.address, services: make(map[string]map[string]string), } if client == nil { @@ -94,6 +93,8 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s } client = conn } + stat.address = client.address + // Client connected one way or another, run health-checks logger.Debug("Checking for nginx availability") if infos, err := checkNginx(client, w.network); err != nil { @@ -214,6 +215,9 @@ func (stats serverStats) render() { if len(stat.address) > len(separator[1]) { separator[1] = strings.Repeat("-", len(stat.address)) } + if len(stat.failure) > len(separator[1]) { + separator[1] = strings.Repeat("-", len(stat.failure)) + } for service, configs := range stat.services { if len(service) > len(separator[2]) { separator[2] = strings.Repeat("-", len(service)) @@ -250,7 +254,11 @@ func (stats serverStats) render() { sort.Strings(services) if len(services) == 0 { - table.Append([]string{server, stats[server].address, "", "", ""}) + if stats[server].failure != "" { + table.Append([]string{server, stats[server].failure, "", "", ""}) + } else { + table.Append([]string{server, stats[server].address, "", "", ""}) + } } for j, service := range services { // Add an empty line between all services From 3e21adc6488be41ac882c316486573374785cc82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 9 Aug 2018 13:46:52 +0300 Subject: [PATCH 028/126] crypto/bn256: fix issues caused by Go 1.11 --- crypto/bn256/cloudflare/gfp_amd64.s | 2 +- crypto/bn256/cloudflare/gfp_decl.go | 7 +++ crypto/bn256/google/bn256.go | 40 ++++++++++------ crypto/bn256/google/curve.go | 10 +++- crypto/bn256/google/twist.go | 10 +++- vendor/golang.org/x/sys/cpu/cpu.go | 38 +++++++++++++++ vendor/golang.org/x/sys/cpu/cpu_arm.go | 7 +++ vendor/golang.org/x/sys/cpu/cpu_arm64.go | 7 +++ vendor/golang.org/x/sys/cpu/cpu_gc_x86.go | 16 +++++++ vendor/golang.org/x/sys/cpu/cpu_gccgo.c | 43 +++++++++++++++++ vendor/golang.org/x/sys/cpu/cpu_gccgo.go | 26 ++++++++++ vendor/golang.org/x/sys/cpu/cpu_mips64x.go | 9 ++++ vendor/golang.org/x/sys/cpu/cpu_mipsx.go | 9 ++++ vendor/golang.org/x/sys/cpu/cpu_ppc64x.go | 9 ++++ vendor/golang.org/x/sys/cpu/cpu_s390x.go | 7 +++ vendor/golang.org/x/sys/cpu/cpu_x86.go | 55 ++++++++++++++++++++++ vendor/golang.org/x/sys/cpu/cpu_x86.s | 27 +++++++++++ vendor/vendor.json | 6 +++ 18 files changed, 311 insertions(+), 17 deletions(-) create mode 100644 vendor/golang.org/x/sys/cpu/cpu.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_arm.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_arm64.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gc_x86.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gccgo.c create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gccgo.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_mips64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_mipsx.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_ppc64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_s390x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_x86.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_x86.s diff --git a/crypto/bn256/cloudflare/gfp_amd64.s b/crypto/bn256/cloudflare/gfp_amd64.s index 3a785d200b..bdb4ffb787 100644 --- a/crypto/bn256/cloudflare/gfp_amd64.s +++ b/crypto/bn256/cloudflare/gfp_amd64.s @@ -110,7 +110,7 @@ TEXT ·gfpMul(SB),0,$160-24 MOVQ b+16(FP), SI // Jump to a slightly different implementation if MULX isn't supported. - CMPB runtime·support_bmi2(SB), $0 + CMPB ·hasBMI2(SB), $0 JE nobmi2Mul mulBMI2(0(DI),8(DI),16(DI),24(DI), 0(SI)) diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index 6a8a4fddb6..fdea5c11a5 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -5,6 +5,13 @@ package bn256 // This file contains forward declarations for the architecture-specific // assembly implementations of these functions, provided that they exist. +import ( + "golang.org/x/sys/cpu" +) + +//nolint:varcheck +var hasBMI2 = cpu.X86.HasBMI2 + // go:noescape func gfpNeg(c, a *gfP) diff --git a/crypto/bn256/google/bn256.go b/crypto/bn256/google/bn256.go index 5da83e033a..e0402e51f0 100644 --- a/crypto/bn256/google/bn256.go +++ b/crypto/bn256/google/bn256.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package bn256 implements a particular bilinear group at the 128-bit security level. +// Package bn256 implements a particular bilinear group. // // Bilinear groups are the basis of many of the new cryptographic protocols // that have been proposed over the past decade. They consist of a triplet of @@ -14,6 +14,10 @@ // Barreto-Naehrig curve as described in // http://cryptojedi.org/papers/dclxvi-20100714.pdf. Its output is compatible // with the implementation described in that paper. +// +// (This package previously claimed to operate at a 128-bit security level. +// However, recent improvements in attacks mean that is no longer true. See +// https://moderncrypto.org/mail-archive/curves/2016/000740.html.) package bn256 import ( @@ -50,8 +54,8 @@ func RandomG1(r io.Reader) (*big.Int, *G1, error) { return k, new(G1).ScalarBaseMult(k), nil } -func (g *G1) String() string { - return "bn256.G1" + g.p.String() +func (e *G1) String() string { + return "bn256.G1" + e.p.String() } // CurvePoints returns p's curve points in big integer @@ -98,15 +102,19 @@ func (e *G1) Neg(a *G1) *G1 { } // Marshal converts n to a byte slice. -func (n *G1) Marshal() []byte { - n.p.MakeAffine(nil) - - xBytes := new(big.Int).Mod(n.p.x, P).Bytes() - yBytes := new(big.Int).Mod(n.p.y, P).Bytes() - +func (e *G1) Marshal() []byte { // Each value is a 256-bit number. const numBytes = 256 / 8 + if e.p.IsInfinity() { + return make([]byte, numBytes*2) + } + + e.p.MakeAffine(nil) + + xBytes := new(big.Int).Mod(e.p.x, P).Bytes() + yBytes := new(big.Int).Mod(e.p.y, P).Bytes() + ret := make([]byte, numBytes*2) copy(ret[1*numBytes-len(xBytes):], xBytes) copy(ret[2*numBytes-len(yBytes):], yBytes) @@ -175,8 +183,8 @@ func RandomG2(r io.Reader) (*big.Int, *G2, error) { return k, new(G2).ScalarBaseMult(k), nil } -func (g *G2) String() string { - return "bn256.G2" + g.p.String() +func (e *G2) String() string { + return "bn256.G2" + e.p.String() } // CurvePoints returns the curve points of p which includes the real @@ -216,6 +224,13 @@ func (e *G2) Add(a, b *G2) *G2 { // Marshal converts n into a byte slice. func (n *G2) Marshal() []byte { + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + if n.p.IsInfinity() { + return make([]byte, numBytes*4) + } + n.p.MakeAffine(nil) xxBytes := new(big.Int).Mod(n.p.x.x, P).Bytes() @@ -223,9 +238,6 @@ func (n *G2) Marshal() []byte { yxBytes := new(big.Int).Mod(n.p.y.x, P).Bytes() yyBytes := new(big.Int).Mod(n.p.y.y, P).Bytes() - // Each value is a 256-bit number. - const numBytes = 256 / 8 - ret := make([]byte, numBytes*4) copy(ret[1*numBytes-len(xxBytes):], xxBytes) copy(ret[2*numBytes-len(xyBytes):], xyBytes) diff --git a/crypto/bn256/google/curve.go b/crypto/bn256/google/curve.go index 3e679fdc7e..819cb81da7 100644 --- a/crypto/bn256/google/curve.go +++ b/crypto/bn256/google/curve.go @@ -245,11 +245,19 @@ func (c *curvePoint) Mul(a *curvePoint, scalar *big.Int, pool *bnPool) *curvePoi return c } +// MakeAffine converts c to affine form and returns c. If c is ∞, then it sets +// c to 0 : 1 : 0. func (c *curvePoint) MakeAffine(pool *bnPool) *curvePoint { if words := c.z.Bits(); len(words) == 1 && words[0] == 1 { return c } - + if c.IsInfinity() { + c.x.SetInt64(0) + c.y.SetInt64(1) + c.z.SetInt64(0) + c.t.SetInt64(0) + return c + } zInv := pool.Get().ModInverse(c.z, P) t := pool.Get().Mul(c.y, zInv) t.Mod(t, P) diff --git a/crypto/bn256/google/twist.go b/crypto/bn256/google/twist.go index 1f5a4d9deb..43364ff5b7 100644 --- a/crypto/bn256/google/twist.go +++ b/crypto/bn256/google/twist.go @@ -225,11 +225,19 @@ func (c *twistPoint) Mul(a *twistPoint, scalar *big.Int, pool *bnPool) *twistPoi return c } +// MakeAffine converts c to affine form and returns c. If c is ∞, then it sets +// c to 0 : 1 : 0. func (c *twistPoint) MakeAffine(pool *bnPool) *twistPoint { if c.z.IsOne() { return c } - + if c.IsInfinity() { + c.x.SetZero() + c.y.SetOne() + c.z.SetZero() + c.t.SetZero() + return c + } zInv := newGFp2(pool).Invert(c.z, pool) t := newGFp2(pool).Mul(c.y, zInv, pool) zInv2 := newGFp2(pool).Square(zInv, pool) diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go new file mode 100644 index 0000000000..3d88f86673 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu.go @@ -0,0 +1,38 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection for +// various CPU architectures. +package cpu + +// CacheLinePad is used to pad structs to avoid false sharing. +type CacheLinePad struct{ _ [cacheLineSize]byte } + +// X86 contains the supported CPU features of the +// current X86/AMD64 platform. If the current platform +// is not X86/AMD64 then all feature flags are false. +// +// X86 is padded to avoid false sharing. Further the HasAVX +// and HasAVX2 are only set if the OS supports XMM and YMM +// registers in addition to the CPUID feature bit being set. +var X86 struct { + _ CacheLinePad + HasAES bool // AES hardware implementation (AES NI) + HasADX bool // Multi-precision add-carry instruction extensions + HasAVX bool // Advanced vector extension + HasAVX2 bool // Advanced vector extension 2 + HasBMI1 bool // Bit manipulation instruction set 1 + HasBMI2 bool // Bit manipulation instruction set 2 + HasERMS bool // Enhanced REP for MOVSB and STOSB + HasFMA bool // Fused-multiply-add instructions + HasOSXSAVE bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers. + HasPCLMULQDQ bool // PCLMULQDQ instruction - most often used for AES-GCM + HasPOPCNT bool // Hamming weight instruction POPCNT. + HasSSE2 bool // Streaming SIMD extension 2 (always available on amd64) + HasSSE3 bool // Streaming SIMD extension 3 + HasSSSE3 bool // Supplemental streaming SIMD extension 3 + HasSSE41 bool // Streaming SIMD extension 4 and 4.1 + HasSSE42 bool // Streaming SIMD extension 4 and 4.2 + _ CacheLinePad +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm.go b/vendor/golang.org/x/sys/cpu/cpu_arm.go new file mode 100644 index 0000000000..d93036f752 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_arm64.go new file mode 100644 index 0000000000..1d2ab2902a --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 64 diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go new file mode 100644 index 0000000000..f7cb46971c --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build !gccgo + +package cpu + +// cpuid is implemented in cpu_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func xgetbv() (eax, edx uint32) diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo.c b/vendor/golang.org/x/sys/cpu/cpu_gccgo.c new file mode 100644 index 0000000000..e363c7d131 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo.c @@ -0,0 +1,43 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build gccgo + +#include +#include + +// Need to wrap __get_cpuid_count because it's declared as static. +int +gccgoGetCpuidCount(uint32_t leaf, uint32_t subleaf, + uint32_t *eax, uint32_t *ebx, + uint32_t *ecx, uint32_t *edx) +{ + return __get_cpuid_count(leaf, subleaf, eax, ebx, ecx, edx); +} + +// xgetbv reads the contents of an XCR (Extended Control Register) +// specified in the ECX register into registers EDX:EAX. +// Currently, the only supported value for XCR is 0. +// +// TODO: Replace with a better alternative: +// +// #include +// +// #pragma GCC target("xsave") +// +// void gccgoXgetbv(uint32_t *eax, uint32_t *edx) { +// unsigned long long x = _xgetbv(0); +// *eax = x & 0xffffffff; +// *edx = (x >> 32) & 0xffffffff; +// } +// +// Note that _xgetbv is defined starting with GCC 8. +void +gccgoXgetbv(uint32_t *eax, uint32_t *edx) +{ + __asm(" xorl %%ecx, %%ecx\n" + " xgetbv" + : "=a"(*eax), "=d"(*edx)); +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo.go new file mode 100644 index 0000000000..ba49b91bd3 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo.go @@ -0,0 +1,26 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build gccgo + +package cpu + +//extern gccgoGetCpuidCount +func gccgoGetCpuidCount(eaxArg, ecxArg uint32, eax, ebx, ecx, edx *uint32) + +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) { + var a, b, c, d uint32 + gccgoGetCpuidCount(eaxArg, ecxArg, &a, &b, &c, &d) + return a, b, c, d +} + +//extern gccgoXgetbv +func gccgoXgetbv(eax, edx *uint32) + +func xgetbv() (eax, edx uint32) { + var a, d uint32 + gccgoXgetbv(&a, &d) + return a, d +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go new file mode 100644 index 0000000000..6165f12124 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips64 mips64le + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_mipsx.go b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go new file mode 100644 index 0000000000..1269eee88d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go new file mode 100644 index 0000000000..d10759a524 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package cpu + +const cacheLineSize = 128 diff --git a/vendor/golang.org/x/sys/cpu/cpu_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_s390x.go new file mode 100644 index 0000000000..684c4f005d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_s390x.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 256 diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.go b/vendor/golang.org/x/sys/cpu/cpu_x86.go new file mode 100644 index 0000000000..71e288b062 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -0,0 +1,55 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 + +package cpu + +const cacheLineSize = 64 + +func init() { + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + _, _, ecx1, edx1 := cpuid(1, 0) + X86.HasSSE2 = isSet(26, edx1) + + X86.HasSSE3 = isSet(0, ecx1) + X86.HasPCLMULQDQ = isSet(1, ecx1) + X86.HasSSSE3 = isSet(9, ecx1) + X86.HasFMA = isSet(12, ecx1) + X86.HasSSE41 = isSet(19, ecx1) + X86.HasSSE42 = isSet(20, ecx1) + X86.HasPOPCNT = isSet(23, ecx1) + X86.HasAES = isSet(25, ecx1) + X86.HasOSXSAVE = isSet(27, ecx1) + + osSupportsAVX := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(1, eax) && isSet(2, eax) + } + + X86.HasAVX = isSet(28, ecx1) && osSupportsAVX + + if maxID < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + X86.HasBMI1 = isSet(3, ebx7) + X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX + X86.HasBMI2 = isSet(8, ebx7) + X86.HasERMS = isSet(9, ebx7) + X86.HasADX = isSet(19, ebx7) +} + +func isSet(bitpos uint, value uint32) bool { + return value&(1< Date: Thu, 16 Aug 2018 19:14:33 +0800 Subject: [PATCH 029/126] miner: regenerate mining work every 3 seconds (#17413) * miner: regenerate mining work every 3 seconds * miner: polish --- core/events.go | 3 - miner/worker.go | 332 ++++++++++++++++++++++++++----------------- miner/worker_test.go | 65 +++++++++ 3 files changed, 267 insertions(+), 133 deletions(-) diff --git a/core/events.go b/core/events.go index 8d200f2a29..710bdb5894 100644 --- a/core/events.go +++ b/core/events.go @@ -29,9 +29,6 @@ type PendingLogsEvent struct { Logs []*types.Log } -// PendingStateEvent is posted pre mining and notifies of pending state changes. -type PendingStateEvent struct{} - // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } diff --git a/miner/worker.go b/miner/worker.go index fae480c84c..81a9eabfd5 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -40,19 +40,27 @@ import ( const ( // resultQueueSize is the size of channel listening to sealing result. resultQueueSize = 10 + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 + // chainHeadChanSize is the size of channel listening to ChainHeadEvent. chainHeadChanSize = 10 + // chainSideChanSize is the size of channel listening to ChainSideEvent. chainSideChanSize = 10 - miningLogAtDepth = 5 + + // miningLogAtDepth is the number of confirmations before logging successful mining. + miningLogAtDepth = 5 + + // blockRecommitInterval is the time interval to recreate the mining block with + // any newly arrived transactions. + blockRecommitInterval = 3 * time.Second ) -// Env is the worker's current environment and holds all of the current state information. -type Env struct { - config *params.ChainConfig +// environment is the worker's current environment and holds all of the current state information. +type environment struct { signer types.Signer state *state.StateDB // apply state changes here @@ -67,105 +75,6 @@ type Env struct { receipts []*types.Receipt } -func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { - snap := env.state.Snapshot() - - receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) - if err != nil { - env.state.RevertToSnapshot(snap) - return err, nil - } - env.txs = append(env.txs, tx) - env.receipts = append(env.receipts, receipt) - - return nil, receipt.Logs -} - -func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { - if env.gasPool == nil { - env.gasPool = new(core.GasPool).AddGas(env.header.GasLimit) - } - - var coalescedLogs []*types.Log - - for { - // If we don't have enough gas for any further transactions then we're done - if env.gasPool.Gas() < params.TxGas { - log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) - break - } - // Retrieve the next transaction and abort if all done - tx := txs.Peek() - if tx == nil { - break - } - // Error may be ignored here. The error has already been checked - // during transaction acceptance is the transaction pool. - // - // We use the eip155 signer regardless of the current hf. - from, _ := types.Sender(env.signer, tx) - // Check whether the tx is replay protected. If we're not in the EIP155 hf - // phase, start ignoring the sender until we do. - if tx.Protected() && !env.config.IsEIP155(env.header.Number) { - log.Trace("Ignoring reply protected transaction", "hash", tx.Hash(), "eip155", env.config.EIP155Block) - - txs.Pop() - continue - } - // Start executing the transaction - env.state.Prepare(tx.Hash(), common.Hash{}, env.tcount) - - err, logs := env.commitTransaction(tx, bc, coinbase, env.gasPool) - switch err { - case core.ErrGasLimitReached: - // Pop the current out-of-gas transaction without shifting in the next from the account - log.Trace("Gas limit exceeded for current block", "sender", from) - txs.Pop() - - case core.ErrNonceTooLow: - // New head notification data race between the transaction pool and miner, shift - log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) - txs.Shift() - - case core.ErrNonceTooHigh: - // Reorg notification data race between the transaction pool and miner, skip account = - log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) - txs.Pop() - - case nil: - // Everything ok, collect the logs and shift in the next transaction from the same account - coalescedLogs = append(coalescedLogs, logs...) - env.tcount++ - txs.Shift() - - default: - // Strange error, discard the transaction and get the next in line (note, the - // nonce-too-high clause will prevent us from executing in vain). - log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) - txs.Shift() - } - } - - if len(coalescedLogs) > 0 || env.tcount > 0 { - // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined - // logs by filling in the block hash when the block was mined by the local miner. This can - // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. - cpy := make([]*types.Log, len(coalescedLogs)) - for i, l := range coalescedLogs { - cpy[i] = new(types.Log) - *cpy[i] = *l - } - go func(logs []*types.Log, tcount int) { - if len(logs) > 0 { - mux.Post(core.PendingLogsEvent{Logs: logs}) - } - if tcount > 0 { - mux.Post(core.PendingStateEvent{}) - } - }(cpy, env.tcount) - } -} - // task contains all information for consensus engine sealing and result submitting. type task struct { receipts []*types.Receipt @@ -174,6 +83,17 @@ type task struct { createdAt time.Time } +const ( + commitInterruptNone int32 = iota + commitInterruptNewHead + commitInterruptResubmit +) + +type newWorkReq struct { + interrupt *int32 + noempty bool +} + // worker is the main object which takes care of submitting new work to consensus engine // and gathering the sealing result. type worker struct { @@ -192,12 +112,13 @@ type worker struct { chainSideSub event.Subscription // Channels - newWork chan struct{} - taskCh chan *task - resultCh chan *task - exitCh chan struct{} + newWorkCh chan *newWorkReq + taskCh chan *task + resultCh chan *task + startCh chan struct{} + exitCh chan struct{} - current *Env // An environment for current running cycle. + current *environment // An environment for current running cycle. possibleUncles map[common.Hash]*types.Block // A set of side blocks as the possible uncle blocks. unconfirmed *unconfirmedBlocks // A set of locally mined blocks pending canonicalness confirmations. @@ -230,10 +151,11 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), - newWork: make(chan struct{}, 1), + newWorkCh: make(chan *newWorkReq), taskCh: make(chan *task), resultCh: make(chan *task, resultQueueSize), exitCh: make(chan struct{}), + startCh: make(chan struct{}, 1), } // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) @@ -242,11 +164,13 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) go worker.mainLoop() + go worker.newWorkLoop() go worker.resultLoop() go worker.taskLoop() // Submit first work to initialize pending state. - worker.newWork <- struct{}{} + worker.startCh <- struct{}{} + return worker } @@ -286,7 +210,7 @@ func (w *worker) pendingBlock() *types.Block { // start sets the running status as 1 and triggers new work submitting. func (w *worker) start() { atomic.StoreInt32(&w.running, 1) - w.newWork <- struct{}{} + w.startCh <- struct{}{} } // stop sets the running status as 0. @@ -313,6 +237,44 @@ func (w *worker) close() { } } +// newWorkLoop is a standalone goroutine to submit new mining work upon received events. +func (w *worker) newWorkLoop() { + var interrupt *int32 + + timer := time.NewTimer(0) + <-timer.C // discard the initial tick + + // recommit aborts in-flight transaction execution with given signal and resubmits a new one. + recommit := func(noempty bool, s int32) { + if interrupt != nil { + atomic.StoreInt32(interrupt, s) + } + interrupt = new(int32) + w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty} + timer.Reset(blockRecommitInterval) + } + + for { + select { + case <-w.startCh: + recommit(false, commitInterruptNewHead) + + case <-w.chainHeadCh: + recommit(false, commitInterruptNewHead) + + case <-timer.C: + // If mining is running resubmit a new work cycle periodically to pull in + // higher priced transactions. Disable this overhead for pending blocks. + if w.isRunning() && (w.config.Clique == nil || w.config.Clique.Period > 0) { + recommit(true, commitInterruptResubmit) + } + + case <-w.exitCh: + return + } + } +} + // mainLoop is a standalone goroutine to regenerate the sealing task based on the received event. func (w *worker) mainLoop() { defer w.txsSub.Unsubscribe() @@ -321,13 +283,8 @@ func (w *worker) mainLoop() { for { select { - case <-w.newWork: - // Submit a work when the worker is created or started. - w.commitNewWork() - - case <-w.chainHeadCh: - // Resubmit a work for new cycle once worker receives chain head event. - w.commitNewWork() + case req := <-w.newWorkCh: + w.commitNewWork(req.interrupt, req.noempty) case ev := <-w.chainSideCh: if _, exist := w.possibleUncles[ev.Block.Hash()]; exist { @@ -364,9 +321,9 @@ func (w *worker) mainLoop() { // already included in the current mining block. These transactions will // be automatically eliminated. if !w.isRunning() && w.current != nil { - w.mu.Lock() + w.mu.RLock() coinbase := w.coinbase - w.mu.Unlock() + w.mu.RUnlock() txs := make(map[common.Address]types.Transactions) for _, tx := range ev.Txs { @@ -374,12 +331,12 @@ func (w *worker) mainLoop() { txs[acc] = append(txs[acc], tx) } txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs) - w.current.commitTransactions(w.mux, txset, w.chain, coinbase) + w.commitTransactions(txset, coinbase, nil) w.updateSnapshot() } else { // If we're mining, but nothing is being processed, wake on new transactions if w.config.Clique != nil && w.config.Clique.Period == 0 { - w.commitNewWork() + w.commitNewWork(nil, false) } } @@ -508,8 +465,7 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { if err != nil { return err } - env := &Env{ - config: w.config, + env := &environment{ signer: types.NewEIP155Signer(w.config.ChainID), state: state, ancestors: mapset.NewSet(), @@ -534,7 +490,7 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { } // commitUncle adds the given block to uncle block set, returns error if failed to add. -func (w *worker) commitUncle(env *Env, uncle *types.Header) error { +func (w *worker) commitUncle(env *environment, uncle *types.Header) error { hash := uncle.Hash() if env.uncles.Contains(hash) { return fmt.Errorf("uncle not unique") @@ -579,8 +535,120 @@ func (w *worker) updateSnapshot() { w.snapshotState = w.current.state.Copy() } +func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { + snap := w.current.state.Snapshot() + + receipt, _, err := core.ApplyTransaction(w.config, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, vm.Config{}) + if err != nil { + w.current.state.RevertToSnapshot(snap) + return nil, err + } + w.current.txs = append(w.current.txs, tx) + w.current.receipts = append(w.current.receipts, receipt) + + return receipt.Logs, nil +} + +func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coinbase common.Address, interrupt *int32) bool { + // Short circuit if current is nil + if w.current == nil { + return true + } + + if w.current.gasPool == nil { + w.current.gasPool = new(core.GasPool).AddGas(w.current.header.GasLimit) + } + + var coalescedLogs []*types.Log + + for { + // In the following three cases, we will interrupt the execution of the transaction. + // (1) new head block event arrival, the interrupt signal is 1 + // (2) worker start or restart, the interrupt signal is 1 + // (3) worker recreate the mining block with any newly arrived transactions, the interrupt signal is 2. + // For the first two cases, the semi-finished work will be discarded. + // For the third case, the semi-finished work will be submitted to the consensus engine. + // TODO(rjl493456442) give feedback to newWorkLoop to adjust resubmit interval if it is too short. + if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { + return atomic.LoadInt32(interrupt) == commitInterruptNewHead + } + // If we don't have enough gas for any further transactions then we're done + if w.current.gasPool.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "have", w.current.gasPool, "want", params.TxGas) + break + } + // Retrieve the next transaction and abort if all done + tx := txs.Peek() + if tx == nil { + break + } + // Error may be ignored here. The error has already been checked + // during transaction acceptance is the transaction pool. + // + // We use the eip155 signer regardless of the current hf. + from, _ := types.Sender(w.current.signer, tx) + // Check whether the tx is replay protected. If we're not in the EIP155 hf + // phase, start ignoring the sender until we do. + if tx.Protected() && !w.config.IsEIP155(w.current.header.Number) { + log.Trace("Ignoring reply protected transaction", "hash", tx.Hash(), "eip155", w.config.EIP155Block) + + txs.Pop() + continue + } + // Start executing the transaction + w.current.state.Prepare(tx.Hash(), common.Hash{}, w.current.tcount) + + logs, err := w.commitTransaction(tx, coinbase) + switch err { + case core.ErrGasLimitReached: + // Pop the current out-of-gas transaction without shifting in the next from the account + log.Trace("Gas limit exceeded for current block", "sender", from) + txs.Pop() + + case core.ErrNonceTooLow: + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + txs.Shift() + + case core.ErrNonceTooHigh: + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) + txs.Pop() + + case nil: + // Everything ok, collect the logs and shift in the next transaction from the same account + coalescedLogs = append(coalescedLogs, logs...) + w.current.tcount++ + txs.Shift() + + default: + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + txs.Shift() + } + } + + if !w.isRunning() && len(coalescedLogs) > 0 { + // We don't push the pendingLogsEvent while we are mining. The reason is that + // when we are mining, the worker will regenerate a mining block every 3 seconds. + // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. + + // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined + // logs by filling in the block hash when the block was mined by the local miner. This can + // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. + cpy := make([]*types.Log, len(coalescedLogs)) + for i, l := range coalescedLogs { + cpy[i] = new(types.Log) + *cpy[i] = *l + } + go w.mux.Post(core.PendingLogsEvent{Logs: cpy}) + } + return false +} + // commitNewWork generates several new sealing tasks based on the parent block. -func (w *worker) commitNewWork() { +func (w *worker) commitNewWork(interrupt *int32, noempty bool) { w.mu.RLock() defer w.mu.RUnlock() @@ -666,9 +734,11 @@ func (w *worker) commitNewWork() { delete(w.possibleUncles, hash) } - // Create an empty block based on temporary copied state for sealing in advance without waiting block - // execution finished. - w.commit(uncles, nil, false, tstart) + if !noempty { + // Create an empty block based on temporary copied state for sealing in advance without waiting block + // execution finished. + w.commit(uncles, nil, false, tstart) + } // Fill the block with all available pending transactions. pending, err := w.eth.TxPool().Pending() @@ -682,7 +752,9 @@ func (w *worker) commitNewWork() { return } txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) - env.commitTransactions(w.mux, txs, w.chain, w.coinbase) + if w.commitTransactions(txs, w.coinbase, interrupt) { + return + } w.commit(uncles, w.fullTaskHook, true, tstart) } diff --git a/miner/worker_test.go b/miner/worker_test.go index 408c47e3b3..34bb7f5f33 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -270,3 +270,68 @@ func TestStreamUncleBlock(t *testing.T) { t.Error("new task timeout") } } + +func TestRegenerateMiningBlockEthash(t *testing.T) { + testRegenerateMiningBlock(t, ethashChainConfig, ethash.NewFaker()) +} + +func TestRegenerateMiningBlockClique(t *testing.T) { + testRegenerateMiningBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, b := newTestWorker(t, chainConfig, engine) + defer w.close() + + var taskCh = make(chan struct{}) + + taskIndex := 0 + w.newTaskHook = func(task *task) { + if task.block.NumberU64() == 1 { + if taskIndex == 2 { + receiptLen, balance := 2, big.NewInt(2000) + if len(task.receipts) != receiptLen { + t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + } + if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { + t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + } + } + taskCh <- struct{}{} + taskIndex += 1 + } + } + w.skipSealHook = func(task *task) bool { + return true + } + w.fullTaskHook = func() { + time.Sleep(100 * time.Millisecond) + } + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + // Ignore the first two works + for i := 0; i < 2; i += 1 { + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } + } + b.txPool.AddLocals(newTxs) + time.Sleep(3 * time.Second) + + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } +} From 62f5137a72c69ae8cffbc3526611af77448d25de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 16 Aug 2018 14:47:49 +0300 Subject: [PATCH 030/126] miner: add gas and fee details to mining logs --- miner/worker.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 81a9eabfd5..e7e2796458 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -780,8 +780,16 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st select { case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now()}: w.unconfirmed.Shift(block.NumberU64() - 1) - log.Info("Commit new mining work", "number", block.Number(), "txs", w.current.tcount, "uncles", len(uncles), - "elapsed", common.PrettyDuration(time.Since(start))) + + feesWei := new(big.Int) + for _, tx := range block.Transactions() { + feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(tx.Gas()), tx.GasPrice())) + } + feesEth := new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) + + log.Info("Commit new mining work", "number", block.Number(), "uncles", len(uncles), "txs", w.current.tcount, + "gas", block.GasUsed(), "fees", feesEth, "elapsed", common.PrettyDuration(time.Since(start))) + case <-w.exitCh: log.Info("Worker has exited") } From 60390878a5128e64228de6873b5a16adddbbd68b Mon Sep 17 00:00:00 2001 From: Sasuke1964 Date: Fri, 17 Aug 2018 02:38:02 -0500 Subject: [PATCH 031/126] accounts: fixed typo (#17421) --- accounts/accounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/accounts.go b/accounts/accounts.go index 76951e1a42..cb1eae2815 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -106,7 +106,7 @@ type Wallet interface { // or optionally with the aid of any location metadata from the embedded URL field. // // If the wallet requires additional authentication to sign the request (e.g. - // a password to decrypt the account, or a PIN code o verify the transaction), + // a password to decrypt the account, or a PIN code to verify the transaction), // an AuthNeededError instance will be returned, containing infos for the user // about which fields or actions are needed. The user may retry by providing // the needed details via SignTxWithPassphrase, or by other means (e.g. unlock From f44046a1c6889049dbf0f9448075a43f5b280b09 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 17 Aug 2018 10:33:39 +0200 Subject: [PATCH 032/126] build: do not require `ethereum-swarm` deb when installing `ethereum` (#17425) --- build/ci.go | 11 ----------- build/deb/ethereum/deb.control | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/build/ci.go b/build/ci.go index ff23e15fdb..40252cbde1 100644 --- a/build/ci.go +++ b/build/ci.go @@ -644,17 +644,6 @@ func (meta debMetadata) ExeName(exe debExecutable) string { return exe.Package() } -// EthereumSwarmPackageName returns the name of the swarm package based on -// environment, e.g. "ethereum-swarm-unstable", or "ethereum-swarm". -// This is needed so that we make sure that "ethereum" package, -// depends on and installs "ethereum-swarm" -func (meta debMetadata) EthereumSwarmPackageName() string { - if isUnstableBuild(meta.Env) { - return debSwarm.Name + "-unstable" - } - return debSwarm.Name -} - // ExeConflicts returns the content of the Conflicts field // for executable packages. func (meta debMetadata) ExeConflicts(exe debExecutable) string { diff --git a/build/deb/ethereum/deb.control b/build/deb/ethereum/deb.control index e693d1d046..defb106fe3 100644 --- a/build/deb/ethereum/deb.control +++ b/build/deb/ethereum/deb.control @@ -10,7 +10,7 @@ Vcs-Browser: https://github.com/ethereum/go-ethereum Package: {{.Name}} Architecture: any -Depends: ${misc:Depends}, {{.EthereumSwarmPackageName}}, {{.ExeList}} +Depends: ${misc:Depends}, {{.ExeList}} Description: Meta-package to install geth, swarm, and other tools Meta-package to install geth, swarm and other tools From 2695fa2213fe5010a80970bca1078834662d5972 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 17 Aug 2018 12:21:53 +0200 Subject: [PATCH 033/126] les: fix crasher in NodeInfo when running as server (#17419) * les: fix crasher in NodeInfo when running as server The ProtocolManager computes CHT and Bloom trie roots by asking the indexers for their current head. It tried to get the indexers from LesOdr, but no LesOdr instance is created in server mode. Attempt to fix this by moving the indexers, protocol creation and NodeInfo to a new lesCommons struct which is embedded into both server and client. All this setup code should really be cleaned up, but this is just a hotfix so we have to do that some other time. * les: fix commons protocol maker --- les/backend.go | 48 ++++++++--------- les/commons.go | 106 +++++++++++++++++++++++++++++++++++++ les/handler.go | 128 ++++++++++----------------------------------- les/helper_test.go | 10 +--- les/server.go | 36 +++++++------ 5 files changed, 177 insertions(+), 151 deletions(-) create mode 100644 les/commons.go diff --git a/les/backend.go b/les/backend.go index 9b8cc1828f..d26c1470fe 100644 --- a/les/backend.go +++ b/les/backend.go @@ -34,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/light" @@ -47,26 +46,24 @@ import ( ) type LightEthereum struct { - config *eth.Config + lesCommons odr *LesOdr relay *LesTxRelay chainConfig *params.ChainConfig // Channel for shutting down the service shutdownChan chan bool - // Handlers - peers *peerSet - txPool *light.TxPool - blockchain *light.LightChain - protocolManager *ProtocolManager - serverPool *serverPool - reqDist *requestDistributor - retriever *retrieveManager - // DB interfaces - chainDb ethdb.Database // Block chain database - bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests - bloomIndexer, chtIndexer, bloomTrieIndexer *core.ChainIndexer + // Handlers + peers *peerSet + txPool *light.TxPool + blockchain *light.LightChain + serverPool *serverPool + reqDist *requestDistributor + retriever *retrieveManager + + bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests + bloomIndexer *core.ChainIndexer ApiBackend *LesApiBackend @@ -95,9 +92,11 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { quitSync := make(chan struct{}) leth := &LightEthereum{ - config: config, + lesCommons: lesCommons{ + chainDb: chainDb, + config: config, + }, chainConfig: chainConfig, - chainDb: chainDb, eventMux: ctx.EventMux, peers: peers, reqDist: newRequestDistributor(peers, quitSync), @@ -112,10 +111,12 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.relay = NewLesTxRelay(peers, leth.reqDist) leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) + leth.odr = NewLesOdr(chainDb, leth.retriever) leth.chtIndexer = light.NewChtIndexer(chainDb, true, leth.odr) leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, true, leth.odr) leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) + // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with // indexers already set but not started yet if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine); err != nil { @@ -125,6 +126,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.bloomIndexer.AddChildIndexer(leth.bloomTrieIndexer) leth.chtIndexer.Start(leth.blockchain) leth.bloomIndexer.Start(leth.blockchain) + // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -133,7 +135,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) - if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, ClientProtocolVersions, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { + if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { return nil, err } leth.ApiBackend = &LesApiBackend{leth, nil} @@ -215,14 +217,14 @@ func (s *LightEthereum) ResetWithGenesisBlock(gb *types.Block) { func (s *LightEthereum) BlockChain() *light.LightChain { return s.blockchain } func (s *LightEthereum) TxPool() *light.TxPool { return s.txPool } func (s *LightEthereum) Engine() consensus.Engine { return s.engine } -func (s *LightEthereum) LesVersion() int { return int(s.protocolManager.SubProtocols[0].Version) } +func (s *LightEthereum) LesVersion() int { return int(ClientProtocolVersions[0]) } func (s *LightEthereum) Downloader() *downloader.Downloader { return s.protocolManager.downloader } func (s *LightEthereum) EventMux() *event.TypeMux { return s.eventMux } // Protocols implements node.Service, returning all the currently configured // network protocols to start. func (s *LightEthereum) Protocols() []p2p.Protocol { - return s.protocolManager.SubProtocols + return s.makeProtocols(ClientProtocolVersions) } // Start implements node.Service, starting all internal goroutines needed by the @@ -242,12 +244,8 @@ func (s *LightEthereum) Start(srvr *p2p.Server) error { // Ethereum protocol. func (s *LightEthereum) Stop() error { s.odr.Stop() - if s.bloomIndexer != nil { - s.bloomIndexer.Close() - } - if s.chtIndexer != nil { - s.chtIndexer.Close() - } + s.bloomIndexer.Close() + s.chtIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() diff --git a/les/commons.go b/les/commons.go new file mode 100644 index 0000000000..251b7a5833 --- /dev/null +++ b/les/commons.go @@ -0,0 +1,106 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" +) + +// lesCommons contains fields needed by both server and client. +type lesCommons struct { + config *eth.Config + chainDb ethdb.Database + protocolManager *ProtocolManager + chtIndexer, bloomTrieIndexer *core.ChainIndexer +} + +// NodeInfo represents a short summary of the Ethereum sub-protocol metadata +// known about the host peer. +type NodeInfo struct { + Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain + Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block + Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules + Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block + CHT light.TrustedCheckpoint `json:"cht"` // Trused CHT checkpoint for fast catchup +} + +// makeProtocols creates protocol descriptors for the given LES versions. +func (c *lesCommons) makeProtocols(versions []uint) []p2p.Protocol { + protos := make([]p2p.Protocol, len(versions)) + for i, version := range versions { + version := version + protos[i] = p2p.Protocol{ + Name: "les", + Version: version, + Length: ProtocolLengths[version], + NodeInfo: c.nodeInfo, + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + return c.protocolManager.runPeer(version, p, rw) + }, + PeerInfo: func(id discover.NodeID) interface{} { + if p := c.protocolManager.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil { + return p.Info() + } + return nil + }, + } + } + return protos +} + +// nodeInfo retrieves some protocol metadata about the running host node. +func (c *lesCommons) nodeInfo() interface{} { + var cht light.TrustedCheckpoint + sections, _, sectionHead := c.chtIndexer.Sections() + sections2, _, sectionHead2 := c.bloomTrieIndexer.Sections() + if sections2 < sections { + sections = sections2 + sectionHead = sectionHead2 + } + if sections > 0 { + sectionIndex := sections - 1 + cht = light.TrustedCheckpoint{ + SectionIdx: sectionIndex, + SectionHead: sectionHead, + CHTRoot: light.GetChtRoot(c.chainDb, sectionIndex, sectionHead), + BloomRoot: light.GetBloomTrieRoot(c.chainDb, sectionIndex, sectionHead), + } + } + + chain := c.protocolManager.blockchain + head := chain.CurrentHeader() + hash := head.Hash() + return &NodeInfo{ + Network: c.config.NetworkId, + Difficulty: chain.GetTd(hash, head.Number.Uint64()), + Genesis: chain.Genesis().Hash(), + Config: chain.Config(), + Head: chain.CurrentHeader().Hash(), + CHT: cht, + } +} diff --git a/les/handler.go b/les/handler.go index ccb4a88448..ca40eaabfa 100644 --- a/les/handler.go +++ b/les/handler.go @@ -20,7 +20,6 @@ package les import ( "encoding/binary" "encoding/json" - "errors" "fmt" "math/big" "net" @@ -40,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -65,10 +63,6 @@ const ( disableClientRemovePeer = false ) -// errIncompatibleConfig is returned if the requested protocols and configs are -// not compatible (low protocol version restrictions and high requirements). -var errIncompatibleConfig = errors.New("incompatible configuration") - func errResp(code errCode, format string, v ...interface{}) error { return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) } @@ -115,8 +109,6 @@ type ProtocolManager struct { peers *peerSet maxPeers int - SubProtocols []p2p.Protocol - eventMux *event.TypeMux // channels for fetcher, syncer, txsyncLoop @@ -131,7 +123,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protocolVersions []uint, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { +func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, @@ -155,54 +147,6 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protoco manager.reqDist = odr.retriever.dist } - // Initiate a sub-protocol for every implemented version we can handle - manager.SubProtocols = make([]p2p.Protocol, 0, len(protocolVersions)) - for _, version := range protocolVersions { - // Compatible, initialize the sub-protocol - version := version // Closure for the run - manager.SubProtocols = append(manager.SubProtocols, p2p.Protocol{ - Name: "les", - Version: version, - Length: ProtocolLengths[version], - Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - var entry *poolEntry - peer := manager.newPeer(int(version), networkId, p, rw) - if manager.serverPool != nil { - addr := p.RemoteAddr().(*net.TCPAddr) - entry = manager.serverPool.connect(peer, addr.IP, uint16(addr.Port)) - } - peer.poolEntry = entry - select { - case manager.newPeerCh <- peer: - manager.wg.Add(1) - defer manager.wg.Done() - err := manager.handle(peer) - if entry != nil { - manager.serverPool.disconnect(entry) - } - return err - case <-manager.quitSync: - if entry != nil { - manager.serverPool.disconnect(entry) - } - return p2p.DiscQuitting - } - }, - NodeInfo: func() interface{} { - return manager.NodeInfo() - }, - PeerInfo: func(id discover.NodeID) interface{} { - if p := manager.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil { - return p.Info() - } - return nil - }, - }) - } - if len(manager.SubProtocols) == 0 { - return nil, errIncompatibleConfig - } - removePeer := manager.removePeer if disableClientRemovePeer { removePeer = func(id string) {} @@ -262,6 +206,32 @@ func (pm *ProtocolManager) Stop() { log.Info("Light Ethereum protocol stopped") } +// runPeer is the p2p protocol run function for the given version. +func (pm *ProtocolManager) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) error { + var entry *poolEntry + peer := pm.newPeer(int(version), pm.networkId, p, rw) + if pm.serverPool != nil { + addr := p.RemoteAddr().(*net.TCPAddr) + entry = pm.serverPool.connect(peer, addr.IP, uint16(addr.Port)) + } + peer.poolEntry = entry + select { + case pm.newPeerCh <- peer: + pm.wg.Add(1) + defer pm.wg.Done() + err := pm.handle(peer) + if entry != nil { + pm.serverPool.disconnect(entry) + } + return err + case <-pm.quitSync: + if entry != nil { + pm.serverPool.disconnect(entry) + } + return p2p.DiscQuitting + } +} + func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { return newPeer(pv, nv, p, newMeteredMsgWriter(rw)) } @@ -1203,50 +1173,6 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { return stats } -// NodeInfo represents a short summary of the Ethereum sub-protocol metadata -// known about the host peer. -type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) - Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain - Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block - Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules - Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block - CHT light.TrustedCheckpoint `json:"cht"` // Trused CHT checkpoint for fast catchup -} - -// NodeInfo retrieves some protocol metadata about the running host node. -func (self *ProtocolManager) NodeInfo() *NodeInfo { - head := self.blockchain.CurrentHeader() - hash := head.Hash() - - var cht light.TrustedCheckpoint - - sections, _, sectionHead := self.odr.ChtIndexer().Sections() - sections2, _, sectionHead2 := self.odr.BloomTrieIndexer().Sections() - if sections2 < sections { - sections = sections2 - sectionHead = sectionHead2 - } - if sections > 0 { - sectionIndex := sections - 1 - cht = light.TrustedCheckpoint{ - SectionIdx: sectionIndex, - SectionHead: sectionHead, - CHTRoot: light.GetChtRoot(self.chainDb, sectionIndex, sectionHead), - BloomRoot: light.GetBloomTrieRoot(self.chainDb, sectionIndex, sectionHead), - } - } - - return &NodeInfo{ - Network: self.networkId, - Difficulty: self.blockchain.GetTd(hash, head.Number.Uint64()), - Genesis: self.blockchain.Genesis().Hash(), - Config: self.blockchain.Config(), - Head: hash, - CHT: cht, - } -} - // downloaderPeerNotify implements peerSetNotify type downloaderPeerNotify ProtocolManager diff --git a/les/helper_test.go b/les/helper_test.go index 50c97e06e1..8817c20c7d 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -172,18 +172,12 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain = blockchain } - var protocolVersions []uint - if lightSync { - protocolVersions = ClientProtocolVersions - } else { - protocolVersions = ServerProtocolVersions - } - pm, err := NewProtocolManager(gspec.Config, lightSync, protocolVersions, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) + pm, err := NewProtocolManager(gspec.Config, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) if err != nil { return nil, err } if !lightSync { - srv := &LesServer{protocolManager: pm} + srv := &LesServer{lesCommons: lesCommons{protocolManager: pm}} pm.server = srv srv.defParams = &flowcontrol.ServerParams{ diff --git a/les/server.go b/les/server.go index a934fbf26e..df98d1e3a8 100644 --- a/les/server.go +++ b/les/server.go @@ -38,21 +38,19 @@ import ( ) type LesServer struct { - config *eth.Config - protocolManager *ProtocolManager - fcManager *flowcontrol.ClientManager // nil if our node is client only - fcCostStats *requestCostStats - defParams *flowcontrol.ServerParams - lesTopics []discv5.Topic - privateKey *ecdsa.PrivateKey - quitSync chan struct{} + lesCommons - chtIndexer, bloomTrieIndexer *core.ChainIndexer + fcManager *flowcontrol.ClientManager // nil if our node is client only + fcCostStats *requestCostStats + defParams *flowcontrol.ServerParams + lesTopics []discv5.Topic + privateKey *ecdsa.PrivateKey + quitSync chan struct{} } func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync := make(chan struct{}) - pm, err := NewProtocolManager(eth.BlockChain().Config(), false, ServerProtocolVersions, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) + pm, err := NewProtocolManager(eth.BlockChain().Config(), false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) if err != nil { return nil, err } @@ -63,13 +61,17 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } srv := &LesServer{ - config: config, - protocolManager: pm, - quitSync: quitSync, - lesTopics: lesTopics, - chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), - bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), + lesCommons: lesCommons{ + config: config, + chainDb: eth.ChainDb(), + chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), + bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), + protocolManager: pm, + }, + quitSync: quitSync, + lesTopics: lesTopics, } + logger := log.New() chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility @@ -104,7 +106,7 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } func (s *LesServer) Protocols() []p2p.Protocol { - return s.protocolManager.SubProtocols + return s.makeProtocols(ServerProtocolVersions) } // Start starts the LES server From 22cd3f70a68da31b1054fcfca74d187133df29b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 17 Aug 2018 15:47:14 +0300 Subject: [PATCH 034/126] miner: update mining log with correct fee calculation --- miner/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index e7e2796458..2f76f2a92c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -782,8 +782,8 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st w.unconfirmed.Shift(block.NumberU64() - 1) feesWei := new(big.Int) - for _, tx := range block.Transactions() { - feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(tx.Gas()), tx.GasPrice())) + for i, tx := range block.Transactions() { + feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), tx.GasPrice())) } feesEth := new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) From 251c868008f30b2991bc6986e60a0e7bbdc78b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 17 Aug 2018 18:12:39 +0300 Subject: [PATCH 035/126] consensus/ethash: reduce notify test aggressiveness --- consensus/ethash/sealer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 6d8a770495..6c7157a5ae 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -70,7 +70,7 @@ func TestRemoteNotify(t *testing.T) { // issues in the notifications. func TestRemoteMultiNotify(t *testing.T) { // Start a simple webserver to capture notifications - sink := make(chan [3]string, 1024) + sink := make(chan [3]string, 64) server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { From 0fd02fe9cff54a5edce87588ed780c76a95329fd Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 18 Aug 2018 06:07:20 +0900 Subject: [PATCH 036/126] console: fixed comment typo --- console/console.go | 2 +- console/console_test.go | 8 ++++---- console/prompter.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/console/console.go b/console/console.go index 56e03837ac..3c397f8006 100644 --- a/console/console.go +++ b/console/console.go @@ -314,7 +314,7 @@ func (c *Console) Interactive() { input = "" // Current user input scheduler = make(chan string) // Channel to send the next prompt on and receive the input ) - // Start a goroutine to listen for promt requests and send back inputs + // Start a goroutine to listen for prompt requests and send back inputs go func() { for { // Read the next user input diff --git a/console/console_test.go b/console/console_test.go index 7b1629c032..26465ca6f4 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -201,7 +201,7 @@ func TestInteractive(t *testing.T) { go tester.console.Interactive() - // Wait for a promt and send a statement back + // Wait for a prompt and send a statement back select { case <-tester.input.scheduler: case <-time.After(time.Second): @@ -212,7 +212,7 @@ func TestInteractive(t *testing.T) { case <-time.After(time.Second): t.Fatalf("input feedback timeout") } - // Wait for the second promt and ensure first statement was evaluated + // Wait for the second prompt and ensure first statement was evaluated select { case <-tester.input.scheduler: case <-time.After(time.Second): @@ -249,7 +249,7 @@ func TestExecute(t *testing.T) { } // Tests that the JavaScript objects returned by statement executions are properly -// pretty printed instead of just displaing "[object]". +// pretty printed instead of just displaying "[object]". func TestPrettyPrint(t *testing.T) { tester := newTester(t, nil) defer tester.Close(t) @@ -300,7 +300,7 @@ func TestIndenting(t *testing.T) { }{ {`var a = 1;`, 0}, {`"some string"`, 0}, - {`"some string with (parentesis`, 0}, + {`"some string with (parenthesis`, 0}, {`"some string with newline ("`, 0}, {`function v(a,b) {}`, 0}, diff --git a/console/prompter.go b/console/prompter.go index c477b48178..9b90034db9 100644 --- a/console/prompter.go +++ b/console/prompter.go @@ -27,7 +27,7 @@ import ( // Only this reader may be used for input because it keeps an internal buffer. var Stdin = newTerminalPrompter() -// UserPrompter defines the methods needed by the console to promt the user for +// UserPrompter defines the methods needed by the console to prompt the user for // various types of inputs. type UserPrompter interface { // PromptInput displays the given prompt to the user and requests some textual From d3488c1affee6843d75db77b36f504c73e5e02e7 Mon Sep 17 00:00:00 2001 From: Wuxiang Date: Mon, 20 Aug 2018 20:07:21 +0800 Subject: [PATCH 037/126] p2p: fix typo (#17446) --- p2p/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/protocol.go b/p2p/protocol.go index ee747ba23d..948aeb494f 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -35,7 +35,7 @@ type Protocol struct { // by the protocol. Length uint64 - // Run is called in a new groutine when the protocol has been + // Run is called in a new goroutine when the protocol has been // negotiated with a peer. It should read and write messages from // rw. The Payload for each message must be fully consumed. // From c4078fc80527b2cf0c2fd3e8771e9db06a597152 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 20 Aug 2018 14:09:50 +0200 Subject: [PATCH 038/126] cmd/swarm: added swarm bootnodes (#17414) --- cmd/swarm/bootnodes.go | 77 ++++++++++++++++++++++++++++++++++++++++++ cmd/swarm/config.go | 8 ----- cmd/swarm/main.go | 51 ++++++++++++---------------- swarm/api/config.go | 2 -- 4 files changed, 98 insertions(+), 40 deletions(-) create mode 100644 cmd/swarm/bootnodes.go diff --git a/cmd/swarm/bootnodes.go b/cmd/swarm/bootnodes.go new file mode 100644 index 0000000000..cbba9970da --- /dev/null +++ b/cmd/swarm/bootnodes.go @@ -0,0 +1,77 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +var SwarmBootnodes = []string{ + // Foundation Swarm Gateway Cluster + "enode://e5c6f9215c919a5450a7b8c14c22535607b69f2c8e1e7f6f430cb25d7a2c27cd1df4c4f18ad7c1d7e5162e271ffcd3f20b1a1467fb6e790e7d727f3b2193de97@52.232.7.187:30399", + "enode://9b2fe07e69ccc7db5fef15793dab7d7d2e697ed92132d6e9548218e68a34613a8671ad03a6658d862b468ed693cae8a0f8f8d37274e4a657ffb59ca84676e45b@52.232.7.187:30400", + "enode://76c1059162c93ef9df0f01097c824d17c492634df211ef4c806935b349082233b63b90c23970254b3b7138d630400f7cf9b71e80355a446a8b733296cb04169a@52.232.7.187:30401", + "enode://ce46bbe2a8263145d65252d52da06e000ad350ed09c876a71ea9544efa42f63c1e1b6cc56307373aaad8f9dd069c90d0ed2dd1530106200e16f4ca681dd8ae2d@52.232.7.187:30402", + "enode://f431e0d6008a6c35c6e670373d828390c8323e53da8158e7bfc43cf07e632cc9e472188be8df01decadea2d4a068f1428caba769b632554a8fb0607bc296988f@52.232.7.187:30403", + "enode://174720abfff83d7392f121108ae50ea54e04889afe020df883655c0f6cb95414db945a0228d8982fe000d86fc9f4b7669161adc89cd7cd56f78f01489ab2b99b@52.232.7.187:30404", + "enode://2ae89be4be61a689b6f9ecee4360a59e185e010ab750f14b63b4ae43d4180e872e18e3437d4386ce44875dc7cc6eb761acba06412fe3178f3dac1dab3b65703e@52.232.7.187:30405", + "enode://24abebe1c0e6d75d6052ce3219a87be8573fd6397b4cb51f0773b83abba9b3d872bfb273cdc07389715b87adfac02f5235f5241442c5089802cbd8d42e310fce@52.232.7.187:30406", + "enode://d08dfa46bfbbdbcaafbb6e34abee4786610f6c91e0b76d7881f0334ac10dda41d8c1f2b6eedffb4493293c335c0ad46776443b2208d1fbbb9e1a90b25ee4eef2@52.232.7.187:30407", + "enode://8d95eb0f837d27581a43668ed3b8783d69dc4e84aa3edd7a0897e026155c8f59c8702fdc0375ee7bac15757c9c78e1315d9b73e4ce59c936db52ea4ae2f501c7@52.232.7.187:30408", + "enode://a5967cc804aebd422baaaba9f06f27c9e695ccab335b61088130f8cbe64e3cdf78793868c7051dfc06eecfe844fad54bc7f6dfaed9db3c7ecef279cb829c25fb@52.232.7.187:30409", + "enode://5f00134d81a8f2ebcc46f8766f627f492893eda48138f811b7de2168308171968f01710bca6da05764e74f14bae41652f554e6321f1aed85fa3461e89d075dbf@52.232.7.187:30410", + "enode://b2142b79b01a5aa66a5e23cc35e78219a8e97bc2412a6698cee24ae02e87078b725d71730711bd62e25ff1aa8658c6633778af8ac14c63814a337c3dd0ebda9f@52.232.7.187:30411", + "enode://1ffa7651094867d6486ce3ef46d27a052c2cb968b618346c6df7040322c7efc3337547ba85d4cbba32e8b31c42c867202554735c06d4c664b9afada2ed0c4b3c@52.232.7.187:30412", + "enode://129e0c3d5f5df12273754f6f703d2424409fa4baa599e0b758c55600169313887855e75b082028d2302ec034b303898cd697cc7ae8256ba924ce927510da2c8d@52.232.7.187:30413", + "enode://419e2dc0d2f5b022cf16b0e28842658284909fa027a0fbbb5e2b755e7f846ea02a8f0b66a7534981edf6a7bcf8a14855344c6668e2cd4476ccd35a11537c9144@52.232.7.187:30414", + "enode://23d55ad900583231b91f2f62e3f72eb498b342afd58b682be3af052eed62b5651094471065981de33d8786f075f05e3cca499503b0ac8ae84b2a06e99f5b0723@52.232.7.187:30415", + "enode://bc56e4158c00e9f616d7ea533def20a89bef959df4e62a768ff238ff4e1e9223f57ecff969941c20921bad98749baae311c0fbebce53bf7bbb9d3dc903640990@52.232.7.187:30416", + "enode://433ce15199c409875e7e72fffd69fdafe746f17b20f0d5555281722a65fde6c80328fab600d37d8624509adc072c445ce0dad4a1c01cff6acf3132c11d429d4d@52.232.7.187:30417", + "enode://632ee95b8f0eac51ef89ceb29313fef3a60050181d66a6b125583b1a225a7694b252edc016efb58aa3b251da756cb73280842a022c658ed405223b2f58626343@52.232.7.187:30418", + "enode://4a0f9bcff7a4b9ee453fb298d0fb222592efe121512e30cd72fef631beb8c6a15153a1456eb073ee18551c0e003c569651a101892dc4124e90b933733a498bb5@52.232.7.187:30419", + "enode://f0d80fbc72d16df30e19aac3051eb56a7aff0c8367686702e01ea132d8b0b3ee00cadd6a859d2cca98ec68d3d574f8a8a87dba2347ec1e2818dc84bc3fa34fae@52.232.7.187:30420", + "enode://a199146906e4f9f2b94b195a8308d9a59a3564b92efaab898a4243fe4c2ad918b7a8e4853d9d901d94fad878270a2669d644591299c3d43de1b298c00b92b4a7@52.232.7.187:30421", + "enode://052036ea8736b37adbfb684d90ce43e11b3591b51f31489d7c726b03618dea4f73b1e659deb928e6bf40564edcdcf08351643f42db3d4ca1c2b5db95dad59e94@52.232.7.187:30422", + "enode://460e2b8c6da8f12fac96c836e7d108f4b7ec55a1c64631bb8992339e117e1c28328fee83af863196e20af1487a655d13e5ceba90e980e92502d5bac5834c1f71@52.232.7.187:30423", + "enode://6d2cdd13741b2e72e9031e1b93c6d9a4e68de2844aa4e939f6a8a8498a7c1d7e2ee4c64217e92a6df08c9a32c6764d173552810ef1bd2ecb356532d389dd2136@52.232.7.187:30424", + "enode://62105fc25ce2cd5b299647f47eaa9211502dc76f0e9f461df915782df7242ac3223e3db04356ae6ed2977ccac20f0b16864406e9ca514a40a004cb6a5d0402aa@52.232.7.187:30425", + "enode://e0e388fc520fd493c33f0ce16685e6f98fb6aec28f2edc14ee6b179594ee519a896425b0025bb6f0e182dd3e468443f19c70885fbc66560d000093a668a86aa8@52.232.7.187:30426", + "enode://63f3353a72521ea10022127a4fe6b4acbef197c3fe668fd9f4805542d8a6fcf79f6335fbab62d180a35e19b739483e740858b113fdd7c13a26ad7b4e318a5aef@52.232.7.187:30427", + "enode://33a42b927085678d4aefd4e70b861cfca6ef5f6c143696c4f755973fd29e64c9e658cad57a66a687a7a156da1e3688b1fbdd17bececff2ee009fff038fa5666b@52.232.7.187:30428", + "enode://259ab5ab5c1daee3eab7e3819ab3177b82d25c29e6c2444fdd3f956e356afae79a72840ccf2d0665fe82c81ebc3b3734da1178ac9fd5d62c67e674b69f86b6be@52.232.7.187:30429", + "enode://558bccad7445ce3fd8db116ed6ab4aed1324fdbdac2348417340c1764dc46d46bffe0728e5b7d5c36f12e794c289f18f57f08f085d2c65c9910a5c7a65b6a66a@52.232.7.187:30430", + "enode://abe60937a0657ffded718e3f84a32987286983be257bdd6004775c4b525747c2b598f4fac49c8de324de5ce75b22673fa541a7ce2d555fb7f8ca325744ae3577@52.232.7.187:30431", + "enode://bce6f0aaa5b230742680084df71d4f026b3eff7f564265599216a1b06b765303fdc9325de30ffd5dfdaf302ce4b14322891d2faea50ce2ca298d7409f5858339@52.232.7.187:30432", + "enode://21b957c4e03277d42be6660730ec1b93f540764f26c6abdb54d006611139c7081248486206dfbf64fcaffd62589e9c6b8ea77a5297e4b21a605f1bcf49483ed0@52.232.7.187:30433", + "enode://ff104e30e64f24c3d7328acee8b13354e5551bc8d60bb25ecbd9632d955c7e34bb2d969482d173355baad91c8282f8b592624eb3929151090da3b4448d4d58fb@52.232.7.187:30434", + "enode://c76e2b5f81a521bceaec1518926a21380a345df9cf463461562c6845795512497fb67679e155fc96a74350f8b78de8f4c135dd52b106dbbb9795452021d09ea5@52.232.7.187:30435", + "enode://3288fd860105164f3e9b69934c4eb18f7146cfab31b5a671f994e21a36e9287766e5f9f075aefbc404538c77f7c2eb2a4495020a7633a1c3970d94e9fa770aeb@52.232.7.187:30436", + "enode://6cea859c7396d46b20cfcaa80f9a11cd112f8684f2f782f7b4c0e1e0af9212113429522075101923b9b957603e6c32095a6a07b5e5e35183c521952ee108dfaf@52.232.7.187:30437", + "enode://f628ec56e4ca8317cc24cc4ac9b27b95edcce7b96e1c7f3b53e30de4a8580fe44f2f0694a513bdb0a431acaf2824074d6ace4690247bbc34c14f426af8c056ea@52.232.7.187:30438", + "enode://055ec8b26fc105c4f97970a1cce9773a5e34c03f511b839db742198a1c571e292c54aa799e9afb991cc8a560529b8cdf3e0c344bc6c282aff2f68eec59361ddf@52.232.7.187:30439", + "enode://48cb0d430c328974226aa33a931d8446cd5a8d40f3ead8f4ce7ad60faa1278192eb6d58bed91258d63e81f255fc107eec2425ce2ae8b22350dd556076e160610@52.232.7.187:30440", + "enode://3fadb7af7f770d5ffc6b073b8d42834bebb18ce1fe8a4fe270d2b799e7051327093960dc61d9a18870db288f7746a0e6ea2a013cd6ab0e5f97ca08199473aace@52.232.7.187:30441", + "enode://a5d7168024c9992769cf380ffa559a64b4f39a29d468f579559863814eb0ae0ed689ac0871a3a2b4c78b03297485ec322d578281131ef5d5c09a4beb6200a97a@52.232.7.187:30442", + "enode://9c57744c5b2c2d71abcbe80512652f9234d4ab041b768a2a886ab390fe6f184860f40e113290698652d7e20a8ac74d27ac8671db23eb475b6c5e6253e4693bf8@52.232.7.187:30443", + "enode://daca9ff0c3176045a0e0ed228dee00ec86bc0939b135dc6b1caa23745d20fd0332e1ee74ad04020e89df56c7146d831a91b89d15ca3df05ba7618769fefab376@52.232.7.187:30444", + "enode://a3f6af59428cb4b9acb198db15ef5554fa43c2b0c18e468a269722d64a27218963a2975eaf82750b6262e42192b5e3669ea51337b4cda62b33987981bc5e0c1a@52.232.7.187:30445", + "enode://fe571422fa4651c3354c85dac61911a6a6520dd3c0332967a49d4133ca30e16a8a4946fa73ca2cb5de77917ea701a905e1c3015b2f4defcd53132b61cc84127a@52.232.7.187:30446", + + // Mainframe + "enode://ee9a5a571ea6c8a59f9a8bb2c569c865e922b41c91d09b942e8c1d4dd2e1725bd2c26149da14de1f6321a2c6fdf1e07c503c3e093fb61696daebf74d6acd916b@54.186.219.160:30399", + "enode://a03f0562ecb8a992ad5242345535e73483cdc18ab934d36bf24b567d43447c2cea68f89f1d51d504dd13acc30f24ebce5a150bea2ccb1b722122ce4271dc199d@52.67.248.147:30399", + "enode://e2cbf9eafd85903d3b1c56743035284320695e0072bc8d7396e0542aa5e1c321b236f67eab66b79c2f15d4447fa4bbe74dd67d0467da23e7eb829f60ec8a812b@13.58.169.1:30399", + "enode://8b8c6bda6047f1cad9fab2db4d3d02b7aa26279902c32879f7bcd4a7d189fee77fdc36ee151ce6b84279b4792e72578fd529d2274d014132465758fbfee51cee@13.209.13.15:30399", + "enode://63f6a8818927e429585287cf2ca0cb9b11fa990b7b9b331c2962cdc6f21807a2473b26e8256225c26caff70d7218e59586d704d49061452c6852e382c885d03c@35.154.106.174:30399", + "enode://ed4bd3b794ed73f18e6dcc70c6624dfec63b5654f6ab54e8f40b16eff8afbd342d4230e099ddea40e84423f81b2d2ea79799dc345257b1fec6f6c422c9d008f7@52.213.20.99:30399", +} diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 1183f8bc81..ae4b5816e6 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -233,10 +233,6 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.Cors = cors } - if ctx.GlobalIsSet(utils.BootnodesFlag.Name) { - currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name) - } - if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" { currentConfig.LocalStoreParams.ChunkDbPath = storePath } @@ -334,10 +330,6 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.Cors = cors } - if bootnodes := os.Getenv(SWARM_ENV_BOOTNODES); bootnodes != "" { - currentConfig.BootNodes = bootnodes - } - return currentConfig } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 76be60cb68..637ae06e96 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" @@ -67,14 +66,7 @@ OPTIONS: ` var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) - testbetBootNodes = []string{ - "enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429", - "enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430", - "enode://fe29b82319b734ce1ec68b84657d57145fee237387e63273989d354486731e59f78858e452ef800a020559da22dcca759536e6aa5517c53930d29ce0b1029286@13.74.157.139:30431", - "enode://1d7187e7bde45cf0bee489ce9852dd6d1a0d9aa67a33a6b8e6db8a4fbc6fcfa6f0f1a5419343671521b863b187d1c73bad3603bae66421d157ffef357669ddb8@13.74.157.139:30432", - "enode://0e4cba800f7b1ee73673afa6a4acead4018f0149d2e3216be3f133318fd165b324cd71b81fbe1e80deac8dbf56e57a49db7be67f8b9bc81bd2b7ee496434fb5d@13.74.157.139:30433", - } + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) ) var ( @@ -619,6 +611,9 @@ func bzzd(ctx *cli.Context) error { if _, err := os.Stat(bzzconfig.Path); err == nil { cfg.DataDir = bzzconfig.Path } + + //optionally set the bootnodes before configuring the node + setSwarmBootstrapNodes(ctx, &cfg) //setup the ethereum node utils.SetNodeConfig(ctx, &cfg) stack, err := node.New(&cfg) @@ -643,16 +638,6 @@ func bzzd(ctx *cli.Context) error { stack.Stop() }() - // Add bootnodes as initial peers. - if bzzconfig.BootNodes != "" { - bootnodes := strings.Split(bzzconfig.BootNodes, ",") - injectBootnodes(stack.Server(), bootnodes) - } else { - if bzzconfig.NetworkID == 3 { - injectBootnodes(stack.Server(), testbetBootNodes) - } - } - stack.Wait() return nil } @@ -760,17 +745,6 @@ func getPassPhrase(prompt string, i int, passwords []string) string { return password } -func injectBootnodes(srv *p2p.Server, nodes []string) { - for _, url := range nodes { - n, err := discover.ParseNode(url) - if err != nil { - log.Error("Invalid swarm bootnode", "err", err) - continue - } - srv.AddPeer(n) - } -} - // addDefaultHelpSubcommand scans through defined CLI commands and adds // a basic help subcommand to each // if a help command is already defined, it will take precedence over the default. @@ -783,3 +757,20 @@ func addDefaultHelpSubcommands(commands []cli.Command) { } } } + +func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) { + if ctx.GlobalIsSet(utils.BootnodesFlag.Name) || ctx.GlobalIsSet(utils.BootnodesV4Flag.Name) { + return + } + + cfg.P2P.BootstrapNodes = []*discover.Node{} + + for _, url := range SwarmBootnodes { + node, err := discover.ParseNode(url) + if err != nil { + log.Error("Bootstrap URL invalid", "enode", url, "err", err) + } + cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node) + } + log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes)) +} diff --git a/swarm/api/config.go b/swarm/api/config.go index bdfffdd05f..3044dc2e52 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -68,7 +68,6 @@ type Config struct { SwapAPI string Cors string BzzAccount string - BootNodes string privateKey *ecdsa.PrivateKey } @@ -93,7 +92,6 @@ func NewConfig() (c *Config) { DeliverySkipCheck: false, SyncUpdateDelay: 15 * time.Second, SwapAPI: "", - BootNodes: "", } return From a8aa89accb0bcd4f24cd430d8f100e0ff9e239d0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 20 Aug 2018 14:10:30 +0200 Subject: [PATCH 039/126] swarm/storage: cleanup task - remove bigger chunks (#17424) --- swarm/storage/ldbstore.go | 67 ++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 7920ee7674..b95aa13b09 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" @@ -384,14 +385,13 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } func (s *LDBStore) Cleanup() { - //Iterates over the database and checks that there are no faulty chunks + //Iterates over the database and checks that there are no chunks bigger than 4kb + var errorsFound, removed, total int + it := s.db.NewIterator() - startPosition := []byte{keyIndex} - it.Seek(startPosition) - var key []byte - var errorsFound, total int - for it.Valid() { - key = it.Key() + defer it.Release() + for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() { + key := it.Key() if (key == nil) || (key[0] != keyIndex) { break } @@ -399,27 +399,50 @@ func (s *LDBStore) Cleanup() { var index dpaDBIndex err := decodeIndex(it.Value(), &index) if err != nil { - it.Next() + log.Warn("Cannot decode") + errorsFound++ continue } - data, err := s.db.Get(getDataKey(index.Idx, s.po(Address(key[1:])))) + hash := key[1:] + po := s.po(hash) + datakey := getDataKey(index.Idx, po) + data, err := s.db.Get(datakey) if err != nil { - log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) - s.delete(index.Idx, getIndexKey(key[1:]), s.po(Address(key[1:]))) - errorsFound++ - } else { - hasher := s.hashfunc() - hasher.Write(data[32:]) - hash := hasher.Sum(nil) - if !bytes.Equal(hash, key[1:]) { - log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:])) - s.delete(index.Idx, getIndexKey(key[1:]), s.po(Address(key[1:]))) + found := false + + // highest possible proximity is 255 + for po = 1; po <= 255; po++ { + datakey = getDataKey(index.Idx, po) + data, err = s.db.Get(datakey) + if err == nil { + found = true + break + } + } + + if !found { + log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key[:])) + errorsFound++ + continue } } - it.Next() + + c := &Chunk{} + ck := data[:32] + decodeData(data, c) + + cs := int64(binary.LittleEndian.Uint64(c.SData[:8])) + log.Trace("chunk", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.SData), "size", cs) + + if len(c.SData) > chunk.DefaultSize+8 { + log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.SData), "size", cs) + s.delete(index.Idx, getIndexKey(key[1:]), po) + removed++ + errorsFound++ + } } - it.Release() - log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) + + log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) } func (s *LDBStore) ReIndex() { From 55d050ccd81d726ce47cfb57ecd9de662129b7f1 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 20 Aug 2018 14:11:08 +0200 Subject: [PATCH 040/126] travis: remove brew update and osxfuse install (#17429) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index afa9ab503f..3ae88aab6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,8 +30,6 @@ matrix: go: 1.10.x script: - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - - brew update - - brew cask install osxfuse - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES From c929030e280ecae5e5943fa277240ee7d09d7506 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 20 Aug 2018 05:14:50 -0700 Subject: [PATCH 041/126] core/types: fix docs about protected Vs (#17436) --- core/types/transaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 82af9335ff..9c6e77be98 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -119,7 +119,7 @@ func isProtectedV(V *big.Int) bool { v := V.Uint64() return v != 27 && v != 28 } - // anything not 27 or 28 are considered unprotected + // anything not 27 or 28 is considered protected return true } From 1de9ada4016d7028d1d1529bc7d0676c98ddb5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 20 Aug 2018 15:49:28 +0200 Subject: [PATCH 042/126] light: new CHTs (#17448) --- light/postprocess.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/light/postprocess.go b/light/postprocess.go index 0b25e1d881..42985aff05 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -61,18 +61,18 @@ type TrustedCheckpoint struct { var ( mainnetCheckpoint = TrustedCheckpoint{ name: "mainnet", - SectionIdx: 179, - SectionHead: common.HexToHash("ae778e455492db1183e566fa0c67f954d256fdd08618f6d5a393b0e24576d0ea"), - CHTRoot: common.HexToHash("646b338f9ca74d936225338916be53710ec84020b89946004a8605f04c817f16"), - BloomRoot: common.HexToHash("d0f978f5dbc86e5bf931d8dd5b2ecbebbda6dc78f8896af6a27b46a3ced0ac25"), + SectionIdx: 187, + SectionHead: common.HexToHash("e6baa034efa31562d71ff23676512dec6562c1ad0301e08843b907e81958c696"), + CHTRoot: common.HexToHash("28001955219719cf06de1b08648969139d123a9835fc760547a1e4dabdabc15a"), + BloomRoot: common.HexToHash("395ca2373fc662720ac6b58b3bbe71f68aa0f38b63b2d3553dd32ff3c51eebc4"), } ropstenCheckpoint = TrustedCheckpoint{ name: "ropsten", - SectionIdx: 107, - SectionHead: common.HexToHash("e1988f95399debf45b873e065e5cd61b416ef2e2e5deec5a6f87c3127086e1ce"), - CHTRoot: common.HexToHash("15cba18e4de0ab1e95e202625199ba30147aec8b0b70384b66ebea31ba6a18e0"), - BloomRoot: common.HexToHash("e00fa6389b2e597d9df52172cd8e936879eed0fca4fa59db99e2c8ed682562f2"), + SectionIdx: 117, + SectionHead: common.HexToHash("9529b38631ae30783f56cbe4c3b9f07575b770ecba4f6e20a274b1e2f40fede1"), + CHTRoot: common.HexToHash("6f48e9f101f1fac98e7d74fbbcc4fda138358271ffd974d40d2506f0308bb363"), + BloomRoot: common.HexToHash("8242342e66e942c0cd893484e6736b9862ceb88b43ca344bb06a8285ac1b6d64"), } ) From 7d38d53ae449c6ec06f7b0579f1a189b02222a60 Mon Sep 17 00:00:00 2001 From: Nilesh Trivedi Date: Mon, 20 Aug 2018 19:24:38 +0530 Subject: [PATCH 043/126] cmd/puppeth: accept ssh identity in the server string (#17407) * cmd/puppeth: Accept identityfile in the server string with fallback to id_rsa * cmd/puppeth: code polishes + fix heath check double ports --- cmd/puppeth/ssh.go | 52 ++++++++++++++++++++--------------- cmd/puppeth/wizard_network.go | 8 +++--- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 158261ce05..c507596065 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -45,33 +45,44 @@ type sshClient struct { // dial establishes an SSH connection to a remote node using the current user and // the user's configured private RSA key. If that fails, password authentication -// is fallen back to. The caller may override the login user via user@server:port. +// is fallen back to. server can be a string like user:identity@server:port. func dial(server string, pubkey []byte) (*sshClient, error) { - // Figure out a label for the server and a logger - label := server - if strings.Contains(label, ":") { - label = label[:strings.Index(label, ":")] - } - login := "" + // Figure out username, identity, hostname and port + hostname := "" + hostport := server + username := "" + identity := "id_rsa" // default + if strings.Contains(server, "@") { - login = label[:strings.Index(label, "@")] - label = label[strings.Index(label, "@")+1:] - server = server[strings.Index(server, "@")+1:] + prefix := server[:strings.Index(server, "@")] + if strings.Contains(prefix, ":") { + username = prefix[:strings.Index(prefix, ":")] + identity = prefix[strings.Index(prefix, ":")+1:] + } else { + username = prefix + } + hostport = server[strings.Index(server, "@")+1:] } - logger := log.New("server", label) + if strings.Contains(hostport, ":") { + hostname = hostport[:strings.Index(hostport, ":")] + } else { + hostname = hostport + hostport += ":22" + } + logger := log.New("server", server) logger.Debug("Attempting to establish SSH connection") user, err := user.Current() if err != nil { return nil, err } - if login == "" { - login = user.Username + if username == "" { + username = user.Username } // Configure the supported authentication methods (private key and password) var auths []ssh.AuthMethod - path := filepath.Join(user.HomeDir, ".ssh", "id_rsa") + path := filepath.Join(user.HomeDir, ".ssh", identity) if buf, err := ioutil.ReadFile(path); err != nil { log.Warn("No SSH key, falling back to passwords", "path", path, "err", err) } else { @@ -94,14 +105,14 @@ func dial(server string, pubkey []byte) (*sshClient, error) { } } auths = append(auths, ssh.PasswordCallback(func() (string, error) { - fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server) + fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) fmt.Println() return string(blob), err })) // Resolve the IP address of the remote server - addr, err := net.LookupHost(label) + addr, err := net.LookupHost(hostname) if err != nil { return nil, err } @@ -109,10 +120,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { return nil, errors.New("no IPs associated with domain") } // Try to dial in to the remote server - logger.Trace("Dialing remote SSH server", "user", login) - if !strings.Contains(server, ":") { - server += ":22" - } + logger.Trace("Dialing remote SSH server", "user", username) keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error { // If no public key is known for SSH, ask the user to confirm if pubkey == nil { @@ -139,13 +147,13 @@ func dial(server string, pubkey []byte) (*sshClient, error) { // We have a mismatch, forbid connecting return errors.New("ssh key mismatch, readd the machine to update") } - client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: login, Auth: auths, HostKeyCallback: keycheck}) + client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck}) if err != nil { return nil, err } // Connection established, return our utility wrapper c := &sshClient{ - server: label, + server: hostname, address: addr[0], pubkey: pubkey, client: client, diff --git a/cmd/puppeth/wizard_network.go b/cmd/puppeth/wizard_network.go index d780c550b1..c0ddcc2a3c 100644 --- a/cmd/puppeth/wizard_network.go +++ b/cmd/puppeth/wizard_network.go @@ -62,14 +62,14 @@ func (w *wizard) manageServers() { } } -// makeServer reads a single line from stdin and interprets it as a hostname to -// connect to. It tries to establish a new SSH session and also executing some -// baseline validations. +// makeServer reads a single line from stdin and interprets it as +// username:identity@hostname to connect to. It tries to establish a +// new SSH session and also executing some baseline validations. // // If connection succeeds, the server is added to the wizards configs! func (w *wizard) makeServer() string { fmt.Println() - fmt.Println("Please enter remote server's address:") + fmt.Println("What is the remote server's address ([username[:identity]@]hostname[:port])?") // Read and dial the server to ensure docker is present input := w.readString() From a6d45a5d00e0f37eab9eba43f1ec60429f0dc120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Aug 2018 18:05:06 +0300 Subject: [PATCH 044/126] crypto/bn256: add missing license file, release wrapper in BSD-3 --- crypto/bn256/LICENSE | 28 ++++++++++++++++++++++++++++ crypto/bn256/bn256_fast.go | 18 +++--------------- crypto/bn256/bn256_fuzz.go | 18 +++--------------- crypto/bn256/bn256_slow.go | 18 +++--------------- crypto/bn256/cloudflare/LICENSE | 27 +++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 45 deletions(-) create mode 100644 crypto/bn256/LICENSE create mode 100644 crypto/bn256/cloudflare/LICENSE diff --git a/crypto/bn256/LICENSE b/crypto/bn256/LICENSE new file mode 100644 index 0000000000..634e0cb2c3 --- /dev/null +++ b/crypto/bn256/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2018 Péter Szilágyi. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crypto/bn256/bn256_fast.go b/crypto/bn256/bn256_fast.go index a8dfa8f67d..5c081493b0 100644 --- a/crypto/bn256/bn256_fast.go +++ b/crypto/bn256/bn256_fast.go @@ -1,18 +1,6 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// Copyright 2018 Péter Szilágyi. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. // +build amd64 arm64 diff --git a/crypto/bn256/bn256_fuzz.go b/crypto/bn256/bn256_fuzz.go index f360b0541f..6aa1421170 100644 --- a/crypto/bn256/bn256_fuzz.go +++ b/crypto/bn256/bn256_fuzz.go @@ -1,18 +1,6 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// Copyright 2018 Péter Szilágyi. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. // +build gofuzz diff --git a/crypto/bn256/bn256_slow.go b/crypto/bn256/bn256_slow.go index 61373763b8..47df49d417 100644 --- a/crypto/bn256/bn256_slow.go +++ b/crypto/bn256/bn256_slow.go @@ -1,18 +1,6 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// Copyright 2018 Péter Szilágyi. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. // +build !amd64,!arm64 diff --git a/crypto/bn256/cloudflare/LICENSE b/crypto/bn256/cloudflare/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/crypto/bn256/cloudflare/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 106d196ec4a6451efedc60ab15957f231fa85639 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 21 Aug 2018 09:48:53 +0200 Subject: [PATCH 045/126] eth: ensure from= 0 { + return nil, fmt.Errorf("end block (#%d) needs to come after start block (#%d)", end, start) + } return api.traceChain(ctx, from, to, config) } From dcd97c41fa2789f2e642cb52b8e18c5bd7bcc81d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 21 Aug 2018 10:34:10 +0200 Subject: [PATCH 046/126] swarm, swarm/network, swarm/pss: log error and fix logs (#17410) * swarm, swarm/network, swarm/pss: log error and fix logs * swarm/pss: log compressed publickey --- swarm/network/hive.go | 4 ++-- swarm/pss/pss.go | 5 +++-- swarm/swarm.go | 34 ++++++++++++++++------------------ 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index a54a17d29d..3660210883 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -102,10 +102,10 @@ func NewHive(params *HiveParams, overlay Overlay, store state.Store) *Hive { // server is used to connect to a peer based on its NodeID or enode URL // these are called on the p2p.Server which runs on the node func (h *Hive) Start(server *p2p.Server) error { - log.Info(fmt.Sprintf("%08x hive starting", h.BaseAddr()[:4])) + log.Info("Starting hive", "baseaddr", fmt.Sprintf("%x", h.BaseAddr()[:4])) // if state store is specified, load peers to prepopulate the overlay address book if h.Store != nil { - log.Info("detected an existing store. trying to load peers") + log.Info("Detected an existing store. trying to load peers") if err := h.loadPeers(); err != nil { log.Error(fmt.Sprintf("%08x hive encoutered an error trying to load peers", h.BaseAddr()[:4])) return err diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 5c060b2486..8459211ddb 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -232,12 +232,13 @@ func (p *Pss) Start(srv *p2p.Server) error { } } }() - log.Debug("Started pss", "public key", common.ToHex(crypto.FromECDSAPub(p.PublicKey()))) + log.Info("Started Pss") + log.Info("Loaded EC keys", "pubkey", common.ToHex(crypto.FromECDSAPub(p.PublicKey())), "secp256", common.ToHex(crypto.CompressPubkey(p.PublicKey()))) return nil } func (p *Pss) Stop() error { - log.Info("pss shutting down") + log.Info("Pss shutting down") close(p.quitC) return nil } diff --git a/swarm/swarm.go b/swarm/swarm.go index a895bdfa55..736cd37de8 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -121,12 +121,10 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e backend: backend, privateKey: config.ShiftPrivateKey(), } - log.Debug(fmt.Sprintf("Setting up Swarm service components")) + log.Debug("Setting up Swarm service components") config.HiveParams.Discovery = true - log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store")) - nodeID, err := discover.HexID(config.NodeID) if err != nil { return nil, err @@ -201,8 +199,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e resourceHandler, } - // setup local store - log.Debug(fmt.Sprintf("Set up local storage")) + log.Debug("Setup local storage") self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) @@ -216,11 +213,9 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e } self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey) - // Manifests for Smart Hosting - log.Debug(fmt.Sprintf("-> Web3 virtual server API")) self.sfs = fuse.NewSwarmFS(self.api) - log.Debug("-> Initializing Fuse file system") + log.Debug("Initialized FUSE filesystem") return self, nil } @@ -341,7 +336,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { // update uaddr to correct enode newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String())) - log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) + log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) // set chequebook if self.config.SwapEnabled { ctx := context.Background() // The initial setup has no deadline. @@ -354,18 +349,17 @@ func (self *Swarm) Start(srv *p2p.Server) error { log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set")) } - log.Warn(fmt.Sprintf("Starting Swarm service")) + log.Info("Starting bzz service") err := self.bzz.Start(srv) if err != nil { log.Error("bzz failed", "err", err) return err } - log.Info(fmt.Sprintf("Swarm network started on bzz address: %x", self.bzz.Hive.Overlay.BaseAddr())) + log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", self.bzz.Hive.Overlay.BaseAddr())) if self.ps != nil { self.ps.Start(srv) - log.Info("Pss started") } // start swarm http proxy server @@ -373,13 +367,17 @@ func (self *Swarm) Start(srv *p2p.Server) error { addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port) server := httpapi.NewServer(self.api, self.config.Cors) - go server.ListenAndServe(addr) - } + if self.config.Cors != "" { + log.Debug("Swarm HTTP proxy CORS headers", "allowedOrigins", self.config.Cors) + } - log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) - - if self.config.Cors != "" { - log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.config.Cors)) + log.Debug("Starting Swarm HTTP proxy", "port", self.config.Port) + go func() { + err := server.ListenAndServe(addr) + if err != nil { + log.Error("Could not start Swarm HTTP proxy", "err", err.Error()) + } + }() } self.periodicallyUpdateGauges() From c582667c9b88fabfca2908e1a6ef9b3a71daef37 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 21 Aug 2018 10:34:40 +0200 Subject: [PATCH 047/126] swarm/network: bump bzz protocol version (#17449) --- swarm/network/protocol.go | 2 +- swarm/network/protocol_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 7e7fee8efe..7f7ca5eed5 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -44,7 +44,7 @@ const ( // BzzSpec is the spec of the generic swarm handshake var BzzSpec = &protocols.Spec{ Name: "bzz", - Version: 5, + Version: 6, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ HandshakeMsg{}, diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index b74b72c68b..c052c536a4 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -31,7 +31,7 @@ import ( ) const ( - TestProtocolVersion = 5 + TestProtocolVersion = 6 TestProtocolNetworkID = 3 ) From 76301ca0517ef2d34e6e2d741b0b24cf0ff98642 Mon Sep 17 00:00:00 2001 From: Pierre Neter Date: Tue, 21 Aug 2018 17:36:38 +0700 Subject: [PATCH 048/126] eth: upgradedb subcommand was dropped (#17464) --- eth/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/backend.go b/eth/backend.go index 6549cb8a3d..588b782568 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -138,7 +138,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if !config.SkipBcVersionCheck { bcVersion := rawdb.ReadDatabaseVersion(chainDb) if bcVersion != core.BlockChainVersion && bcVersion != 0 { - return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d). Run geth upgradedb.\n", bcVersion, core.BlockChainVersion) + return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d).\n", bcVersion, core.BlockChainVersion) } rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } From 355fc47d396298bccf93c37bdbba9b9e88864790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 21 Aug 2018 13:58:10 +0200 Subject: [PATCH 049/126] les: fix CHT field in nodeInfo (#17465) --- les/commons.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/les/commons.go b/les/commons.go index 251b7a5833..d8e9412951 100644 --- a/les/commons.go +++ b/les/commons.go @@ -76,18 +76,30 @@ func (c *lesCommons) makeProtocols(versions []uint) []p2p.Protocol { // nodeInfo retrieves some protocol metadata about the running host node. func (c *lesCommons) nodeInfo() interface{} { var cht light.TrustedCheckpoint - sections, _, sectionHead := c.chtIndexer.Sections() - sections2, _, sectionHead2 := c.bloomTrieIndexer.Sections() + sections, _, _ := c.chtIndexer.Sections() + sections2, _, _ := c.bloomTrieIndexer.Sections() + + if !c.protocolManager.lightSync { + // convert to client section size if running in server mode + sections /= light.CHTFrequencyClient / light.CHTFrequencyServer + } + if sections2 < sections { sections = sections2 - sectionHead = sectionHead2 } if sections > 0 { sectionIndex := sections - 1 + sectionHead := c.bloomTrieIndexer.SectionHead(sectionIndex) + var chtRoot common.Hash + if c.protocolManager.lightSync { + chtRoot = light.GetChtRoot(c.chainDb, sectionIndex, sectionHead) + } else { + chtRoot = light.GetChtV2Root(c.chainDb, sectionIndex, sectionHead) + } cht = light.TrustedCheckpoint{ SectionIdx: sectionIndex, SectionHead: sectionHead, - CHTRoot: light.GetChtRoot(c.chainDb, sectionIndex, sectionHead), + CHTRoot: chtRoot, BloomRoot: light.GetBloomTrieRoot(c.chainDb, sectionIndex, sectionHead), } } From 9f036647e4b3e7c3aa8941dc239f85326a5e5ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 21 Aug 2018 14:39:28 +0300 Subject: [PATCH 050/126] consensus/clique, light: light client snapshots on Rinkeby --- consensus/clique/clique.go | 31 ++++++++++++++-------------- consensus/clique/snapshot_test.go | 2 +- light/lightchain.go | 34 ++++++++++++++++++++----------- light/postprocess.go | 25 ++++++++++++----------- params/config.go | 1 + 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 59bb3d40b4..0859447014 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -387,22 +387,23 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo break } } - // If we're at block zero, make a snapshot - if number == 0 { - genesis := chain.GetHeaderByNumber(0) - if err := c.VerifyHeader(chain, genesis, false); err != nil { - return nil, err + // If we're at an checkpoint block, make a snapshot if it's known + if number%c.config.Epoch == 0 { + checkpoint := chain.GetHeaderByNumber(number) + if checkpoint != nil { + hash := checkpoint.Hash() + + signers := make([]common.Address, (len(checkpoint.Extra)-extraVanity-extraSeal)/common.AddressLength) + for i := 0; i < len(signers); i++ { + copy(signers[i][:], checkpoint.Extra[extraVanity+i*common.AddressLength:]) + } + snap = newSnapshot(c.config, c.signatures, number, hash, signers) + if err := snap.store(c.db); err != nil { + return nil, err + } + log.Info("Stored checkpoint snapshot to disk", "number", number, "hash", hash) + break } - signers := make([]common.Address, (len(genesis.Extra)-extraVanity-extraSeal)/common.AddressLength) - for i := 0; i < len(signers); i++ { - copy(signers[i][:], genesis.Extra[extraVanity+i*common.AddressLength:]) - } - snap = newSnapshot(c.config, c.signatures, 0, genesis.Hash(), signers) - if err := snap.store(c.db); err != nil { - return nil, err - } - log.Trace("Stored genesis voting snapshot to disk") - break } // No snapshot for this header, gather the header and move backward var header *types.Header diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 5ac730c9e7..17719884f0 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -84,7 +84,7 @@ func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header { if number == 0 { return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0) } - panic("not supported") + return nil } // Tests that voting is evaluated correctly for various simple and complex scenarios. diff --git a/light/lightchain.go b/light/lightchain.go index b7e629e88b..b5afe1f0e6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -464,22 +464,32 @@ func (self *LightChain) GetHeaderByNumberOdr(ctx context.Context, number uint64) func (self *LightChain) Config() *params.ChainConfig { return self.hc.Config() } func (self *LightChain) SyncCht(ctx context.Context) bool { + // If we don't have a CHT indexer, abort if self.odr.ChtIndexer() == nil { return false } - headNum := self.CurrentHeader().Number.Uint64() - chtCount, _, _ := self.odr.ChtIndexer().Sections() - if headNum+1 < chtCount*CHTFrequencyClient { - num := chtCount*CHTFrequencyClient - 1 - header, err := GetHeaderByNumber(ctx, self.odr, num) - if header != nil && err == nil { - self.mu.Lock() - if self.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() { - self.hc.SetCurrentHeader(header) - } - self.mu.Unlock() - return true + // Ensure the remote CHT head is ahead of us + head := self.CurrentHeader().Number.Uint64() + sections, _, _ := self.odr.ChtIndexer().Sections() + + latest := sections*CHTFrequencyClient - 1 + if clique := self.hc.Config().Clique; clique != nil { + latest -= latest % clique.Epoch // epoch snapshot for clique + } + if head >= latest { + return false + } + // Retrieve the latest useful header and update to it + if header, err := GetHeaderByNumber(ctx, self.odr, latest); header != nil && err == nil { + self.mu.Lock() + defer self.mu.Unlock() + + // Ensure the chain didn't move past the latest block while retrieving it + if self.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() { + log.Info("Updated latest header based on CHT", "number", header.Number, "hash", header.Hash()) + self.hc.SetCurrentHeader(header) } + return true } return false } diff --git a/light/postprocess.go b/light/postprocess.go index 42985aff05..f105d57b55 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -58,28 +58,29 @@ type TrustedCheckpoint struct { SectionHead, CHTRoot, BloomRoot common.Hash } -var ( - mainnetCheckpoint = TrustedCheckpoint{ +// trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to +var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ + params.MainnetGenesisHash: { name: "mainnet", SectionIdx: 187, SectionHead: common.HexToHash("e6baa034efa31562d71ff23676512dec6562c1ad0301e08843b907e81958c696"), CHTRoot: common.HexToHash("28001955219719cf06de1b08648969139d123a9835fc760547a1e4dabdabc15a"), BloomRoot: common.HexToHash("395ca2373fc662720ac6b58b3bbe71f68aa0f38b63b2d3553dd32ff3c51eebc4"), - } - - ropstenCheckpoint = TrustedCheckpoint{ + }, + params.TestnetGenesisHash: { name: "ropsten", SectionIdx: 117, SectionHead: common.HexToHash("9529b38631ae30783f56cbe4c3b9f07575b770ecba4f6e20a274b1e2f40fede1"), CHTRoot: common.HexToHash("6f48e9f101f1fac98e7d74fbbcc4fda138358271ffd974d40d2506f0308bb363"), BloomRoot: common.HexToHash("8242342e66e942c0cd893484e6736b9862ceb88b43ca344bb06a8285ac1b6d64"), - } -) - -// trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to -var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ - params.MainnetGenesisHash: mainnetCheckpoint, - params.TestnetGenesisHash: ropstenCheckpoint, + }, + params.RinkebyGenesisHash: { + name: "rinkeby", + SectionIdx: 85, + SectionHead: common.HexToHash("92cfa67afc4ad8ab0dcbc6fa49efd14b5b19402442e7317e6bc879d85f89d64d"), + CHTRoot: common.HexToHash("2802ec92cd7a54a75bca96afdc666ae7b99e5d96cf8192dcfb09588812f51564"), + BloomRoot: common.HexToHash("ebefeb31a9a42866d8cf2d2477704b4c3d7c20d0e4e9b5aaa77f396e016a1263"), + }, } var ( diff --git a/params/config.go b/params/config.go index b9e9bb8d6e..70a1edead4 100644 --- a/params/config.go +++ b/params/config.go @@ -27,6 +27,7 @@ import ( var ( MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") TestnetGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") ) var ( From 86acdf1a5bb31196adbc92e4923e0398868ce876 Mon Sep 17 00:00:00 2001 From: Jeremy Schlatter Date: Tue, 21 Aug 2018 06:13:03 -0700 Subject: [PATCH 051/126] vendor: update rjeczalik/notify so that it compiles on go1.11 (#17467) --- vendor/github.com/rjeczalik/notify/README.md | 1 + .../github.com/rjeczalik/notify/appveyor.yml | 12 +- .../rjeczalik/notify/debug_debug.go | 4 +- .../rjeczalik/notify/debug_nodebug.go | 4 +- .../rjeczalik/notify/watcher_fsevents_cgo.go | 12 +- .../notify/watcher_fsevents_go1.10.go | 14 ++ .../notify/watcher_fsevents_go1.11.go | 9 ++ .../notify/watcher_notimplemented.go | 15 +++ .../rjeczalik/notify/watcher_readdcw.go | 123 +++++++++++------- .../rjeczalik/notify/watcher_stub.go | 22 +--- .../rjeczalik/notify/watcher_trigger.go | 3 +- vendor/vendor.json | 6 +- 12 files changed, 143 insertions(+), 82 deletions(-) create mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go create mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go create mode 100644 vendor/github.com/rjeczalik/notify/watcher_notimplemented.go diff --git a/vendor/github.com/rjeczalik/notify/README.md b/vendor/github.com/rjeczalik/notify/README.md index 02a5f32908..ad743b2a24 100644 --- a/vendor/github.com/rjeczalik/notify/README.md +++ b/vendor/github.com/rjeczalik/notify/README.md @@ -19,3 +19,4 @@ Filesystem event notification library on steroids. (under active development) - [github.com/cortesi/devd](https://github.com/cortesi/devd) - [github.com/cortesi/modd](https://github.com/cortesi/modd) - [github.com/syncthing/syncthing-inotify](https://github.com/syncthing/syncthing-inotify) +- [github.com/OrlovEvgeny/TinyJPG](https://github.com/OrlovEvgeny/TinyJPG) diff --git a/vendor/github.com/rjeczalik/notify/appveyor.yml b/vendor/github.com/rjeczalik/notify/appveyor.yml index a495bd7c73..a0bdc37a38 100644 --- a/vendor/github.com/rjeczalik/notify/appveyor.yml +++ b/vendor/github.com/rjeczalik/notify/appveyor.yml @@ -7,16 +7,20 @@ clone_folder: c:\projects\src\github.com\rjeczalik\notify environment: PATH: c:\projects\bin;%PATH% GOPATH: c:\projects - NOTIFY_TIMEOUT: 5s + NOTIFY_TIMEOUT: 10s + GOVERSION: 1.10.3 install: + - rmdir c:\go /s /q + - appveyor DownloadFile https://storage.googleapis.com/golang/go%GOVERSION%.windows-amd64.zip + - 7z x go%GOVERSION%.windows-amd64.zip -y -oC:\ > NUL + + - cd %APPVEYOR_BUILD_FOLDER% - go version - - go get -v -t ./... build_script: - - go tool vet -all . - go build ./... - - go test -v -timeout 60s -race ./... + - go test -v -timeout 120s -race ./... test: off diff --git a/vendor/github.com/rjeczalik/notify/debug_debug.go b/vendor/github.com/rjeczalik/notify/debug_debug.go index 6fca891ab3..9d234cedda 100644 --- a/vendor/github.com/rjeczalik/notify/debug_debug.go +++ b/vendor/github.com/rjeczalik/notify/debug_debug.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. @@ -6,4 +6,4 @@ package notify -var debugTag bool = true +var debugTag = true diff --git a/vendor/github.com/rjeczalik/notify/debug_nodebug.go b/vendor/github.com/rjeczalik/notify/debug_nodebug.go index be391a2769..9ebf880d88 100644 --- a/vendor/github.com/rjeczalik/notify/debug_nodebug.go +++ b/vendor/github.com/rjeczalik/notify/debug_nodebug.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. @@ -6,4 +6,4 @@ package notify -var debugTag bool = false +var debugTag = false diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index a2b332a2e0..95ee704444 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -90,6 +90,10 @@ func gostream(_, info uintptr, n C.size_t, paths, flags, ids uintptr) { if n == 0 { return } + fn := streamFuncs.get(info) + if fn == nil { + return + } ev := make([]FSEvent, 0, int(n)) for i := uintptr(0); i < uintptr(n); i++ { switch flags := *(*uint32)(unsafe.Pointer((flags + i*offflag))); { @@ -104,7 +108,7 @@ func gostream(_, info uintptr, n C.size_t, paths, flags, ids uintptr) { } } - streamFuncs.get(info)(ev) + fn(ev) } // StreamFunc is a callback called when stream receives file events. @@ -162,8 +166,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) - path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) + path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go new file mode 100644 index 0000000000..dd2433d20a --- /dev/null +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go @@ -0,0 +1,14 @@ +// Copyright (c) 2018 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue,cgo,!go1.11 + +package notify + +/* + #include +*/ +import "C" + +var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go new file mode 100644 index 0000000000..92b406ce7c --- /dev/null +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go @@ -0,0 +1,9 @@ +// Copyright (c) 2018 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue,go1.11 + +package notify + +const refZero = 0 diff --git a/vendor/github.com/rjeczalik/notify/watcher_notimplemented.go b/vendor/github.com/rjeczalik/notify/watcher_notimplemented.go new file mode 100644 index 0000000000..bb0672fd88 --- /dev/null +++ b/vendor/github.com/rjeczalik/notify/watcher_notimplemented.go @@ -0,0 +1,15 @@ +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build !darwin,!linux,!freebsd,!dragonfly,!netbsd,!openbsd,!windows +// +build !kqueue,!solaris + +package notify + +import "errors" + +// newWatcher stub. +func newWatcher(chan<- EventInfo) watcher { + return watcherStub{errors.New("notify: not implemented")} +} diff --git a/vendor/github.com/rjeczalik/notify/watcher_readdcw.go b/vendor/github.com/rjeczalik/notify/watcher_readdcw.go index 1494fcd799..b69811a690 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_readdcw.go +++ b/vendor/github.com/rjeczalik/notify/watcher_readdcw.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. @@ -22,7 +22,7 @@ import ( const readBufferSize = 4096 // Since all operations which go through the Windows completion routine are done -// asynchronously, filter may set one of the constants belor. They were defined +// asynchronously, filter may set one of the constants below. They were defined // in order to distinguish whether current folder should be re-registered in // ReadDirectoryChangesW function or some control operations need to be executed. const ( @@ -109,8 +109,13 @@ func (g *grip) register(cph syscall.Handle) (err error) { // buffer. Directory changes that occur between calls to this function are added // to the buffer and then, returned with the next call. func (g *grip) readDirChanges() error { + handle := syscall.Handle(atomic.LoadUintptr((*uintptr)(&g.handle))) + if handle == syscall.InvalidHandle { + return nil // Handle was closed. + } + return syscall.ReadDirectoryChanges( - g.handle, + handle, &g.buffer[0], uint32(unsafe.Sizeof(g.buffer)), g.recursive, @@ -220,12 +225,27 @@ func (wd *watched) updateGrip(idx int, cph syscall.Handle, reset bool, // returned from the operating system kernel. func (wd *watched) closeHandle() (err error) { for _, g := range wd.digrip { - if g != nil && g.handle != syscall.InvalidHandle { - switch suberr := syscall.CloseHandle(g.handle); { - case suberr == nil: - g.handle = syscall.InvalidHandle - case err == nil: - err = suberr + if g == nil { + continue + } + + for { + handle := syscall.Handle(atomic.LoadUintptr((*uintptr)(&g.handle))) + if handle == syscall.InvalidHandle { + break // Already closed. + } + + e := syscall.CloseHandle(handle) + if e != nil && err == nil { + err = e + } + + // Set invalid handle even when CloseHandle fails. This will leak + // the handle but, since we can't close it anyway, there won't be + // any difference. + if atomic.CompareAndSwapUintptr((*uintptr)(&g.handle), + (uintptr)(handle), (uintptr)(syscall.InvalidHandle)) { + break } } } @@ -272,50 +292,49 @@ func (r *readdcw) RecursiveWatch(path string, event Event) error { // watch inserts a directory to the group of watched folders. If watched folder // already exists, function tries to rewatch it with new filters(NOT VALID). Moreover, // watch starts the main event loop goroutine when called for the first time. -func (r *readdcw) watch(path string, event Event, recursive bool) (err error) { +func (r *readdcw) watch(path string, event Event, recursive bool) error { if event&^(All|fileNotifyChangeAll) != 0 { return errors.New("notify: unknown event") } + r.Lock() - wd, ok := r.m[path] - r.Unlock() - if !ok { - if err = r.lazyinit(); err != nil { - return - } - r.Lock() - defer r.Unlock() - if wd, ok = r.m[path]; ok { - dbgprint("watch: exists already") - return - } - if wd, err = newWatched(r.cph, uint32(event), recursive, path); err != nil { - return - } - r.m[path] = wd - dbgprint("watch: new watch added") - } else { - dbgprint("watch: exists already") + defer r.Unlock() + + if wd, ok := r.m[path]; ok { + dbgprint("watch: already exists") + wd.filter &^= stateUnwatch + return nil } + + if err := r.lazyinit(); err != nil { + return err + } + + wd, err := newWatched(r.cph, uint32(event), recursive, path) + if err != nil { + return err + } + + r.m[path] = wd + dbgprint("watch: new watch added") + return nil } -// lazyinit creates an I/O completion port and starts the main event processing -// loop. This method uses Double-Checked Locking optimization. +// lazyinit creates an I/O completion port and starts the main event loop. func (r *readdcw) lazyinit() (err error) { invalid := uintptr(syscall.InvalidHandle) + if atomic.LoadUintptr((*uintptr)(&r.cph)) == invalid { - r.Lock() - defer r.Unlock() - if atomic.LoadUintptr((*uintptr)(&r.cph)) == invalid { - cph := syscall.InvalidHandle - if cph, err = syscall.CreateIoCompletionPort(cph, 0, 0, 0); err != nil { - return - } - r.cph, r.start = cph, true - go r.loop() + cph := syscall.InvalidHandle + if cph, err = syscall.CreateIoCompletionPort(cph, 0, 0, 0); err != nil { + return } + + r.cph, r.start = cph, true + go r.loop() } + return } @@ -364,6 +383,7 @@ func (r *readdcw) loopstate(overEx *overlappedEx) { overEx.parent.parent.recreate(r.cph) case stateUnwatch: dbgprint("loopstate unwatch") + overEx.parent.parent.closeHandle() delete(r.m, syscall.UTF16ToString(overEx.parent.pathw)) case stateCPClose: default: @@ -495,27 +515,30 @@ func (r *readdcw) RecursiveUnwatch(path string) error { // TODO : pknap func (r *readdcw) unwatch(path string) (err error) { var wd *watched + r.Lock() defer r.Unlock() if wd, err = r.nonStateWatchedLocked(path); err != nil { return } + wd.filter |= stateUnwatch - if err = wd.closeHandle(); err != nil { - wd.filter &^= stateUnwatch - return - } + dbgprint("unwatch: set unwatch state") + if _, attrErr := syscall.GetFileAttributes(&wd.pathw[0]); attrErr != nil { for _, g := range wd.digrip { - if g != nil { - dbgprint("unwatch: posting") - if err = syscall.PostQueuedCompletionStatus(r.cph, 0, 0, (*syscall.Overlapped)(unsafe.Pointer(g.ovlapped))); err != nil { - wd.filter &^= stateUnwatch - return - } + if g == nil { + continue + } + + dbgprint("unwatch: posting") + if err = syscall.PostQueuedCompletionStatus(r.cph, 0, 0, (*syscall.Overlapped)(unsafe.Pointer(g.ovlapped))); err != nil { + wd.filter &^= stateUnwatch + return } } } + return } diff --git a/vendor/github.com/rjeczalik/notify/watcher_stub.go b/vendor/github.com/rjeczalik/notify/watcher_stub.go index 68b9c135b0..9b284ddc85 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_stub.go +++ b/vendor/github.com/rjeczalik/notify/watcher_stub.go @@ -1,23 +1,13 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. -// +build !darwin,!linux,!freebsd,!dragonfly,!netbsd,!openbsd,!windows -// +build !kqueue,!solaris - package notify -import "errors" - -type stub struct{ error } - -// newWatcher stub. -func newWatcher(chan<- EventInfo) watcher { - return stub{errors.New("notify: not implemented")} -} +type watcherStub struct{ error } // Following methods implement notify.watcher interface. -func (s stub) Watch(string, Event) error { return s } -func (s stub) Rewatch(string, Event, Event) error { return s } -func (s stub) Unwatch(string) (err error) { return s } -func (s stub) Close() error { return s } +func (s watcherStub) Watch(string, Event) error { return s } +func (s watcherStub) Rewatch(string, Event, Event) error { return s } +func (s watcherStub) Unwatch(string) (err error) { return s } +func (s watcherStub) Close() error { return s } diff --git a/vendor/github.com/rjeczalik/notify/watcher_trigger.go b/vendor/github.com/rjeczalik/notify/watcher_trigger.go index 78151f909b..1ebe04829e 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_trigger.go +++ b/vendor/github.com/rjeczalik/notify/watcher_trigger.go @@ -106,7 +106,8 @@ func newWatcher(c chan<- EventInfo) watcher { } t.t = newTrigger(t.pthLkp) if err := t.t.Init(); err != nil { - panic(err) + t.Close() + return watcherStub{fmt.Errorf("failed setting up watcher: %v", err)} } go t.monitor() return t diff --git a/vendor/vendor.json b/vendor/vendor.json index 78886c7bb3..0bfc1ca0c1 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -370,10 +370,10 @@ "revisionTime": "2017-08-14T17:01:13Z" }, { - "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", + "checksumSHA1": "D8AVDI39CJ+jvw0HOotYU2gz54c=", "path": "github.com/rjeczalik/notify", - "revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9", - "revisionTime": "2018-02-03T14:01:15Z" + "revision": "4e54e7fd043e865c50bda93359fb78813a8d165b", + "revisionTime": "2018-08-08T20:39:25Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", From a063fe9b2defdb595068483b4c5df41f1a3a4860 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 21 Aug 2018 22:33:04 +0800 Subject: [PATCH 052/126] miner: fix uncle iteration logic (#17469) --- miner/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 2f76f2a92c..23cfaf2256 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -308,7 +308,7 @@ func (w *worker) mainLoop() { return false } uncles = append(uncles, uncle.Header()) - return true + return false }) w.commit(uncles, nil, true, start) } @@ -522,7 +522,7 @@ func (w *worker) updateSnapshot() { return false } uncles = append(uncles, uncle.Header()) - return true + return false }) w.snapshotBlock = types.NewBlock( From 522cfc68ff496aee4205add982db049dc3092024 Mon Sep 17 00:00:00 2001 From: Geon Kim Date: Wed, 22 Aug 2018 03:13:33 +0900 Subject: [PATCH 053/126] swarm: fix typos (#17473) --- swarm/network/simulation/simulation_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/swarm/network/simulation/simulation_test.go b/swarm/network/simulation/simulation_test.go index 8576732c9f..eed09bf508 100644 --- a/swarm/network/simulation/simulation_test.go +++ b/swarm/network/simulation/simulation_test.go @@ -63,7 +63,7 @@ func TestRun(t *testing.T) { } }) - t.Run("cancelation", func(t *testing.T) { + t.Run("cancellation", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() @@ -164,7 +164,7 @@ func TestDone(t *testing.T) { select { case <-time.After(timeout): - t.Error("done channel closing timmed out") + t.Error("done channel closing timed out") case <-sim.Done(): if d := time.Since(start); d < sleep { t.Errorf("done channel closed sooner then expected: %s", d) @@ -172,7 +172,7 @@ func TestDone(t *testing.T) { } } -// a helper map for usual services that do not do anyting +// a helper map for usual services that do not do anything var noopServiceFuncMap = map[string]ServiceFunc{ "noop": noopServiceFunc, } From b2c644ffb5c283a171ddf3889693673939917541 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 22 Aug 2018 03:56:54 +0800 Subject: [PATCH 054/126] cmd, eth, miner: make recommit configurable (#17444) * cmd, eth, miner: make recommit configurable * cmd, eth, les, miner: polish a bit * miner: filter duplicate sealing work * cmd: remove uncessary conversion * miner: avoid microptimization in favor of cleaner code --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 22 ++-- eth/api.go | 6 ++ eth/backend.go | 8 +- eth/config.go | 15 +-- eth/gen_config.go | 53 ++++++++-- internal/web3ext/web3ext.go | 5 + les/backend.go | 2 +- miner/miner.go | 10 +- miner/worker.go | 202 +++++++++++++++++++++++++++++------- miner/worker_test.go | 106 ++++++++++++++++++- 12 files changed, 360 insertions(+), 71 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a063860513..2e87bb8200 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -106,6 +106,7 @@ var ( utils.MinerLegacyEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerLegacyExtraDataFlag, + utils.MinerRecommitIntervalFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 9e18f70472..674c5d9015 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -190,6 +190,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerGasTargetFlag, utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, + utils.MinerRecommitIntervalFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e3a8cc2eac..7317655836 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -335,12 +335,12 @@ var ( MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", Usage: "Minimal gas price for mining a transactions", - Value: eth.DefaultConfig.GasPrice, + Value: eth.DefaultConfig.MinerGasPrice, } MinerLegacyGasPriceFlag = BigFlag{ Name: "gasprice", Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)", - Value: eth.DefaultConfig.GasPrice, + Value: eth.DefaultConfig.MinerGasPrice, } MinerEtherbaseFlag = cli.StringFlag{ Name: "miner.etherbase", @@ -360,6 +360,11 @@ var ( Name: "extradata", Usage: "Block extra data set by the miner (default = client version, deprecated, use --miner.extradata)", } + MinerRecommitIntervalFlag = cli.DurationFlag{ + Name: "miner.recommit", + Usage: "Time interval to recreate the block being mined.", + Value: 3 * time.Second, + } // Account settings UnlockedAccountFlag = cli.StringFlag{ Name: "unlock", @@ -1124,16 +1129,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } if ctx.GlobalIsSet(MinerLegacyExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name)) + cfg.MinerExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name)) } if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) + cfg.MinerExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) + cfg.MinerGasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) } if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) + cfg.MinerGasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) + } + if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { + cfg.MinerRecommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode @@ -1176,7 +1184,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address) if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { - cfg.GasPrice = big.NewInt(1) + cfg.MinerGasPrice = big.NewInt(1) } } // TODO(fjl): move trie cache generations into config diff --git a/eth/api.go b/eth/api.go index c1fbcb6d40..4b0ba8edbd 100644 --- a/eth/api.go +++ b/eth/api.go @@ -25,6 +25,7 @@ import ( "math/big" "os" "strings" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -160,6 +161,11 @@ func (api *PrivateMinerAPI) SetEtherbase(etherbase common.Address) bool { return true } +// SetRecommitInterval updates the interval for miner sealing work recommitting. +func (api *PrivateMinerAPI) SetRecommitInterval(interval int) { + api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) +} + // GetHashrate returns the current hashrate of the miner. func (api *PrivateMinerAPI) GetHashrate() uint64 { return api.e.miner.HashRate() diff --git a/eth/backend.go b/eth/backend.go index 588b782568..648175acfd 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -127,7 +127,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb), shutdownChan: make(chan bool), networkID: config.NetworkId, - gasPrice: config.GasPrice, + gasPrice: config.MinerGasPrice, etherbase: config.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, bloomConfirms), @@ -167,13 +167,13 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) - eth.miner.SetExtra(makeExtraData(config.ExtraData)) + eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine, config.MinerRecommit) + eth.miner.SetExtra(makeExtraData(config.MinerExtraData)) eth.APIBackend = &EthAPIBackend{eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { - gpoParams.Default = config.GasPrice + gpoParams.Default = config.MinerGasPrice } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) diff --git a/eth/config.go b/eth/config.go index 0c82f29232..cbd02416be 100644 --- a/eth/config.go +++ b/eth/config.go @@ -48,7 +48,7 @@ var DefaultConfig = Config{ DatabaseCache: 768, TrieCache: 256, TrieTimeout: 60 * time.Minute, - GasPrice: big.NewInt(18 * params.Shannon), + MinerGasPrice: big.NewInt(18 * params.Shannon), TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ @@ -95,11 +95,12 @@ type Config struct { TrieTimeout time.Duration // Mining-related options - Etherbase common.Address `toml:",omitempty"` - MinerThreads int `toml:",omitempty"` - MinerNotify []string `toml:",omitempty"` - ExtraData []byte `toml:",omitempty"` - GasPrice *big.Int + Etherbase common.Address `toml:",omitempty"` + MinerThreads int `toml:",omitempty"` + MinerNotify []string `toml:",omitempty"` + MinerExtraData []byte `toml:",omitempty"` + MinerGasPrice *big.Int + MinerRecommit time.Duration // Ethash options Ethash ethash.Config @@ -118,5 +119,5 @@ type Config struct { } type configMarshaling struct { - ExtraData hexutil.Bytes + MinerExtraData hexutil.Bytes } diff --git a/eth/gen_config.go b/eth/gen_config.go index 4f2e82d941..62556be7e1 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -4,6 +4,7 @@ package eth import ( "math/big" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -15,20 +16,26 @@ import ( var _ = (*configMarshaling)(nil) +// MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode downloader.SyncMode + NoPruning bool LightServ int `toml:",omitempty"` LightPeers int `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int + TrieCache int + TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` - ExtraData hexutil.Bytes `toml:",omitempty"` - GasPrice *big.Int + MinerNotify []string `toml:",omitempty"` + MinerExtraData hexutil.Bytes `toml:",omitempty"` + MinerGasPrice *big.Int + MinerRecommit time.Duration Ethash ethash.Config TxPool core.TxPoolConfig GPO gasprice.Config @@ -39,15 +46,20 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Genesis = c.Genesis enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode + enc.NoPruning = c.NoPruning enc.LightServ = c.LightServ enc.LightPeers = c.LightPeers enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache + enc.TrieCache = c.TrieCache + enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase enc.MinerThreads = c.MinerThreads - enc.ExtraData = c.ExtraData - enc.GasPrice = c.GasPrice + enc.MinerNotify = c.MinerNotify + enc.MinerExtraData = c.MinerExtraData + enc.MinerGasPrice = c.MinerGasPrice + enc.MinerRecommit = c.MinerRecommit enc.Ethash = c.Ethash enc.TxPool = c.TxPool enc.GPO = c.GPO @@ -56,20 +68,26 @@ func (c Config) MarshalTOML() (interface{}, error) { return &enc, nil } +// UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *downloader.SyncMode + NoPruning *bool LightServ *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int + TrieCache *int + TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` MinerThreads *int `toml:",omitempty"` - ExtraData *hexutil.Bytes `toml:",omitempty"` - GasPrice *big.Int + MinerNotify []string `toml:",omitempty"` + MinerExtraData *hexutil.Bytes `toml:",omitempty"` + MinerGasPrice *big.Int + MinerRecommit *time.Duration Ethash *ethash.Config TxPool *core.TxPoolConfig GPO *gasprice.Config @@ -89,6 +107,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SyncMode != nil { c.SyncMode = *dec.SyncMode } + if dec.NoPruning != nil { + c.NoPruning = *dec.NoPruning + } if dec.LightServ != nil { c.LightServ = *dec.LightServ } @@ -104,17 +125,29 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseCache != nil { c.DatabaseCache = *dec.DatabaseCache } + if dec.TrieCache != nil { + c.TrieCache = *dec.TrieCache + } + if dec.TrieTimeout != nil { + c.TrieTimeout = *dec.TrieTimeout + } if dec.Etherbase != nil { c.Etherbase = *dec.Etherbase } if dec.MinerThreads != nil { c.MinerThreads = *dec.MinerThreads } - if dec.ExtraData != nil { - c.ExtraData = *dec.ExtraData + if dec.MinerNotify != nil { + c.MinerNotify = dec.MinerNotify } - if dec.GasPrice != nil { - c.GasPrice = dec.GasPrice + if dec.MinerExtraData != nil { + c.MinerExtraData = *dec.MinerExtraData + } + if dec.MinerGasPrice != nil { + c.MinerGasPrice = dec.MinerGasPrice + } + if dec.MinerRecommit != nil { + c.MinerRecommit = *dec.MinerRecommit } if dec.Ethash != nil { c.Ethash = *dec.Ethash diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 000e3728de..f4eb47a12a 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -519,6 +519,11 @@ web3._extend({ params: 1, inputFormatter: [web3._extend.utils.fromDecimal] }), + new web3._extend.Method({ + name: 'setRecommitInterval', + call: 'miner_setRecommitInterval', + params: 1, + }), new web3._extend.Method({ name: 'getHashrate', call: 'miner_getHashrate' diff --git a/les/backend.go b/les/backend.go index d26c1470fe..00025ba634 100644 --- a/les/backend.go +++ b/les/backend.go @@ -141,7 +141,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.ApiBackend = &LesApiBackend{leth, nil} gpoParams := config.GPO if gpoParams.Default == nil { - gpoParams.Default = config.GasPrice + gpoParams.Default = config.MinerGasPrice } leth.ApiBackend.gpo = gasprice.NewOracle(leth.ApiBackend, gpoParams) return leth, nil diff --git a/miner/miner.go b/miner/miner.go index e350e456e9..c5a0c9d62a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -20,6 +20,7 @@ package miner import ( "fmt" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -51,13 +52,13 @@ type Miner struct { shouldStart int32 // should start indicates whether we should start after sync } -func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine) *Miner { +func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, recommit time.Duration) *Miner { miner := &Miner{ eth: eth, mux: mux, engine: engine, exitCh: make(chan struct{}), - worker: newWorker(config, engine, eth, mux), + worker: newWorker(config, engine, eth, mux, recommit), canStart: 1, } go miner.update() @@ -144,6 +145,11 @@ func (self *Miner) SetExtra(extra []byte) error { return nil } +// SetRecommitInterval sets the interval for sealing work resubmitting. +func (self *Miner) SetRecommitInterval(interval time.Duration) { + self.worker.setRecommitInterval(interval) +} + // Pending returns the currently pending block and associated state. func (self *Miner) Pending() (*types.Block, *state.StateDB) { return self.worker.pending() diff --git a/miner/worker.go b/miner/worker.go index 23cfaf2256..c299ff9dc1 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -51,12 +51,27 @@ const ( // chainSideChanSize is the size of channel listening to ChainSideEvent. chainSideChanSize = 10 + // resubmitAdjustChanSize is the size of resubmitting interval adjustment channel. + resubmitAdjustChanSize = 10 + // miningLogAtDepth is the number of confirmations before logging successful mining. miningLogAtDepth = 5 - // blockRecommitInterval is the time interval to recreate the mining block with + // minRecommitInterval is the minimal time interval to recreate the mining block with // any newly arrived transactions. - blockRecommitInterval = 3 * time.Second + minRecommitInterval = 1 * time.Second + + // maxRecommitInterval is the maximum time interval to recreate the mining block with + // any newly arrived transactions. + maxRecommitInterval = 15 * time.Second + + // intervalAdjustRatio is the impact a single interval adjustment has on sealing work + // resubmitting interval. + intervalAdjustRatio = 0.1 + + // intervalAdjustBias is applied during the new resubmit interval calculation in favor of + // increasing upper limit or decreasing lower limit so that the limit can be reachable. + intervalAdjustBias = 200 * 1000.0 * 1000.0 ) // environment is the worker's current environment and holds all of the current state information. @@ -89,11 +104,18 @@ const ( commitInterruptResubmit ) +// newWorkReq represents a request for new sealing work submitting with relative interrupt notifier. type newWorkReq struct { interrupt *int32 noempty bool } +// intervalAdjust represents a resubmitting interval adjustment. +type intervalAdjust struct { + ratio float64 + inc bool +} + // worker is the main object which takes care of submitting new work to consensus engine // and gathering the sealing result. type worker struct { @@ -112,11 +134,13 @@ type worker struct { chainSideSub event.Subscription // Channels - newWorkCh chan *newWorkReq - taskCh chan *task - resultCh chan *task - startCh chan struct{} - exitCh chan struct{} + newWorkCh chan *newWorkReq + taskCh chan *task + resultCh chan *task + startCh chan struct{} + exitCh chan struct{} + resubmitIntervalCh chan time.Duration + resubmitAdjustCh chan *intervalAdjust current *environment // An environment for current running cycle. possibleUncles map[common.Hash]*types.Block // A set of side blocks as the possible uncle blocks. @@ -132,30 +156,34 @@ type worker struct { // atomic status counters running int32 // The indicator whether the consensus engine is running or not. + newTxs int32 // New arrival transaction count since last sealing work submitting. // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task - skipSealHook func(*task) bool // Method to decide whether skipping the sealing. - fullTaskHook func() // Method to call before pushing the full sealing task + newTaskHook func(*task) // Method to call upon receiving a new sealing task. + skipSealHook func(*task) bool // Method to decide whether skipping the sealing. + fullTaskHook func() // Method to call before pushing the full sealing task. + resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, recommit time.Duration) *worker { worker := &worker{ - config: config, - engine: engine, - eth: eth, - mux: mux, - chain: eth.BlockChain(), - possibleUncles: make(map[common.Hash]*types.Block), - unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), - newWorkCh: make(chan *newWorkReq), - taskCh: make(chan *task), - resultCh: make(chan *task, resultQueueSize), - exitCh: make(chan struct{}), - startCh: make(chan struct{}, 1), + config: config, + engine: engine, + eth: eth, + mux: mux, + chain: eth.BlockChain(), + possibleUncles: make(map[common.Hash]*types.Block), + unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), + txsCh: make(chan core.NewTxsEvent, txChanSize), + chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), + chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), + newWorkCh: make(chan *newWorkReq), + taskCh: make(chan *task), + resultCh: make(chan *task, resultQueueSize), + exitCh: make(chan struct{}), + startCh: make(chan struct{}, 1), + resubmitIntervalCh: make(chan time.Duration), + resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), } // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) @@ -163,8 +191,14 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) + // Sanitize recommit interval if the user-specified one is too short. + if recommit < minRecommitInterval { + log.Warn("Sanitizing miner recommit interval", "provided", recommit, "updated", minRecommitInterval) + recommit = minRecommitInterval + } + go worker.mainLoop() - go worker.newWorkLoop() + go worker.newWorkLoop(recommit) go worker.resultLoop() go worker.taskLoop() @@ -188,6 +222,11 @@ func (w *worker) setExtra(extra []byte) { w.extra = extra } +// setRecommitInterval updates the interval for miner sealing work recommitting. +func (w *worker) setRecommitInterval(interval time.Duration) { + w.resubmitIntervalCh <- interval +} + // pending returns the pending state and corresponding block. func (w *worker) pending() (*types.Block, *state.StateDB) { // return a snapshot to avoid contention on currentMu mutex @@ -238,35 +277,94 @@ func (w *worker) close() { } // newWorkLoop is a standalone goroutine to submit new mining work upon received events. -func (w *worker) newWorkLoop() { - var interrupt *int32 +func (w *worker) newWorkLoop(recommit time.Duration) { + var ( + interrupt *int32 + minRecommit = recommit // minimal resubmit interval specified by user. + ) timer := time.NewTimer(0) <-timer.C // discard the initial tick - // recommit aborts in-flight transaction execution with given signal and resubmits a new one. - recommit := func(noempty bool, s int32) { + // commit aborts in-flight transaction execution with given signal and resubmits a new one. + commit := func(noempty bool, s int32) { if interrupt != nil { atomic.StoreInt32(interrupt, s) } interrupt = new(int32) w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty} - timer.Reset(blockRecommitInterval) + timer.Reset(recommit) + atomic.StoreInt32(&w.newTxs, 0) + } + // recalcRecommit recalculates the resubmitting interval upon feedback. + recalcRecommit := func(target float64, inc bool) { + var ( + prev = float64(recommit.Nanoseconds()) + next float64 + ) + if inc { + next = prev*(1-intervalAdjustRatio) + intervalAdjustRatio*(target+intervalAdjustBias) + // Recap if interval is larger than the maximum time interval + if next > float64(maxRecommitInterval.Nanoseconds()) { + next = float64(maxRecommitInterval.Nanoseconds()) + } + } else { + next = prev*(1-intervalAdjustRatio) + intervalAdjustRatio*(target-intervalAdjustBias) + // Recap if interval is less than the user specified minimum + if next < float64(minRecommit.Nanoseconds()) { + next = float64(minRecommit.Nanoseconds()) + } + } + recommit = time.Duration(int64(next)) } for { select { case <-w.startCh: - recommit(false, commitInterruptNewHead) + commit(false, commitInterruptNewHead) case <-w.chainHeadCh: - recommit(false, commitInterruptNewHead) + commit(false, commitInterruptNewHead) case <-timer.C: // If mining is running resubmit a new work cycle periodically to pull in // higher priced transactions. Disable this overhead for pending blocks. if w.isRunning() && (w.config.Clique == nil || w.config.Clique.Period > 0) { - recommit(true, commitInterruptResubmit) + // Short circuit if no new transaction arrives. + if atomic.LoadInt32(&w.newTxs) == 0 { + timer.Reset(recommit) + continue + } + commit(true, commitInterruptResubmit) + } + + case interval := <-w.resubmitIntervalCh: + // Adjust resubmit interval explicitly by user. + if interval < minRecommitInterval { + log.Warn("Sanitizing miner recommit interval", "provided", interval, "updated", minRecommitInterval) + interval = minRecommitInterval + } + log.Info("Miner recommit interval update", "from", minRecommit, "to", interval) + minRecommit, recommit = interval, interval + + if w.resubmitHook != nil { + w.resubmitHook(minRecommit, recommit) + } + + case adjust := <-w.resubmitAdjustCh: + // Adjust resubmit interval by feedback. + if adjust.inc { + before := recommit + recalcRecommit(float64(recommit.Nanoseconds())/adjust.ratio, true) + log.Trace("Increase miner recommit interval", "from", before, "to", recommit) + } else { + before := recommit + recalcRecommit(float64(minRecommit.Nanoseconds()), false) + log.Trace("Decrease miner recommit interval", "from", before, "to", recommit) + } + + if w.resubmitHook != nil { + w.resubmitHook(minRecommit, recommit) } case <-w.exitCh: @@ -339,6 +437,7 @@ func (w *worker) mainLoop() { w.commitNewWork(nil, false) } } + atomic.AddInt32(&w.newTxs, int32(len(ev.Txs))) // System stopped case <-w.exitCh: @@ -383,7 +482,10 @@ func (w *worker) seal(t *task, stop <-chan struct{}) { // taskLoop is a standalone goroutine to fetch sealing task from the generator and // push them to consensus engine. func (w *worker) taskLoop() { - var stopCh chan struct{} + var ( + stopCh chan struct{} + prev common.Hash + ) // interrupt aborts the in-flight sealing task. interrupt := func() { @@ -398,8 +500,13 @@ func (w *worker) taskLoop() { if w.newTaskHook != nil { w.newTaskHook(task) } + // Reject duplicate sealing work due to resubmitting. + if task.block.HashNoNonce() == prev { + continue + } interrupt() stopCh = make(chan struct{}) + prev = task.block.HashNoNonce() go w.seal(task, stopCh) case <-w.exitCh: interrupt() @@ -414,11 +521,15 @@ func (w *worker) resultLoop() { for { select { case result := <-w.resultCh: + // Short circuit when receiving empty result. if result == nil { continue } + // Short circuit when receiving duplicate result caused by resubmitting. block := result.block - + if w.chain.HasBlock(block.Hash(), block.NumberU64()) { + continue + } // Update the block hash in all logs since it is now available and not when the // receipt/log of individual transactions were created. for _, r := range result.receipts { @@ -568,8 +679,18 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin // (3) worker recreate the mining block with any newly arrived transactions, the interrupt signal is 2. // For the first two cases, the semi-finished work will be discarded. // For the third case, the semi-finished work will be submitted to the consensus engine. - // TODO(rjl493456442) give feedback to newWorkLoop to adjust resubmit interval if it is too short. if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { + // Notify resubmit loop to increase resubmitting interval due to too frequent commits. + if atomic.LoadInt32(interrupt) == commitInterruptResubmit { + ratio := float64(w.current.header.GasLimit-w.current.gasPool.Gas()) / float64(w.current.header.GasLimit) + if ratio < 0.1 { + ratio = 0.1 + } + w.resubmitAdjustCh <- &intervalAdjust{ + ratio: ratio, + inc: true, + } + } return atomic.LoadInt32(interrupt) == commitInterruptNewHead } // If we don't have enough gas for any further transactions then we're done @@ -644,6 +765,11 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin } go w.mux.Post(core.PendingLogsEvent{Logs: cpy}) } + // Notify resubmit loop to decrease resubmitting interval if current interval is larger + // than the user-specified one. + if interrupt != nil { + w.resubmitAdjustCh <- &intervalAdjust{inc: false} + } return false } diff --git a/miner/worker_test.go b/miner/worker_test.go index 34bb7f5f33..16708c18c6 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -119,7 +119,7 @@ func (b *testWorkerBackend) PostChainEvents(events []interface{}) { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine) backend.txPool.AddLocals(pendingTxs) - w := newWorker(chainConfig, engine, backend, new(event.TypeMux)) + w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second) w.setEtherbase(testBankAddress) return w, backend } @@ -327,7 +327,7 @@ func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, en } } b.txPool.AddLocals(newTxs) - time.Sleep(3 * time.Second) + time.Sleep(time.Second) select { case <-taskCh: @@ -335,3 +335,105 @@ func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, en t.Error("new task timeout") } } + +func TestAdjustIntervalEthash(t *testing.T) { + testAdjustInterval(t, ethashChainConfig, ethash.NewFaker()) +} + +func TestAdjustIntervalClique(t *testing.T) { + testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, _ := newTestWorker(t, chainConfig, engine) + defer w.close() + + w.skipSealHook = func(task *task) bool { + return true + } + w.fullTaskHook = func() { + time.Sleep(100 * time.Millisecond) + } + var ( + progress = make(chan struct{}, 10) + result = make([]float64, 0, 10) + index = 0 + start = false + ) + w.resubmitHook = func(minInterval time.Duration, recommitInterval time.Duration) { + // Short circuit if interval checking hasn't started. + if !start { + return + } + var wantMinInterval, wantRecommitInterval time.Duration + + switch index { + case 0: + wantMinInterval, wantRecommitInterval = 3*time.Second, 3*time.Second + case 1: + origin := float64(3 * time.Second.Nanoseconds()) + estimate := origin*(1-intervalAdjustRatio) + intervalAdjustRatio*(origin/0.8+intervalAdjustBias) + wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(int(estimate))*time.Nanosecond + case 2: + estimate := result[index-1] + min := float64(3 * time.Second.Nanoseconds()) + estimate = estimate*(1-intervalAdjustRatio) + intervalAdjustRatio*(min-intervalAdjustBias) + wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(int(estimate))*time.Nanosecond + case 3: + wantMinInterval, wantRecommitInterval = time.Second, time.Second + } + + // Check interval + if minInterval != wantMinInterval { + t.Errorf("resubmit min interval mismatch want %s has %s", wantMinInterval, minInterval) + } + if recommitInterval != wantRecommitInterval { + t.Errorf("resubmit interval mismatch want %s has %s", wantRecommitInterval, recommitInterval) + } + result = append(result, float64(recommitInterval.Nanoseconds())) + index += 1 + progress <- struct{}{} + } + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + + time.Sleep(time.Second) + + start = true + w.setRecommitInterval(3 * time.Second) + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } + + w.resubmitAdjustCh <- &intervalAdjust{inc: true, ratio: 0.8} + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } + + w.resubmitAdjustCh <- &intervalAdjust{inc: false} + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } + + w.setRecommitInterval(500 * time.Millisecond) + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } +} From e0d0e64ce22111a2d5492fe6f6d6a0023477e51f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 21 Aug 2018 20:30:06 +0300 Subject: [PATCH 055/126] cmd, core, miner: add --txpool.locals and priority mining --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 14 ++++++++++++++ core/tx_pool.go | 39 +++++++++++++++++++++++++++++++++++---- miner/worker.go | 23 +++++++++++++++++++---- 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2e87bb8200..4b86382bdb 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -72,6 +72,7 @@ var ( utils.EthashDatasetDirFlag, utils.EthashDatasetsInMemoryFlag, utils.EthashDatasetsOnDiskFlag, + utils.TxPoolLocalsFlag, utils.TxPoolNoLocalsFlag, utils.TxPoolJournalFlag, utils.TxPoolRejournalFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 674c5d9015..1e27d0ae8d 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -114,6 +114,7 @@ var AppHelpFlagGroups = []flagGroup{ { Name: "TRANSACTION POOL", Flags: []cli.Flag{ + utils.TxPoolLocalsFlag, utils.TxPoolNoLocalsFlag, utils.TxPoolJournalFlag, utils.TxPoolRejournalFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7317655836..cfca7b4ab4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -233,6 +233,10 @@ var ( Value: eth.DefaultConfig.Ethash.DatasetsOnDisk, } // Transaction pool settings + TxPoolLocalsFlag = cli.StringFlag{ + Name: "txpool.locals", + Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", + } TxPoolNoLocalsFlag = cli.BoolFlag{ Name: "txpool.nolocals", Usage: "Disables price exemptions for locally submitted transactions", @@ -977,6 +981,16 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config) { } func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { + if ctx.GlobalIsSet(TxPoolLocalsFlag.Name) { + locals := strings.Split(ctx.GlobalString(TxPoolLocalsFlag.Name), ",") + for _, account := range locals { + if trimmed := strings.TrimSpace(account); !common.IsHexAddress(trimmed) { + Fatalf("Invalid account in --txpool.locals: %s", trimmed) + } else { + cfg.Locals = append(cfg.Locals, common.HexToAddress(account)) + } + } + } if ctx.GlobalIsSet(TxPoolNoLocalsFlag.Name) { cfg.NoLocals = ctx.GlobalBool(TxPoolNoLocalsFlag.Name) } diff --git a/core/tx_pool.go b/core/tx_pool.go index 7007f85ddf..46ae2759bc 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -123,9 +123,10 @@ type blockChain interface { // TxPoolConfig are the configuration parameters of the transaction pool. type TxPoolConfig struct { - NoLocals bool // Whether local transaction handling should be disabled - Journal string // Journal of local transactions to survive node restarts - Rejournal time.Duration // Time interval to regenerate the local transaction journal + Locals []common.Address // Addresses that should be treated by default as local + NoLocals bool // Whether local transaction handling should be disabled + Journal string // Journal of local transactions to survive node restarts + Rejournal time.Duration // Time interval to regenerate the local transaction journal PriceLimit uint64 // Minimum gas price to enforce for acceptance into the pool PriceBump uint64 // Minimum price bump percentage to replace an already existing transaction (nonce) @@ -231,6 +232,10 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) + for _, addr := range config.Locals { + log.Info("Setting new local account", "address", addr) + pool.locals.add(addr) + } pool.priced = newTxPricedList(pool.all) pool.reset(nil, chain.CurrentBlock().Header()) @@ -534,6 +539,14 @@ func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) { return pending, nil } +// Locals retrieves the accounts currently considered local by the pool. +func (pool *TxPool) Locals() []common.Address { + pool.mu.Lock() + defer pool.mu.Unlock() + + return pool.locals.flatten() +} + // local retrieves all currently known local transactions, groupped by origin // account and sorted by nonce. The returned transaction set is a copy and can be // freely modified by calling code. @@ -665,7 +678,10 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { } // Mark local addresses and journal local transactions if local { - pool.locals.add(from) + if !pool.locals.contains(from) { + log.Info("Setting new local account", "address", from) + pool.locals.add(from) + } } pool.journalTx(from, tx) @@ -1138,6 +1154,7 @@ func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } type accountSet struct { accounts map[common.Address]struct{} signer types.Signer + cache *[]common.Address } // newAccountSet creates a new address set with an associated signer for sender @@ -1167,6 +1184,20 @@ func (as *accountSet) containsTx(tx *types.Transaction) bool { // add inserts a new address into the set to track. func (as *accountSet) add(addr common.Address) { as.accounts[addr] = struct{}{} + as.cache = nil +} + +// flatten returns the list of addresses within this set, also caching it for later +// reuse. The returned slice should not be changed! +func (as *accountSet) flatten() []common.Address { + if as.cache == nil { + accounts := make([]common.Address, 0, len(as.accounts)) + for account := range as.accounts { + accounts = append(accounts, account) + } + as.cache = &accounts + } + return *as.cache } // txLookup is used internally by TxPool to track transactions while allowing lookup without diff --git a/miner/worker.go b/miner/worker.go index c299ff9dc1..8c3337ba45 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -877,11 +877,26 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { w.updateSnapshot() return } - txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) - if w.commitTransactions(txs, w.coinbase, interrupt) { - return + // Split the pending transactions into locals and remotes + localTxs, remoteTxs := make(map[common.Address]types.Transactions), pending + for _, account := range w.eth.TxPool().Locals() { + if txs := remoteTxs[account]; len(txs) > 0 { + delete(remoteTxs, account) + localTxs[account] = txs + } + } + if len(localTxs) > 0 { + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs) + if w.commitTransactions(txs, w.coinbase, interrupt) { + return + } + } + if len(remoteTxs) > 0 { + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, remoteTxs) + if w.commitTransactions(txs, w.coinbase, interrupt) { + return + } } - w.commit(uncles, w.fullTaskHook, true, tstart) } From af85d8e2b342aa6feff68cec59db9be3c1e85022 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 22 Aug 2018 14:47:50 +0800 Subject: [PATCH 056/126] cmd, eth: apply default miner recommit setting (#17479) --- cmd/utils/flags.go | 2 +- eth/config.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index cfca7b4ab4..c9a936dd5f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -367,7 +367,7 @@ var ( MinerRecommitIntervalFlag = cli.DurationFlag{ Name: "miner.recommit", Usage: "Time interval to recreate the block being mined.", - Value: 3 * time.Second, + Value: eth.DefaultConfig.MinerRecommit, } // Account settings UnlockedAccountFlag = cli.StringFlag{ diff --git a/eth/config.go b/eth/config.go index cbd02416be..1398a55145 100644 --- a/eth/config.go +++ b/eth/config.go @@ -49,6 +49,7 @@ var DefaultConfig = Config{ TrieCache: 256, TrieTimeout: 60 * time.Minute, MinerGasPrice: big.NewInt(18 * params.Shannon), + MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ From 316fc7ecfc10d06603f1358c1f4c1020ec36dd2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Aug 2018 11:39:00 +0300 Subject: [PATCH 057/126] params, swarm: release Geth v1.8.14 and Swarm v0.3.2 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 44c148e9c3..9cb027bd55 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 14 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 14 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index bb9935e24a..1f0c646197 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 2993fb519d7bb70163765c33bc95c5260ecb003a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Aug 2018 11:42:03 +0300 Subject: [PATCH 058/126] params, swarm: begin geth v1.8.15 and swarm v0.3.3 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 9cb027bd55..f65236a85c 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 14 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 15 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 1f0c646197..5672e19f81 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From f34f361ca6635690f6dd81d6f3bddfff498e9fd6 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 22 Aug 2018 16:53:33 +0200 Subject: [PATCH 059/126] swarm/api/http: fixed resolver bug (#17483) --- swarm/api/http/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index b5ea0c23d4..2aa1963969 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -820,7 +820,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.ResolveURI(r.Context(), uri, credentials) + manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFileFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) From 1df1187d831a1c62f61e8c90901dc78b5f86943a Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Thu, 23 Aug 2018 16:47:43 +0800 Subject: [PATCH 060/126] p2p: fix comment typo (#17491) --- p2p/dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/dial.go b/p2p/dial.go index d8feceb9f3..4baca7d931 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -146,7 +146,7 @@ func newDialState(static []*discover.Node, bootnodes []*discover.Node, ntab disc } func (s *dialstate) addStatic(n *discover.Node) { - // This overwites the task instead of updating an existing + // This overwrites the task instead of updating an existing // entry, giving users the opportunity to force a resolve operation. s.static[n.ID] = &dialTask{flags: staticDialedConn, dest: n} } From 92381ee00999398d0d0928d2c4be82b3334b912b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Aug 2018 13:02:36 +0300 Subject: [PATCH 061/126] cmd, eth: clean up miner startup API, drop noop config field --- cmd/geth/main.go | 20 ++++-------- cmd/utils/flags.go | 6 ---- eth/api.go | 47 +++++++--------------------- eth/backend.go | 76 +++++++++++++++++++++++++++++++++++----------- eth/config.go | 1 - eth/gen_config.go | 5 --- 6 files changed, 75 insertions(+), 80 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4b86382bdb..7b8a8244ee 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -336,26 +336,18 @@ func startNode(ctx *cli.Context, stack *node.Node) { if err := stack.Service(ðereum); err != nil { utils.Fatalf("Ethereum service not running: %v", err) } - // Use a reduced number of threads if requested - threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name) - if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { - threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name) - } - if threads > 0 { - type threaded interface { - SetThreads(threads int) - } - if th, ok := ethereum.Engine().(threaded); ok { - th.SetThreads(threads) - } - } // Set the gas price to the limits from the CLI and start mining gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name) if ctx.IsSet(utils.MinerGasPriceFlag.Name) { gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) } ethereum.TxPool().SetGasPrice(gasprice) - if err := ethereum.StartMining(true); err != nil { + + threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name) + if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { + threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name) + } + if err := ethereum.StartMining(threads); err != nil { utils.Fatalf("Failed to start mining: %v", err) } } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c9a936dd5f..b9a33ffe70 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1130,12 +1130,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } - if ctx.GlobalIsSet(MinerLegacyThreadsFlag.Name) { - cfg.MinerThreads = ctx.GlobalInt(MinerLegacyThreadsFlag.Name) - } - if ctx.GlobalIsSet(MinerThreadsFlag.Name) { - cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name) - } if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") } diff --git a/eth/api.go b/eth/api.go index 4b0ba8edbd..708f75a784 100644 --- a/eth/api.go +++ b/eth/api.go @@ -24,6 +24,7 @@ import ( "io" "math/big" "os" + "runtime" "strings" "time" @@ -34,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" @@ -94,47 +94,22 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { return &PrivateMinerAPI{e: e} } -// Start the miner with the given number of threads. If threads is nil the number -// of workers started is equal to the number of logical CPUs that are usable by -// this process. If mining is already running, this method adjust the number of -// threads allowed to use and updates the minimum price required by the transaction -// pool. +// Start starts the miner with the given number of threads. If threads is nil, +// the number of workers started is equal to the number of logical CPUs that are +// usable by this process. If mining is already running, this method adjust the +// number of threads allowed to use and updates the minimum price required by the +// transaction pool. func (api *PrivateMinerAPI) Start(threads *int) error { - // Set the number of threads if the seal engine supports it if threads == nil { - threads = new(int) - } else if *threads == 0 { - *threads = -1 // Disable the miner from within + return api.e.StartMining(runtime.NumCPU()) } - type threaded interface { - SetThreads(threads int) - } - if th, ok := api.e.engine.(threaded); ok { - log.Info("Updated mining threads", "threads", *threads) - th.SetThreads(*threads) - } - // Start the miner and return - if !api.e.IsMining() { - // Propagate the initial price point to the transaction pool - api.e.lock.RLock() - price := api.e.gasPrice - api.e.lock.RUnlock() - api.e.txPool.SetGasPrice(price) - return api.e.StartMining(true) - } - return nil + return api.e.StartMining(*threads) } -// Stop the miner -func (api *PrivateMinerAPI) Stop() bool { - type threaded interface { - SetThreads(threads int) - } - if th, ok := api.e.engine.(threaded); ok { - th.SetThreads(-1) - } +// Stop terminates the miner, both at the consensus engine level as well as at +// the block creation level. +func (api *PrivateMinerAPI) Stop() { api.e.StopMining() - return true } // SetExtra sets the extra data string that is included when this miner mines a block. diff --git a/eth/backend.go b/eth/backend.go index 648175acfd..c530d3c7c3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -102,12 +102,18 @@ func (s *Ethereum) AddLesServer(ls LesServer) { // New creates a new Ethereum object (including the // initialisation of the common Ethereum object) func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { + // Ensure configuration values are compatible and sane if config.SyncMode == downloader.LightSync { return nil, errors.New("can't run eth.Ethereum in light sync mode, use les.LightEthereum") } if !config.SyncMode.IsValid() { return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode) } + if config.MinerGasPrice == nil || config.MinerGasPrice.Cmp(common.Big0) <= 0 { + log.Warn("Sanitizing invalid miner gas price", "provided", config.MinerGasPrice, "updated", DefaultConfig.MinerGasPrice) + config.MinerGasPrice = new(big.Int).Set(DefaultConfig.MinerGasPrice) + } + // Assemble the Ethereum object chainDb, err := CreateDB(ctx, config, "chaindata") if err != nil { return nil, err @@ -333,32 +339,66 @@ func (s *Ethereum) SetEtherbase(etherbase common.Address) { s.miner.SetEtherbase(etherbase) } -func (s *Ethereum) StartMining(local bool) error { - eb, err := s.Etherbase() - if err != nil { - log.Error("Cannot start mining without etherbase", "err", err) - return fmt.Errorf("etherbase missing: %v", err) +// StartMining starts the miner with the given number of CPU threads. If mining +// is already running, this method adjust the number of threads allowed to use +// and updates the minimum price required by the transaction pool. +func (s *Ethereum) StartMining(threads int) error { + // Update the thread count within the consensus engine + type threaded interface { + SetThreads(threads int) } - if clique, ok := s.engine.(*clique.Clique); ok { - wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) - if wallet == nil || err != nil { - log.Error("Etherbase account unavailable locally", "err", err) - return fmt.Errorf("signer missing: %v", err) + if th, ok := s.engine.(threaded); ok { + log.Info("Updated mining threads", "threads", threads) + if threads == 0 { + threads = -1 // Disable the miner from within } - clique.Authorize(eb, wallet.SignHash) + th.SetThreads(threads) } - if local { - // If local (CPU) mining is started, we can disable the transaction rejection - // mechanism introduced to speed sync times. CPU mining on mainnet is ludicrous - // so none will ever hit this path, whereas marking sync done on CPU mining - // will ensure that private networks work in single miner mode too. + // If the miner was not running, initialize it + if !s.IsMining() { + // Propagate the initial price point to the transaction pool + s.lock.RLock() + price := s.gasPrice + s.lock.RUnlock() + s.txPool.SetGasPrice(price) + + // Configure the local mining addess + eb, err := s.Etherbase() + if err != nil { + log.Error("Cannot start mining without etherbase", "err", err) + return fmt.Errorf("etherbase missing: %v", err) + } + if clique, ok := s.engine.(*clique.Clique); ok { + wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) + if wallet == nil || err != nil { + log.Error("Etherbase account unavailable locally", "err", err) + return fmt.Errorf("signer missing: %v", err) + } + clique.Authorize(eb, wallet.SignHash) + } + // If mining is started, we can disable the transaction rejection mechanism + // introduced to speed sync times. atomic.StoreUint32(&s.protocolManager.acceptTxs, 1) + + go s.miner.Start(eb) } - go s.miner.Start(eb) return nil } -func (s *Ethereum) StopMining() { s.miner.Stop() } +// StopMining terminates the miner, both at the consensus engine level as well as +// at the block creation level. +func (s *Ethereum) StopMining() { + // Update the thread count within the consensus engine + type threaded interface { + SetThreads(threads int) + } + if th, ok := s.engine.(threaded); ok { + th.SetThreads(-1) + } + // Stop the block creating itself + s.miner.Stop() +} + func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } diff --git a/eth/config.go b/eth/config.go index 1398a55145..5b2eca585c 100644 --- a/eth/config.go +++ b/eth/config.go @@ -97,7 +97,6 @@ type Config struct { // Mining-related options Etherbase common.Address `toml:",omitempty"` - MinerThreads int `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData []byte `toml:",omitempty"` MinerGasPrice *big.Int diff --git a/eth/gen_config.go b/eth/gen_config.go index 62556be7e1..a72e35bcd1 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -31,7 +31,6 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieCache int TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` - MinerThreads int `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData hexutil.Bytes `toml:",omitempty"` MinerGasPrice *big.Int @@ -55,7 +54,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieCache = c.TrieCache enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase - enc.MinerThreads = c.MinerThreads enc.MinerNotify = c.MinerNotify enc.MinerExtraData = c.MinerExtraData enc.MinerGasPrice = c.MinerGasPrice @@ -134,9 +132,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Etherbase != nil { c.Etherbase = *dec.Etherbase } - if dec.MinerThreads != nil { - c.MinerThreads = *dec.MinerThreads - } if dec.MinerNotify != nil { c.MinerNotify = dec.MinerNotify } From 1e63a015a56ae45825c842b5a2b17d2cf2ac1ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Aug 2018 14:03:13 +0300 Subject: [PATCH 062/126] miner: add two stress tests based on clique and ethash --- miner/stress_clique.go | 217 +++++++++++++++++++++++++++++++++++++++++ miner/stress_ethash.go | 197 +++++++++++++++++++++++++++++++++++++ 2 files changed, 414 insertions(+) create mode 100644 miner/stress_clique.go create mode 100644 miner/stress_ethash.go diff --git a/miner/stress_clique.go b/miner/stress_clique.go new file mode 100644 index 0000000000..8545b9413a --- /dev/null +++ b/miner/stress_clique.go @@ -0,0 +1,217 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build none + +// This file contains a miner stress test based on the Clique consensus engine. +package main + +import ( + "bytes" + "crypto/ecdsa" + "fmt" + "io/ioutil" + "math/big" + "math/rand" + "os" + "time" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" +) + +func main() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + sealers := make([]*ecdsa.PrivateKey, 4) + for i := 0; i < len(sealers); i++ { + sealers[i], _ = crypto.GenerateKey() + } + // Create a Clique network based off of the Rinkeby config + genesis := makeGenesis(faucets, sealers) + + var ( + nodes []*node.Node + enodes []string + ) + for _, sealer := range sealers { + // Start the node and wait until it's up + node, err := makeSealer(genesis, enodes) + if err != nil { + panic(err) + } + defer node.Stop() + + for node.Server().NodeInfo().Ports.Listener == 0 { + time.Sleep(250 * time.Millisecond) + } + // Connect the node to al the previous ones + for _, enode := range enodes { + enode, err := discover.ParseNode(enode) + if err != nil { + panic(err) + } + node.Server().AddPeer(enode) + } + // Start tracking the node and it's enode url + nodes = append(nodes, node) + + enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) + enodes = append(enodes, enode) + + // Inject the signer key and start sealing with it + store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) + signer, err := store.ImportECDSA(sealer, "") + if err != nil { + panic(err) + } + if err := store.Unlock(signer, ""); err != nil { + panic(err) + } + } + // Iterate over all the nodes and start signing with them + time.Sleep(3 * time.Second) + + for _, node := range nodes { + var ethereum *eth.Ethereum + if err := node.Service(ðereum); err != nil { + panic(err) + } + if err := ethereum.StartMining(1); err != nil { + panic(err) + } + } + time.Sleep(3 * time.Second) + + // Start injecting transactions from the faucet like crazy + nonces := make([]uint64, len(faucets)) + for { + index := rand.Intn(len(faucets)) + + // Fetch the accessor for the relevant signer + var ethereum *eth.Ethereum + if err := nodes[index%len(nodes)].Service(ðereum); err != nil { + panic(err) + } + // Create a self transaction and inject into the pool + tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), types.HomesteadSigner{}, faucets[index]) + if err != nil { + panic(err) + } + if err := ethereum.TxPool().AddLocal(tx); err != nil { + panic(err) + } + nonces[index]++ + + // Wait if we're too saturated + if pend, _ := ethereum.TxPool().Stats(); pend > 2048 { + time.Sleep(100 * time.Millisecond) + } + } +} + +// makeGenesis creates a custom Clique genesis block based on some pre-defined +// signer and faucet accounts. +func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core.Genesis { + // Create a Clique network based off of the Rinkeby config + genesis := core.DefaultRinkebyGenesisBlock() + genesis.GasLimit = 25000000 + + genesis.Config.ChainID = big.NewInt(18) + genesis.Config.Clique.Period = 1 + genesis.Config.EIP150Hash = common.Hash{} + + genesis.Alloc = core.GenesisAlloc{} + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + // Sort the signers and embed into the extra-data section + signers := make([]common.Address, len(sealers)) + for i, sealer := range sealers { + signers[i] = crypto.PubkeyToAddress(sealer.PublicKey) + } + for i := 0; i < len(signers); i++ { + for j := i + 1; j < len(signers); j++ { + if bytes.Compare(signers[i][:], signers[j][:]) > 0 { + signers[i], signers[j] = signers[j], signers[i] + } + } + } + genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65) + for i, signer := range signers { + copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:]) + } + // Return the genesis block for initialization + return genesis +} + +func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := ioutil.TempDir("", "") + + config := &node.Config{ + Name: "geth", + Version: params.Version, + DataDir: datadir, + P2P: p2p.Config{ + ListenAddr: "0.0.0.0:0", + NoDiscovery: true, + MaxPeers: 25, + }, + NoUSB: true, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, err + } + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + return eth.New(ctx, ð.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: core.DefaultTxPoolConfig, + GPO: eth.DefaultConfig.GPO, + MinerGasPrice: big.NewInt(1), + MinerRecommit: time.Second, + }) + }); err != nil { + return nil, err + } + // Start the node and return if successful + return stack, stack.Start() +} diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go new file mode 100644 index 0000000000..4ed8aeee4f --- /dev/null +++ b/miner/stress_ethash.go @@ -0,0 +1,197 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build none + +// This file contains a miner stress test based on the Ethash consensus engine. +package main + +import ( + "crypto/ecdsa" + "fmt" + "io/ioutil" + "math/big" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" +) + +func main() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + // Pre-generate the ethash mining DAG so we don't race + ethash.MakeDataset(1, filepath.Join(os.Getenv("HOME"), ".ethash")) + + // Create an Ethash network based off of the Ropsten config + genesis := makeGenesis(faucets) + + var ( + nodes []*node.Node + enodes []string + ) + for i := 0; i < 4; i++ { + // Start the node and wait until it's up + node, err := makeMiner(genesis, enodes) + if err != nil { + panic(err) + } + defer node.Stop() + + for node.Server().NodeInfo().Ports.Listener == 0 { + time.Sleep(250 * time.Millisecond) + } + // Connect the node to al the previous ones + for _, enode := range enodes { + enode, err := discover.ParseNode(enode) + if err != nil { + panic(err) + } + node.Server().AddPeer(enode) + } + // Start tracking the node and it's enode url + nodes = append(nodes, node) + + enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) + enodes = append(enodes, enode) + + // Inject the signer key and start sealing with it + store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) + if _, err := store.NewAccount(""); err != nil { + panic(err) + } + } + // Iterate over all the nodes and start signing with them + time.Sleep(3 * time.Second) + + for _, node := range nodes { + var ethereum *eth.Ethereum + if err := node.Service(ðereum); err != nil { + panic(err) + } + if err := ethereum.StartMining(1); err != nil { + panic(err) + } + } + time.Sleep(3 * time.Second) + + // Start injecting transactions from the faucets like crazy + nonces := make([]uint64, len(faucets)) + for { + index := rand.Intn(len(faucets)) + + // Fetch the accessor for the relevant signer + var ethereum *eth.Ethereum + if err := nodes[index%len(nodes)].Service(ðereum); err != nil { + panic(err) + } + // Create a self transaction and inject into the pool + tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000+rand.Int63n(65536)), nil), types.HomesteadSigner{}, faucets[index]) + if err != nil { + panic(err) + } + if err := ethereum.TxPool().AddLocal(tx); err != nil { + panic(err) + } + nonces[index]++ + + // Wait if we're too saturated + if pend, _ := ethereum.TxPool().Stats(); pend > 2048 { + time.Sleep(100 * time.Millisecond) + } + } +} + +// makeGenesis creates a custom Ethash genesis block based on some pre-defined +// faucet accounts. +func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { + genesis := core.DefaultTestnetGenesisBlock() + genesis.Difficulty = params.MinimumDifficulty + genesis.GasLimit = 25000000 + + genesis.Config.ChainID = big.NewInt(18) + genesis.Config.EIP150Hash = common.Hash{} + + genesis.Alloc = core.GenesisAlloc{} + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + return genesis +} + +func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := ioutil.TempDir("", "") + + config := &node.Config{ + Name: "geth", + Version: params.Version, + DataDir: datadir, + P2P: p2p.Config{ + ListenAddr: "0.0.0.0:0", + NoDiscovery: true, + MaxPeers: 25, + }, + NoUSB: true, + UseLightweightKDF: true, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, err + } + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + return eth.New(ctx, ð.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: core.DefaultTxPoolConfig, + GPO: eth.DefaultConfig.GPO, + Ethash: eth.DefaultConfig.Ethash, + MinerGasPrice: big.NewInt(1), + MinerRecommit: time.Second, + }) + }); err != nil { + return nil, err + } + // Start the node and return if successful + return stack, stack.Start() +} From 1136269a79e6ee8bc97f5bf277bf8ec12286b79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Aug 2018 15:44:27 +0300 Subject: [PATCH 063/126] miner: differentiate between uncle and lost block --- miner/unconfirmed.go | 35 +++++++++++++++++++++++++++-------- miner/unconfirmed_test.go | 13 ++++++++----- miner/worker.go | 2 +- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/miner/unconfirmed.go b/miner/unconfirmed.go index 7a4fc7cc5a..3eb29bc705 100644 --- a/miner/unconfirmed.go +++ b/miner/unconfirmed.go @@ -25,11 +25,14 @@ import ( "github.com/ethereum/go-ethereum/log" ) -// headerRetriever is used by the unconfirmed block set to verify whether a previously +// chainRetriever is used by the unconfirmed block set to verify whether a previously // mined block is part of the canonical chain or not. -type headerRetriever interface { +type chainRetriever interface { // GetHeaderByNumber retrieves the canonical header associated with a block number. GetHeaderByNumber(number uint64) *types.Header + + // GetBlockByNumber retrieves the canonical block associated with a block number. + GetBlockByNumber(number uint64) *types.Block } // unconfirmedBlock is a small collection of metadata about a locally mined block @@ -44,14 +47,14 @@ type unconfirmedBlock struct { // used by the miner to provide logs to the user when a previously mined block // has a high enough guarantee to not be reorged out of the canonical chain. type unconfirmedBlocks struct { - chain headerRetriever // Blockchain to verify canonical status through - depth uint // Depth after which to discard previous blocks - blocks *ring.Ring // Block infos to allow canonical chain cross checks - lock sync.RWMutex // Protects the fields from concurrent access + chain chainRetriever // Blockchain to verify canonical status through + depth uint // Depth after which to discard previous blocks + blocks *ring.Ring // Block infos to allow canonical chain cross checks + lock sync.RWMutex // Protects the fields from concurrent access } // newUnconfirmedBlocks returns new data structure to track currently unconfirmed blocks. -func newUnconfirmedBlocks(chain headerRetriever, depth uint) *unconfirmedBlocks { +func newUnconfirmedBlocks(chain chainRetriever, depth uint) *unconfirmedBlocks { return &unconfirmedBlocks{ chain: chain, depth: depth, @@ -103,7 +106,23 @@ func (set *unconfirmedBlocks) Shift(height uint64) { case header.Hash() == next.hash: log.Info("🔗 block reached canonical chain", "number", next.index, "hash", next.hash) default: - log.Info("⑂ block became a side fork", "number", next.index, "hash", next.hash) + // Block is not canonical, check whether we have an uncle or a lost block + included := false + for number := next.index; !included && number < next.index+uint64(set.depth) && number <= height; number++ { + if block := set.chain.GetBlockByNumber(number); block != nil { + for _, uncle := range block.Uncles() { + if uncle.Hash() == next.hash { + included = true + break + } + } + } + } + if included { + log.Info("⑂ block became an uncle", "number", next.index, "hash", next.hash) + } else { + log.Info("😱 block lost", "number", next.index, "hash", next.hash) + } } // Drop the block out of the ring if set.blocks.Value == set.blocks.Next().Value { diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go index 456af1764a..42e77f3e64 100644 --- a/miner/unconfirmed_test.go +++ b/miner/unconfirmed_test.go @@ -23,11 +23,14 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// noopHeaderRetriever is an implementation of headerRetriever that always +// noopChainRetriever is an implementation of headerRetriever that always // returns nil for any requested headers. -type noopHeaderRetriever struct{} +type noopChainRetriever struct{} -func (r *noopHeaderRetriever) GetHeaderByNumber(number uint64) *types.Header { +func (r *noopChainRetriever) GetHeaderByNumber(number uint64) *types.Header { + return nil +} +func (r *noopChainRetriever) GetBlockByNumber(number uint64) *types.Block { return nil } @@ -36,7 +39,7 @@ func (r *noopHeaderRetriever) GetHeaderByNumber(number uint64) *types.Header { func TestUnconfirmedInsertBounds(t *testing.T) { limit := uint(10) - pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit) + pool := newUnconfirmedBlocks(new(noopChainRetriever), limit) for depth := uint64(0); depth < 2*uint64(limit); depth++ { // Insert multiple blocks for the same level just to stress it for i := 0; i < int(depth); i++ { @@ -58,7 +61,7 @@ func TestUnconfirmedShifts(t *testing.T) { // Create a pool with a few blocks on various depths limit, start := uint(10), uint64(25) - pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit) + pool := newUnconfirmedBlocks(new(noopChainRetriever), limit) for depth := start; depth < start+uint64(limit); depth++ { pool.Insert(depth, common.Hash([32]byte{byte(depth)})) } diff --git a/miner/worker.go b/miner/worker.go index 8c3337ba45..cedc158b51 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -55,7 +55,7 @@ const ( resubmitAdjustChanSize = 10 // miningLogAtDepth is the number of confirmations before logging successful mining. - miningLogAtDepth = 5 + miningLogAtDepth = 7 // minRecommitInterval is the minimal time interval to recreate the mining block with // any newly arrived transactions. From c3f7e3be3b60df3edd168e80aa89ee2992932b0d Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 23 Aug 2018 20:59:58 +0800 Subject: [PATCH 064/126] core/statedb: deep copy logs (#17489) --- core/state/statedb.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 92d394ae32..101b03a127 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -489,10 +489,13 @@ func (self *StateDB) Copy() *StateDB { state.stateObjectsDirty[addr] = struct{}{} } } - for hash, logs := range self.logs { - state.logs[hash] = make([]*types.Log, len(logs)) - copy(state.logs[hash], logs) + cpy := make([]*types.Log, len(logs)) + for i, l := range logs { + cpy[i] = new(types.Log) + *cpy[i] = *l + } + state.logs[hash] = cpy } for hash, preimage := range self.preimages { state.preimages[hash] = preimage From 40a71f28cf1ada0bf6bdcdc2f3c6f31a8da134a2 Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 23 Aug 2018 21:02:57 +0800 Subject: [PATCH 065/126] miner: fix state commit, track old work packages too (#17490) * miner: commit state which is relative with sealing result * consensus, core, miner, mobile: introduce sealHash interface * miner: evict pending task with threshold * miner: go fmt --- consensus/clique/clique.go | 5 +++ consensus/consensus.go | 3 ++ consensus/ethash/consensus.go | 29 ++++++++++++- consensus/ethash/ethash_test.go | 11 +++-- consensus/ethash/sealer.go | 4 +- consensus/ethash/sealer_test.go | 2 +- core/types/block.go | 23 ----------- miner/worker.go | 73 +++++++++++++++++++++++---------- mobile/types.go | 35 ++++++++-------- 9 files changed, 112 insertions(+), 73 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 0859447014..3730c91f62 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -673,6 +673,11 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int { return new(big.Int).Set(diffNoTurn) } +// SealHash returns the hash of a block prior to it being sealed. +func (c *Clique) SealHash(header *types.Header) common.Hash { + return sigHash(header) +} + // Close implements consensus.Engine. It's a noop for clique as there is are no background threads. func (c *Clique) Close() error { return nil diff --git a/consensus/consensus.go b/consensus/consensus.go index 8271754445..27799f13c0 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -90,6 +90,9 @@ type Engine interface { // seal place on top. Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) + // SealHash returns the hash of a block prior to it being sealed. + SealHash(header *types.Header) common.Hash + // CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty // that a new block should have. CalcDifficulty(chain ChainReader, time uint64, parent *types.Header) *big.Int diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 86fd997ae4..259cc5056b 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -31,7 +31,9 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" ) // Ethash proof-of-work protocol constants. @@ -495,7 +497,7 @@ func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Head if fulldag { dataset := ethash.dataset(number, true) if dataset.generated() { - digest, result = hashimotoFull(dataset.dataset, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + digest, result = hashimotoFull(dataset.dataset, ethash.SealHash(header).Bytes(), header.Nonce.Uint64()) // Datasets are unmapped in a finalizer. Ensure that the dataset stays alive // until after the call to hashimotoFull so it's not unmapped while being used. @@ -513,7 +515,7 @@ func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Head if ethash.config.PowMode == ModeTest { size = 32 * 1024 } - digest, result = hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + digest, result = hashimotoLight(size, cache.cache, ethash.SealHash(header).Bytes(), header.Nonce.Uint64()) // Caches are unmapped in a finalizer. Ensure that the cache stays alive // until after the call to hashimotoLight so it's not unmapped while being used. @@ -552,6 +554,29 @@ func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header return types.NewBlock(header, txs, uncles, receipts), nil } +// SealHash returns the hash of a block prior to it being sealed. +func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { + hasher := sha3.NewKeccak256() + + rlp.Encode(hasher, []interface{}{ + header.ParentHash, + header.UncleHash, + header.Coinbase, + header.Root, + header.TxHash, + header.ReceiptHash, + header.Bloom, + header.Difficulty, + header.Number, + header.GasLimit, + header.GasUsed, + header.Time, + header.Extra, + }) + hasher.Sum(hash[:0]) + return hash +} + // Some weird constants to avoid constant memory allocs for them. var ( big8 = big.NewInt(8) diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 87ac17c2b2..b190d63d6b 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -94,6 +94,7 @@ func TestRemoteSealer(t *testing.T) { } header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) + sealhash := ethash.SealHash(header) // Push new work. ethash.Seal(nil, block, nil) @@ -102,27 +103,29 @@ func TestRemoteSealer(t *testing.T) { work [3]string err error ) - if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { + if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { t.Error("expect to return a mining work has same hash") } - if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { + if res := api.SubmitWork(types.BlockNonce{}, sealhash, common.Hash{}); res { t.Error("expect to return false when submit a fake solution") } // Push new block with same block number to replace the original one. header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} block = types.NewBlockWithHeader(header) + sealhash = ethash.SealHash(header) ethash.Seal(nil, block, nil) - if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { + if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { t.Error("expect to return the latest pushed work") } // Push block with higher block number. newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} newBlock := types.NewBlockWithHeader(newHead) + newSealhash := ethash.SealHash(newHead) ethash.Seal(nil, newBlock, nil) - if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { + if res := api.SubmitWork(types.BlockNonce{}, newSealhash, common.Hash{}); res { t.Error("expect to return false when submit a stale solution") } } diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index c3b2c86d12..a458c60f66 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -111,7 +111,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s // Extract some data from the header var ( header = block.Header() - hash = header.HashNoNonce().Bytes() + hash = ethash.SealHash(header).Bytes() target = new(big.Int).Div(two256, header.Difficulty) number = header.Number.Uint64() dataset = ethash.dataset(number, false) @@ -213,7 +213,7 @@ func (ethash *Ethash) remote(notify []string) { // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty makeWork := func(block *types.Block) { - hash := block.HashNoNonce() + hash := ethash.SealHash(block.Header()) currentWork[0] = hash.Hex() currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 6c7157a5ae..d1b66f9cf3 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -51,7 +51,7 @@ func TestRemoteNotify(t *testing.T) { ethash.Seal(nil, block, nil) select { case work := <-sink: - if want := header.HashNoNonce().Hex(); work[0] != want { + if want := ethash.SealHash(header).Hex(); work[0] != want { t.Errorf("work packet hash mismatch: have %s, want %s", work[0], want) } if want := common.BytesToHash(SeedHash(header.Number.Uint64())).Hex(); work[1] != want { diff --git a/core/types/block.go b/core/types/block.go index ae1b4299d3..8a21bba1e7 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -102,25 +102,6 @@ func (h *Header) Hash() common.Hash { return rlpHash(h) } -// HashNoNonce returns the hash which is used as input for the proof-of-work search. -func (h *Header) HashNoNonce() common.Hash { - return rlpHash([]interface{}{ - h.ParentHash, - h.UncleHash, - h.Coinbase, - h.Root, - h.TxHash, - h.ReceiptHash, - h.Bloom, - h.Difficulty, - h.Number, - h.GasLimit, - h.GasUsed, - h.Time, - h.Extra, - }) -} - // Size returns the approximate memory used by all internal contents. It is used // to approximate and limit the memory consumption of various caches. func (h *Header) Size() common.StorageSize { @@ -324,10 +305,6 @@ func (b *Block) Header() *Header { return CopyHeader(b.header) } // Body returns the non-header content of the block. func (b *Block) Body() *Body { return &Body{b.transactions, b.uncles} } -func (b *Block) HashNoNonce() common.Hash { - return b.header.HashNoNonce() -} - // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previsouly cached value. func (b *Block) Size() common.StorageSize { diff --git a/miner/worker.go b/miner/worker.go index 8c3337ba45..18fb12e451 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -72,6 +72,9 @@ const ( // intervalAdjustBias is applied during the new resubmit interval calculation in favor of // increasing upper limit or decreasing lower limit so that the limit can be reachable. intervalAdjustBias = 200 * 1000.0 * 1000.0 + + // staleThreshold is the maximum distance of the acceptable stale block. + staleThreshold = 7 ) // environment is the worker's current environment and holds all of the current state information. @@ -150,6 +153,9 @@ type worker struct { coinbase common.Address extra []byte + pendingMu sync.RWMutex + pendingTasks map[common.Hash]*task + snapshotMu sync.RWMutex // The lock used to protect the block snapshot and state snapshot snapshotBlock *types.Block snapshotState *state.StateDB @@ -174,6 +180,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, chain: eth.BlockChain(), possibleUncles: make(map[common.Hash]*types.Block), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), + pendingTasks: make(map[common.Hash]*task), txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), @@ -317,13 +324,25 @@ func (w *worker) newWorkLoop(recommit time.Duration) { } recommit = time.Duration(int64(next)) } + // clearPending cleans the stale pending tasks. + clearPending := func(number uint64) { + w.pendingMu.Lock() + for h, t := range w.pendingTasks { + if t.block.NumberU64()+staleThreshold <= number { + delete(w.pendingTasks, h) + } + } + w.pendingMu.Unlock() + } for { select { case <-w.startCh: + clearPending(w.chain.CurrentBlock().NumberU64()) commit(false, commitInterruptNewHead) - case <-w.chainHeadCh: + case head := <-w.chainHeadCh: + clearPending(head.Block.NumberU64()) commit(false, commitInterruptNewHead) case <-timer.C: @@ -454,28 +473,37 @@ func (w *worker) mainLoop() { // seal pushes a sealing task to consensus engine and submits the result. func (w *worker) seal(t *task, stop <-chan struct{}) { - var ( - err error - res *task - ) - if w.skipSealHook != nil && w.skipSealHook(t) { return } + // The reason for caching task first is: + // A previous sealing action will be canceled by subsequent actions, + // however, remote miner may submit a result based on the cancelled task. + // So we should only submit the pending state corresponding to the seal result. + // TODO(rjl493456442) Replace the seal-wait logic structure + w.pendingMu.Lock() + w.pendingTasks[w.engine.SealHash(t.block.Header())] = t + w.pendingMu.Unlock() - if t.block, err = w.engine.Seal(w.chain, t.block, stop); t.block != nil { - log.Info("Successfully sealed new block", "number", t.block.Number(), "hash", t.block.Hash(), - "elapsed", common.PrettyDuration(time.Since(t.createdAt))) - res = t - } else { - if err != nil { - log.Warn("Block sealing failed", "err", err) + if block, err := w.engine.Seal(w.chain, t.block, stop); block != nil { + sealhash := w.engine.SealHash(block.Header()) + w.pendingMu.RLock() + task, exist := w.pendingTasks[sealhash] + w.pendingMu.RUnlock() + if !exist { + log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash()) + return } - res = nil - } - select { - case w.resultCh <- res: - case <-w.exitCh: + // Assemble sealing result + task.block = block + log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash(), + "elapsed", common.PrettyDuration(time.Since(task.createdAt))) + select { + case w.resultCh <- task: + case <-w.exitCh: + } + } else if err != nil { + log.Warn("Block sealing failed", "err", err) } } @@ -501,12 +529,13 @@ func (w *worker) taskLoop() { w.newTaskHook(task) } // Reject duplicate sealing work due to resubmitting. - if task.block.HashNoNonce() == prev { + sealHash := w.engine.SealHash(task.block.Header()) + if sealHash == prev { continue } interrupt() stopCh = make(chan struct{}) - prev = task.block.HashNoNonce() + prev = sealHash go w.seal(task, stopCh) case <-w.exitCh: interrupt() @@ -928,8 +957,8 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st } feesEth := new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) - log.Info("Commit new mining work", "number", block.Number(), "uncles", len(uncles), "txs", w.current.tcount, - "gas", block.GasUsed(), "fees", feesEth, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Commit new mining work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), + "uncles", len(uncles), "txs", w.current.tcount, "gas", block.GasUsed(), "fees", feesEth, "elapsed", common.PrettyDuration(time.Since(start))) case <-w.exitCh: log.Info("Worker has exited") diff --git a/mobile/types.go b/mobile/types.go index f32b4918f3..b2780f3076 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -168,25 +168,22 @@ func (b *Block) EncodeJSON() (string, error) { return string(data), err } -func (b *Block) GetParentHash() *Hash { return &Hash{b.block.ParentHash()} } -func (b *Block) GetUncleHash() *Hash { return &Hash{b.block.UncleHash()} } -func (b *Block) GetCoinbase() *Address { return &Address{b.block.Coinbase()} } -func (b *Block) GetRoot() *Hash { return &Hash{b.block.Root()} } -func (b *Block) GetTxHash() *Hash { return &Hash{b.block.TxHash()} } -func (b *Block) GetReceiptHash() *Hash { return &Hash{b.block.ReceiptHash()} } -func (b *Block) GetBloom() *Bloom { return &Bloom{b.block.Bloom()} } -func (b *Block) GetDifficulty() *BigInt { return &BigInt{b.block.Difficulty()} } -func (b *Block) GetNumber() int64 { return b.block.Number().Int64() } -func (b *Block) GetGasLimit() int64 { return int64(b.block.GasLimit()) } -func (b *Block) GetGasUsed() int64 { return int64(b.block.GasUsed()) } -func (b *Block) GetTime() int64 { return b.block.Time().Int64() } -func (b *Block) GetExtra() []byte { return b.block.Extra() } -func (b *Block) GetMixDigest() *Hash { return &Hash{b.block.MixDigest()} } -func (b *Block) GetNonce() int64 { return int64(b.block.Nonce()) } - -func (b *Block) GetHash() *Hash { return &Hash{b.block.Hash()} } -func (b *Block) GetHashNoNonce() *Hash { return &Hash{b.block.HashNoNonce()} } - +func (b *Block) GetParentHash() *Hash { return &Hash{b.block.ParentHash()} } +func (b *Block) GetUncleHash() *Hash { return &Hash{b.block.UncleHash()} } +func (b *Block) GetCoinbase() *Address { return &Address{b.block.Coinbase()} } +func (b *Block) GetRoot() *Hash { return &Hash{b.block.Root()} } +func (b *Block) GetTxHash() *Hash { return &Hash{b.block.TxHash()} } +func (b *Block) GetReceiptHash() *Hash { return &Hash{b.block.ReceiptHash()} } +func (b *Block) GetBloom() *Bloom { return &Bloom{b.block.Bloom()} } +func (b *Block) GetDifficulty() *BigInt { return &BigInt{b.block.Difficulty()} } +func (b *Block) GetNumber() int64 { return b.block.Number().Int64() } +func (b *Block) GetGasLimit() int64 { return int64(b.block.GasLimit()) } +func (b *Block) GetGasUsed() int64 { return int64(b.block.GasUsed()) } +func (b *Block) GetTime() int64 { return b.block.Time().Int64() } +func (b *Block) GetExtra() []byte { return b.block.Extra() } +func (b *Block) GetMixDigest() *Hash { return &Hash{b.block.MixDigest()} } +func (b *Block) GetNonce() int64 { return int64(b.block.Nonce()) } +func (b *Block) GetHash() *Hash { return &Hash{b.block.Hash()} } func (b *Block) GetHeader() *Header { return &Header{b.block.Header()} } func (b *Block) GetUncles() *Headers { return &Headers{b.block.Uncles()} } func (b *Block) GetTransactions() *Transactions { return &Transactions{b.block.Transactions()} } From 70398d300d4da97c89f96f5c9629caa327de5c39 Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Sat, 25 Aug 2018 02:08:48 +0800 Subject: [PATCH 066/126] trie: fix typo (#17498) --- trie/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/sync.go b/trie/sync.go index ccec80c9e3..88d6eb7799 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -73,7 +73,7 @@ func newSyncMemBatch() *syncMemBatch { // and reconstructs the trie step by step until all is done. type Sync struct { database DatabaseReader // Persistent database to check for existing entries - membatch *syncMemBatch // Memory buffer to avoid frequest database writes + membatch *syncMemBatch // Memory buffer to avoid frequent database writes requests map[common.Hash]*request // Pending requests pertaining to a key hash queue *prque.Prque // Priority queue with the pending requests } From d1aa605f1e8639769cdd75bcec3064c29a62b34a Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Mon, 27 Aug 2018 16:49:29 +0800 Subject: [PATCH 067/126] all: remove the duplicate 'the' in annotations (#17509) --- accounts/abi/bind/bind.go | 4 ++-- cmd/wnode/main.go | 2 +- common/bitutil/compress_test.go | 4 ++-- console/prompter.go | 4 ++-- core/block_validator.go | 2 +- core/state_processor.go | 2 +- core/vm/contract.go | 2 +- crypto/secp256k1/secp256.go | 2 +- eth/downloader/queue.go | 2 +- eth/fetcher/fetcher.go | 2 +- p2p/server.go | 2 +- swarm/api/api.go | 2 +- swarm/network/README.md | 2 +- swarm/network/stream/intervals/intervals.go | 2 +- swarm/network/stream/snapshot_sync_test.go | 2 +- whisper/whisperv5/whisper.go | 2 +- whisper/whisperv6/whisper.go | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 411177057b..172bed8fa0 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -207,7 +207,7 @@ func bindTypeGo(kind abi.Type) string { // The inner function of bindTypeGo, this finds the inner type of stringKind. // (Or just the type itself if it is not an array or slice) -// The length of the matched part is returned, with the the translated type. +// The length of the matched part is returned, with the translated type. func bindUnnestedTypeGo(stringKind string) (int, string) { switch { @@ -255,7 +255,7 @@ func bindTypeJava(kind abi.Type) string { // The inner function of bindTypeJava, this finds the inner type of stringKind. // (Or just the type itself if it is not an array or slice) -// The length of the matched part is returned, with the the translated type. +// The length of the matched part is returned, with the translated type. func bindUnnestedTypeJava(stringKind string) (int, string) { switch { diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 31b65c1da9..ae79841f6b 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -754,7 +754,7 @@ func extractIDFromEnode(s string) []byte { return n.ID[:] } -// obfuscateBloom adds 16 random bits to the the bloom +// obfuscateBloom adds 16 random bits to the bloom // filter, in order to obfuscate the containing topics. // it does so deterministically within every session. // despite additional bits, it will match on average diff --git a/common/bitutil/compress_test.go b/common/bitutil/compress_test.go index 9bd1de103a..13a13011dc 100644 --- a/common/bitutil/compress_test.go +++ b/common/bitutil/compress_test.go @@ -117,7 +117,7 @@ func TestDecodingCycle(t *testing.T) { // TestCompression tests that compression works by returning either the bitset // encoded input, or the actual input if the bitset version is longer. func TestCompression(t *testing.T) { - // Check the the compression returns the bitset encoding is shorter + // Check the compression returns the bitset encoding is shorter in := hexutil.MustDecode("0x4912385c0e7b64000000") out := hexutil.MustDecode("0x80fe4912385c0e7b64") @@ -127,7 +127,7 @@ func TestCompression(t *testing.T) { if data, err := DecompressBytes(out, len(in)); err != nil || !bytes.Equal(data, in) { t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err) } - // Check the the compression returns the input if the bitset encoding is longer + // Check the compression returns the input if the bitset encoding is longer in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") diff --git a/console/prompter.go b/console/prompter.go index 9b90034db9..29a53aeadd 100644 --- a/console/prompter.go +++ b/console/prompter.go @@ -43,7 +43,7 @@ type UserPrompter interface { // choice to be made, returning that choice. PromptConfirm(prompt string) (bool, error) - // SetHistory sets the the input scrollback history that the prompter will allow + // SetHistory sets the input scrollback history that the prompter will allow // the user to scroll back to. SetHistory(history []string) @@ -149,7 +149,7 @@ func (p *terminalPrompter) PromptConfirm(prompt string) (bool, error) { return false, err } -// SetHistory sets the the input scrollback history that the prompter will allow +// SetHistory sets the input scrollback history that the prompter will allow // the user to scroll back to. func (p *terminalPrompter) SetHistory(history []string) { p.State.ReadHistory(strings.NewReader(strings.Join(history, "\n"))) diff --git a/core/block_validator.go b/core/block_validator.go index 98958809b7..ecd6a89bc2 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -45,7 +45,7 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin return validator } -// ValidateBody validates the given block's uncles and verifies the the block +// ValidateBody validates the given block's uncles and verifies the block // header's transaction and uncle roots. The headers are assumed to be already // validated at this point. func (v *BlockValidator) ValidateBody(block *types.Block) error { diff --git a/core/state_processor.go b/core/state_processor.go index 8e238ce1f0..1a91a57ab4 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -61,7 +61,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg allLogs []*types.Log gp = new(GasPool).AddGas(block.GasLimit()) ) - // Mutate the the block and state according to any hard-fork specs + // Mutate the block and state according to any hard-fork specs if p.config.DAOForkSupport && p.config.DAOForkBlock != nil && p.config.DAOForkBlock.Cmp(block.Number()) == 0 { misc.ApplyDAOHardFork(statedb) } diff --git a/core/vm/contract.go b/core/vm/contract.go index b466681dbd..26bca68951 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -40,7 +40,7 @@ type AccountRef common.Address func (ar AccountRef) Address() common.Address { return (common.Address)(ar) } // Contract represents an ethereum contract in the state database. It contains -// the the contract code, calling arguments. Contract implements ContractRef +// the contract code, calling arguments. Contract implements ContractRef type Contract struct { // CallerAddress is the result of the caller which initialised this // contract. However when the "call method" is delegated this value diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index 843fb12525..35d0eef34a 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -86,7 +86,7 @@ func Sign(msg []byte, seckey []byte) ([]byte, error) { return sig, nil } -// RecoverPubkey returns the the public key of the signer. +// RecoverPubkey returns the public key of the signer. // msg must be the 32-byte hash of the message to be signed. // sig must be a 65-byte compact ECDSA signature containing the // recovery id as the last element. diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 984dd13d68..8529535ba7 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -662,7 +662,7 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, for _, header := range request.Headers { taskQueue.Push(header, -float32(header.Number.Uint64())) } - // Add the peer to the expiry report along the the number of failed requests + // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) } } diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 6383596dce..277f14b81a 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -204,7 +204,7 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time } } -// Enqueue tries to fill gaps the the fetcher's future import queue. +// Enqueue tries to fill gaps the fetcher's future import queue. func (f *Fetcher) Enqueue(peer string, block *types.Block) error { op := &inject{ origin: peer, diff --git a/p2p/server.go b/p2p/server.go index 8f3a511f31..f9a38fcc64 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -77,7 +77,7 @@ type Config struct { // Disabling is useful for protocol debugging (manual topology). NoDiscovery bool - // DiscoveryV5 specifies whether the the new topic-discovery based V5 discovery + // DiscoveryV5 specifies whether the new topic-discovery based V5 discovery // protocol should be started or not. DiscoveryV5 bool `toml:",omitempty"` diff --git a/swarm/api/api.go b/swarm/api/api.go index adf469cfaa..0d08eeea87 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -153,7 +153,7 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // Resolve resolves address by choosing a Resolver by TLD. // If there are more default Resolvers, or for a specific TLD, -// the Hash from the the first one which does not return error +// the Hash from the first one which does not return error // will be returned. func (m *MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs, err := m.getResolveValidator(addr) diff --git a/swarm/network/README.md b/swarm/network/README.md index ad429b38be..e9c9d30d0b 100644 --- a/swarm/network/README.md +++ b/swarm/network/README.md @@ -133,7 +133,7 @@ As part of the deletion protocol then, hashes of insured chunks to be removed ar Downstream peer on the other hand needs to make sure that they can only be finger pointed about a chunk they did receive and store. For this the check of a state should be exhaustive. If historical syncing finishes on one state, all hashes before are covered, no surprises. In other words historical syncing this process is self verifying. With session syncing however, it is not enough to check going back covering the range from old offset to new. Continuity (i.e., that the new state is extension of the old) needs to be verified: after downstream peer reads the range into a buffer, it appends the buffer the last known state at the last known offset and verifies the resulting hash matches -the latest state. Past intervals of historical syncing are checked via the the session root. +the latest state. Past intervals of historical syncing are checked via the session root. Upstream peer signs the states, downstream peers can use as handover proofs. Downstream peers sign off on a state together with an initial offset. diff --git a/swarm/network/stream/intervals/intervals.go b/swarm/network/stream/intervals/intervals.go index 5fd820da87..562c3df9ae 100644 --- a/swarm/network/stream/intervals/intervals.go +++ b/swarm/network/stream/intervals/intervals.go @@ -101,7 +101,7 @@ func (i *Intervals) add(start, end uint64) { } } -// Merge adds all the intervals from the the m Interval to current one. +// Merge adds all the intervals from the m Interval to current one. func (i *Intervals) Merge(m *Intervals) { m.mu.RLock() defer m.mu.RUnlock() diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 6acab50af4..4e1ab09fcf 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -182,7 +182,7 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { conf.addrToIDMap[string(a)] = n } - //get the the node at that index + //get the node at that index //this is the node selected for upload node := sim.RandomUpNode() item, ok := sim.NodeItem(node.ID, bucketKeyStore) diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 5a2b25c9ec..8e0662327a 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -291,7 +291,7 @@ func (w *Whisper) AddKeyPair(key *ecdsa.PrivateKey) (string, error) { return id, nil } -// HasKeyPair checks if the the whisper node is configured with the private key +// HasKeyPair checks if the whisper node is configured with the private key // of the specified public pair. func (w *Whisper) HasKeyPair(id string) bool { w.keyMu.RLock() diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 76d485808d..34c1256a6f 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -423,7 +423,7 @@ func (whisper *Whisper) AddKeyPair(key *ecdsa.PrivateKey) (string, error) { return id, nil } -// HasKeyPair checks if the the whisper node is configured with the private key +// HasKeyPair checks if the whisper node is configured with the private key // of the specified public pair. func (whisper *Whisper) HasKeyPair(id string) bool { whisper.keyMu.RLock() From f0488e80f7837094ac74327f0a4f5f6a270641f4 Mon Sep 17 00:00:00 2001 From: Caesar Chad Date: Mon, 27 Aug 2018 16:51:26 +0800 Subject: [PATCH 068/126] signer/storage: fix typo (#17504) --- signer/storage/aes_gcm_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index 225276667f..399637a443 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -36,7 +36,7 @@ type storedCredential struct { CipherText []byte `json:"c"` } -// AESEncryptedStorage is a storage type which is backed by a json-faile. The json-file contains +// AESEncryptedStorage is a storage type which is backed by a json-file. The json-file contains // key-value mappings, where the keys are _not_ encrypted, only the values are. type AESEncryptedStorage struct { // File to read/write credentials From 1acefafe2201ba061a58b51afd7a7edf4aa0f120 Mon Sep 17 00:00:00 2001 From: Geon Kim Date: Mon, 27 Aug 2018 17:51:58 +0900 Subject: [PATCH 069/126] swarm/api: fix typo (#17500) --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 0d08eeea87..d733ad989c 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -111,7 +111,7 @@ func (e *NoResolverError) Error() string { } // MultiResolver is used to resolve URL addresses based on their TLDs. -// Each TLD can have multiple resolvers, and the resoluton from the +// Each TLD can have multiple resolvers, and the resolution from the // first one in the sequence will be returned. type MultiResolver struct { resolvers map[string][]ResolveValidator From f236ac710e075f2d3ef9352a0dc19b3bdf6f5bd8 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 28 Aug 2018 09:03:33 +0200 Subject: [PATCH 070/126] vendor: github.com/rjeczalik/notify update to master (#17527) --- .../rjeczalik/notify/watcher_fsevents_cgo.go | 6 +++--- .../rjeczalik/notify/watcher_fsevents_go1.10.go | 14 -------------- .../rjeczalik/notify/watcher_fsevents_go1.11.go | 9 --------- vendor/vendor.json | 6 +++--- 4 files changed, 6 insertions(+), 29 deletions(-) delete mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go delete mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index 95ee704444..cf0416c37e 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(C.kCFAllocatorDefault, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -166,8 +166,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) - path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(C.kCFAllocatorDefault, C.CString(s.path), C.kCFStringEncodingUTF8, C.kCFAllocatorDefault) + path := C.CFArrayCreate(C.kCFAllocatorDefault, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go deleted file mode 100644 index dd2433d20a..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2018 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,cgo,!go1.11 - -package notify - -/* - #include -*/ -import "C" - -var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go deleted file mode 100644 index 92b406ce7c..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2018 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,go1.11 - -package notify - -const refZero = 0 diff --git a/vendor/vendor.json b/vendor/vendor.json index 0bfc1ca0c1..fea2c804ea 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -370,10 +370,10 @@ "revisionTime": "2017-08-14T17:01:13Z" }, { - "checksumSHA1": "D8AVDI39CJ+jvw0HOotYU2gz54c=", + "checksumSHA1": "lU41NL1TEDtsrr0yUdp3SMB4Y9o=", "path": "github.com/rjeczalik/notify", - "revision": "4e54e7fd043e865c50bda93359fb78813a8d165b", - "revisionTime": "2018-08-08T20:39:25Z" + "revision": "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8", + "revisionTime": "2018-08-27T19:31:19Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", From 0bec85f2e27d7b79f7700643cd9b38e74cecadfa Mon Sep 17 00:00:00 2001 From: Sheldon <11510383@mail.sustc.edu.cn> Date: Tue, 28 Aug 2018 15:04:33 +0800 Subject: [PATCH 071/126] core: fix typos in comment (#17531) --- core/tx_journal.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/tx_journal.go b/core/tx_journal.go index 1397e9fd34..41b5156d4a 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -34,7 +34,7 @@ var errNoActiveJournal = errors.New("no active journal") // devNull is a WriteCloser that just discards anything written into it. Its // goal is to allow the transaction journal to write into a fake journal when // loading transactions on startup without printing warnings due to no file -// being readt for write. +// being read for write. type devNull struct{} func (*devNull) Write(p []byte) (n int, err error) { return len(p), nil } @@ -57,7 +57,7 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. func (journal *txJournal) load(add func([]*types.Transaction) []error) error { - // Skip the parsing if the journal file doens't exist at all + // Skip the parsing if the journal file doesn't exist at all if _, err := os.Stat(journal.path); os.IsNotExist(err) { return nil } @@ -78,7 +78,7 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { // Create a method to load a limited batch of transactions and bump the // appropriate progress counters. Then use this method to load all the - // journalled transactions in small-ish batches. + // journaled transactions in small-ish batches. loadBatch := func(txs types.Transactions) { for _, err := range add(txs) { if err != nil { @@ -103,7 +103,7 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { } break } - // New transaction parsed, queue up for later, import if threnshold is reached + // New transaction parsed, queue up for later, import if threshold is reached total++ if batch = append(batch, tx); batch.Len() > 1024 { From c64d72bea207ccaca3f6aded25d8730a4b8696cd Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Tue, 28 Aug 2018 15:05:25 +0800 Subject: [PATCH 072/126] consensus/ethash: remove unnecessary type declaration (#17529) --- consensus/ethash/consensus.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 259cc5056b..af0e733ae7 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -38,10 +38,10 @@ import ( // Ethash proof-of-work protocol constants. var ( - FrontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - ByzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium - maxUncles = 2 // Maximum number of uncles allowed in a single block - allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks + FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + maxUncles = 2 // Maximum number of uncles allowed in a single block + allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks ) // Various error messages to mark blocks invalid. These should be private to From b69476b372a26679e5bdb33db3d508f2c955e7ff Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 28 Aug 2018 15:08:16 +0800 Subject: [PATCH 073/126] all: make indexer configurable (#17188) --- core/chain_indexer.go | 1 - eth/backend.go | 4 +- eth/bloombits.go | 15 ++-- les/api_backend.go | 2 +- les/backend.go | 13 +-- les/bloombits.go | 4 +- les/commons.go | 6 +- les/handler.go | 13 +-- les/handler_test.go | 177 ++++++++++++++++++++------------------- les/helper_test.go | 159 +++++++++++++++++++++++++++++++---- les/odr.go | 15 +++- les/odr_requests.go | 20 ++++- les/odr_test.go | 45 ++++------ les/peer.go | 30 +++---- les/request_test.go | 43 +++------- les/server.go | 14 ++-- light/lightchain.go | 20 +++-- light/lightchain_test.go | 9 +- light/odr.go | 5 +- light/odr_test.go | 11 ++- light/odr_util.go | 18 ++-- light/postprocess.go | 174 ++++++++++++++++++++++---------------- light/trie_test.go | 2 +- light/txpool_test.go | 2 +- params/network_params.go | 32 ++++++- 25 files changed, 513 insertions(+), 321 deletions(-) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 11a7c96fa0..b80b517d91 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -322,7 +322,6 @@ func (c *ChainIndexer) updateLoop() { updating = false c.log.Info("Finished upgrading chain index") } - c.cascadedHead = c.storedSections*c.sectionSize - 1 for _, child := range c.children { c.log.Trace("Cascading chain index update", "head", c.cascadedHead) diff --git a/eth/backend.go b/eth/backend.go index c530d3c7c3..da7e0b2cd5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -136,7 +136,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { gasPrice: config.MinerGasPrice, etherbase: config.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, bloomConfirms), + bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) @@ -426,7 +426,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { // Ethereum protocol implementation. func (s *Ethereum) Start(srvr *p2p.Server) error { // Start the bloom bits servicing goroutines - s.startBloomHandlers() + s.startBloomHandlers(params.BloomBitsBlocks) // Start the RPC service s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) diff --git a/eth/bloombits.go b/eth/bloombits.go index eb18565e2c..c7bb561402 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" ) const ( @@ -50,7 +49,7 @@ const ( // startBloomHandlers starts a batch of goroutines to accept bloom bit database // retrievals from possibly a range of filters and serving the data to satisfy. -func (eth *Ethereum) startBloomHandlers() { +func (eth *Ethereum) startBloomHandlers(sectionSize uint64) { for i := 0; i < bloomServiceThreads; i++ { go func() { for { @@ -62,9 +61,9 @@ func (eth *Ethereum) startBloomHandlers() { task := <-request task.Bitsets = make([][]byte, len(task.Sections)) for i, section := range task.Sections { - head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) + head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*sectionSize-1) if compVector, err := rawdb.ReadBloomBits(eth.chainDb, task.Bit, section, head); err == nil { - if blob, err := bitutil.DecompressBytes(compVector, int(params.BloomBitsBlocks)/8); err == nil { + if blob, err := bitutil.DecompressBytes(compVector, int(sectionSize/8)); err == nil { task.Bitsets[i] = blob } else { task.Error = err @@ -81,10 +80,6 @@ func (eth *Ethereum) startBloomHandlers() { } const ( - // bloomConfirms is the number of confirmation blocks before a bloom section is - // considered probably final and its rotated bits are calculated. - bloomConfirms = 256 - // bloomThrottling is the time to wait between processing two consecutive index // sections. It's useful during chain upgrades to prevent disk overload. bloomThrottling = 100 * time.Millisecond @@ -102,14 +97,14 @@ type BloomIndexer struct { // NewBloomIndexer returns a chain indexer that generates bloom bits data for the // canonical chain for fast logs filtering. -func NewBloomIndexer(db ethdb.Database, size, confReq uint64) *core.ChainIndexer { +func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *core.ChainIndexer { backend := &BloomIndexer{ db: db, size: size, } table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) - return core.NewChainIndexer(db, table, backend, size, confReq, bloomThrottling, "bloombits") + return core.NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") } // Reset implements core.ChainIndexerBackend, starting a new bloombits index diff --git a/les/api_backend.go b/les/api_backend.go index 4232d3ae07..aa748a4ea1 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -192,7 +192,7 @@ func (b *LesApiBackend) BloomStatus() (uint64, uint64) { return 0, 0 } sections, _, _ := b.eth.bloomIndexer.Sections() - return light.BloomTrieFrequency, sections + return params.BloomBitsBlocksClient, sections } func (b *LesApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { diff --git a/les/backend.go b/les/backend.go index 00025ba634..75049da085 100644 --- a/les/backend.go +++ b/les/backend.go @@ -95,6 +95,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { lesCommons: lesCommons{ chainDb: chainDb, config: config, + iConfig: light.DefaultClientIndexerConfig, }, chainConfig: chainConfig, eventMux: ctx.EventMux, @@ -105,16 +106,16 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { shutdownChan: make(chan bool), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: eth.NewBloomIndexer(chainDb, light.BloomTrieFrequency, light.HelperTrieConfirmations), + bloomIndexer: eth.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), } leth.relay = NewLesTxRelay(peers, leth.reqDist) leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) - leth.odr = NewLesOdr(chainDb, leth.retriever) - leth.chtIndexer = light.NewChtIndexer(chainDb, true, leth.odr) - leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, true, leth.odr) + leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.retriever) + leth.chtIndexer = light.NewChtIndexer(chainDb, leth.odr, params.CHTFrequencyClient, params.HelperTrieConfirmations) + leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, leth.odr, params.BloomBitsBlocksClient, params.BloomTrieFrequency) leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with @@ -135,7 +136,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) - if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { + if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, light.DefaultClientIndexerConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { return nil, err } leth.ApiBackend = &LesApiBackend{leth, nil} @@ -230,8 +231,8 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { // Start implements node.Service, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *LightEthereum) Start(srvr *p2p.Server) error { - s.startBloomHandlers() log.Warn("Light client mode is an experimental feature") + s.startBloomHandlers(params.BloomBitsBlocksClient) s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.networkId) // clients are searching for the first advertised protocol in the list protocolVersion := AdvertiseProtocolVersions[0] diff --git a/les/bloombits.go b/les/bloombits.go index 2871a90064..aea0fcd5f4 100644 --- a/les/bloombits.go +++ b/les/bloombits.go @@ -43,7 +43,7 @@ const ( // startBloomHandlers starts a batch of goroutines to accept bloom bit database // retrievals from possibly a range of filters and serving the data to satisfy. -func (eth *LightEthereum) startBloomHandlers() { +func (eth *LightEthereum) startBloomHandlers(sectionSize uint64) { for i := 0; i < bloomServiceThreads; i++ { go func() { for { @@ -57,7 +57,7 @@ func (eth *LightEthereum) startBloomHandlers() { compVectors, err := light.GetBloomBits(task.Context, eth.odr, task.Bit, task.Sections) if err == nil { for i := range task.Sections { - if blob, err := bitutil.DecompressBytes(compVectors[i], int(light.BloomTrieFrequency/8)); err == nil { + if blob, err := bitutil.DecompressBytes(compVectors[i], int(sectionSize/8)); err == nil { task.Bitsets[i] = blob } else { task.Error = err diff --git a/les/commons.go b/les/commons.go index d8e9412951..a97687993d 100644 --- a/les/commons.go +++ b/les/commons.go @@ -33,6 +33,7 @@ import ( // lesCommons contains fields needed by both server and client. type lesCommons struct { config *eth.Config + iConfig *light.IndexerConfig chainDb ethdb.Database protocolManager *ProtocolManager chtIndexer, bloomTrieIndexer *core.ChainIndexer @@ -81,7 +82,7 @@ func (c *lesCommons) nodeInfo() interface{} { if !c.protocolManager.lightSync { // convert to client section size if running in server mode - sections /= light.CHTFrequencyClient / light.CHTFrequencyServer + sections /= c.iConfig.PairChtSize / c.iConfig.ChtSize } if sections2 < sections { @@ -94,7 +95,8 @@ func (c *lesCommons) nodeInfo() interface{} { if c.protocolManager.lightSync { chtRoot = light.GetChtRoot(c.chainDb, sectionIndex, sectionHead) } else { - chtRoot = light.GetChtV2Root(c.chainDb, sectionIndex, sectionHead) + idxV2 := (sectionIndex+1)*c.iConfig.PairChtSize/c.iConfig.ChtSize - 1 + chtRoot = light.GetChtRoot(c.chainDb, idxV2, sectionHead) } cht = light.TrustedCheckpoint{ SectionIdx: sectionIndex, diff --git a/les/handler.go b/les/handler.go index ca40eaabfa..243a6dabd4 100644 --- a/les/handler.go +++ b/les/handler.go @@ -94,6 +94,7 @@ type ProtocolManager struct { txrelay *LesTxRelay networkId uint64 chainConfig *params.ChainConfig + iConfig *light.IndexerConfig blockchain BlockChain chainDb ethdb.Database odr *LesOdr @@ -123,13 +124,14 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { +func NewProtocolManager(chainConfig *params.ChainConfig, indexerConfig *light.IndexerConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, eventMux: mux, blockchain: blockchain, chainConfig: chainConfig, + iConfig: indexerConfig, chainDb: chainDb, odr: odr, networkId: networkId, @@ -882,7 +884,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*pm.iConfig.ChtSize-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { trie, err := trie.New(root, trieDb) if err != nil { @@ -1137,10 +1139,11 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common. func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) { switch id { case htCanonical: - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) - return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix + idxV1 := (idx+1)*(pm.iConfig.PairChtSize/pm.iConfig.ChtSize) - 1 + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idxV1+1)*pm.iConfig.ChtSize-1) + return light.GetChtRoot(pm.chainDb, idxV1, sectionHead), light.ChtTablePrefix case htBloomBits: - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*pm.iConfig.BloomTrieSize-1) return light.GetBloomTrieRoot(pm.chainDb, idx, sectionHead), light.BloomTrieTablePrefix } return common.Hash{}, "" diff --git a/les/handler_test.go b/les/handler_test.go index 31aad3ed45..43be7f41b1 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -51,10 +51,9 @@ func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, downloader.MaxHashFetch+15, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Create a "random" unknown hash for testing var unknown common.Hash @@ -167,9 +166,9 @@ func testGetBlockHeaders(t *testing.T, protocol int) { } // Send the hash request and verify the response reqID++ - cost := peer.GetRequestCost(GetBlockHeadersMsg, int(tt.query.Amount)) - sendRequest(peer.app, GetBlockHeadersMsg, reqID, cost, tt.query) - if err := expectResponse(peer.app, BlockHeadersMsg, reqID, testBufLimit, headers); err != nil { + cost := server.tPeer.GetRequestCost(GetBlockHeadersMsg, int(tt.query.Amount)) + sendRequest(server.tPeer.app, GetBlockHeadersMsg, reqID, cost, tt.query) + if err := expectResponse(server.tPeer.app, BlockHeadersMsg, reqID, testBufLimit, headers); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } } @@ -180,10 +179,9 @@ func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, downloader.MaxBlockFetch+15, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Create a batch of tests for various scenarios limit := MaxBodyFetch @@ -243,9 +241,9 @@ func testGetBlockBodies(t *testing.T, protocol int) { } reqID++ // Send the hash request and verify the response - cost := peer.GetRequestCost(GetBlockBodiesMsg, len(hashes)) - sendRequest(peer.app, GetBlockBodiesMsg, reqID, cost, hashes) - if err := expectResponse(peer.app, BlockBodiesMsg, reqID, testBufLimit, bodies); err != nil { + cost := server.tPeer.GetRequestCost(GetBlockBodiesMsg, len(hashes)) + sendRequest(server.tPeer.app, GetBlockBodiesMsg, reqID, cost, hashes) + if err := expectResponse(server.tPeer.app, BlockBodiesMsg, reqID, testBufLimit, bodies); err != nil { t.Errorf("test %d: bodies mismatch: %v", i, err) } } @@ -257,10 +255,9 @@ func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, ethdb.NewMemDatabase()) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, 4, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) var codereqs []*CodeReq var codes [][]byte @@ -277,9 +274,9 @@ func testGetCode(t *testing.T, protocol int) { } } - cost := peer.GetRequestCost(GetCodeMsg, len(codereqs)) - sendRequest(peer.app, GetCodeMsg, 42, cost, codereqs) - if err := expectResponse(peer.app, CodeMsg, 42, testBufLimit, codes); err != nil { + cost := server.tPeer.GetRequestCost(GetCodeMsg, len(codereqs)) + sendRequest(server.tPeer.app, GetCodeMsg, 42, cost, codereqs) + if err := expectResponse(server.tPeer.app, CodeMsg, 42, testBufLimit, codes); err != nil { t.Errorf("codes mismatch: %v", err) } } @@ -290,11 +287,9 @@ func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, 4, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Collect the hashes to request, and the response to expect hashes, receipts := []common.Hash{}, []types.Receipts{} @@ -302,12 +297,12 @@ func testGetReceipt(t *testing.T, protocol int) { block := bc.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, rawdb.ReadReceipts(db, block.Hash(), block.NumberU64())) + receipts = append(receipts, rawdb.ReadReceipts(server.db, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response - cost := peer.GetRequestCost(GetReceiptsMsg, len(hashes)) - sendRequest(peer.app, GetReceiptsMsg, 42, cost, hashes) - if err := expectResponse(peer.app, ReceiptsMsg, 42, testBufLimit, receipts); err != nil { + cost := server.tPeer.GetRequestCost(GetReceiptsMsg, len(hashes)) + sendRequest(server.tPeer.app, GetReceiptsMsg, 42, cost, hashes) + if err := expectResponse(server.tPeer.app, ReceiptsMsg, 42, testBufLimit, receipts); err != nil { t.Errorf("receipts mismatch: %v", err) } } @@ -318,11 +313,9 @@ func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, 4, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) var ( proofreqs []ProofReq @@ -334,7 +327,7 @@ func testGetProofs(t *testing.T, protocol int) { for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) root := header.Root - trie, _ := trie.New(root, trie.NewDatabase(db)) + trie, _ := trie.New(root, trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -356,15 +349,15 @@ func testGetProofs(t *testing.T, protocol int) { // Send the proof request and verify the response switch protocol { case 1: - cost := peer.GetRequestCost(GetProofsV1Msg, len(proofreqs)) - sendRequest(peer.app, GetProofsV1Msg, 42, cost, proofreqs) - if err := expectResponse(peer.app, ProofsV1Msg, 42, testBufLimit, proofsV1); err != nil { + cost := server.tPeer.GetRequestCost(GetProofsV1Msg, len(proofreqs)) + sendRequest(server.tPeer.app, GetProofsV1Msg, 42, cost, proofreqs) + if err := expectResponse(server.tPeer.app, ProofsV1Msg, 42, testBufLimit, proofsV1); err != nil { t.Errorf("proofs mismatch: %v", err) } case 2: - cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs)) - sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs) - if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { + cost := server.tPeer.GetRequestCost(GetProofsV2Msg, len(proofreqs)) + sendRequest(server.tPeer.app, GetProofsV2Msg, 42, cost, proofreqs) + if err := expectResponse(server.tPeer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -375,28 +368,33 @@ func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) } func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) } func testGetCHTProofs(t *testing.T, protocol int) { - // Figure out the client's CHT frequency - frequency := uint64(light.CHTFrequencyClient) - if protocol == 1 { - frequency = uint64(light.CHTFrequencyServer) + config := light.TestServerIndexerConfig + frequency := config.ChtSize + if protocol == 2 { + frequency = config.PairChtSize } - // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() - // Wait a while for the CHT indexer to process the new headers - time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling - time.Sleep(250 * time.Millisecond) // CI tester slack + waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + expectSections := frequency / config.ChtSize + for { + cs, _, _ := cIndexer.Sections() + bs, _, _ := bIndexer.Sections() + if cs >= expectSections && bs >= expectSections { + break + } + time.Sleep(10 * time.Millisecond) + } + } + server, tearDown := newServerEnv(t, int(frequency+config.ChtConfirms), protocol, waitIndexers) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Assemble the proofs from the different protocols - header := bc.GetHeaderByNumber(frequency) + header := bc.GetHeaderByNumber(frequency - 1) rlp, _ := rlp.EncodeToBytes(header) key := make([]byte, 8) - binary.BigEndian.PutUint64(key, frequency) + binary.BigEndian.PutUint64(key, frequency-1) proofsV1 := []ChtResp{{ Header: header, @@ -406,41 +404,41 @@ func testGetCHTProofs(t *testing.T, protocol int) { } switch protocol { case 1: - root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.ChtTablePrefix))) var proof light.NodeList trie.Prove(key, 0, &proof) proofsV1[0].Proof = proof case 2: - root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + root := light.GetChtRoot(server.db, (frequency/config.ChtSize)-1, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) } // Assemble the requests for the different protocols requestsV1 := []ChtReq{{ - ChtNum: 1, - BlockNum: frequency, + ChtNum: frequency / config.ChtSize, + BlockNum: frequency - 1, }} requestsV2 := []HelperTrieReq{{ Type: htCanonical, - TrieIdx: 0, + TrieIdx: frequency/config.PairChtSize - 1, Key: key, AuxReq: auxHeader, }} // Send the proof request and verify the response switch protocol { case 1: - cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1)) - sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1) - if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil { + cost := server.tPeer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1)) + sendRequest(server.tPeer.app, GetHeaderProofsMsg, 42, cost, requestsV1) + if err := expectResponse(server.tPeer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil { t.Errorf("proofs mismatch: %v", err) } case 2: - cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2)) - sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2) - if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { + cost := server.tPeer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2)) + sendRequest(server.tPeer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2) + if err := expectResponse(server.tPeer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -448,24 +446,31 @@ func testGetCHTProofs(t *testing.T, protocol int) { // Tests that bloombits proofs can be correctly retrieved. func TestGetBloombitsProofs(t *testing.T) { - // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", 2, pm, true) - defer peer.close() + config := light.TestServerIndexerConfig - // Wait a while for the bloombits indexer to process the new headers - time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling - time.Sleep(250 * time.Millisecond) // CI tester slack + waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + cs, _, _ := cIndexer.Sections() + bs, _, _ := bIndexer.Sections() + bts, _, _ := btIndexer.Sections() + if cs >= 8 && bs >= 8 && bts >= 1 { + break + } + time.Sleep(10 * time.Millisecond) + } + } + server, tearDown := newServerEnv(t, int(config.BloomTrieSize+config.BloomTrieConfirms), 2, waitIndexers) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Request and verify each bit of the bloom bits proofs for bit := 0; bit < 2048; bit++ { - // Assemble therequest and proofs for the bloombits + // Assemble the request and proofs for the bloombits key := make([]byte, 10) binary.BigEndian.PutUint16(key[:2], uint16(bit)) - binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency)) + // Only the first bloom section has data. + binary.BigEndian.PutUint64(key[2:], 0) requests := []HelperTrieReq{{ Type: htBloomBits, @@ -474,14 +479,14 @@ func TestGetBloombitsProofs(t *testing.T) { }} var proofs HelperTrieResps - root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix))) + root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response - cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) - sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests) - if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { + cost := server.tPeer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) + sendRequest(server.tPeer.app, GetHelperTrieProofsMsg, 42, cost, requests) + if err := expectResponse(server.tPeer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { t.Errorf("bit %d: proofs mismatch: %v", bit, err) } } diff --git a/les/helper_test.go b/les/helper_test.go index 8817c20c7d..206ee2d920 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -24,6 +24,7 @@ import ( "math/big" "sync" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -123,6 +124,15 @@ func testChainGen(i int, block *core.BlockGen) { } } +// testIndexers creates a set of indexers with specified params for testing purpose. +func testIndexers(db ethdb.Database, odr light.OdrBackend, iConfig *light.IndexerConfig) (*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer) { + chtIndexer := light.NewChtIndexer(db, odr, iConfig.ChtSize, iConfig.ChtConfirms) + bloomIndexer := eth.NewBloomIndexer(db, iConfig.BloomSize, iConfig.BloomConfirms) + bloomTrieIndexer := light.NewBloomTrieIndexer(db, odr, iConfig.BloomSize, iConfig.BloomTrieSize) + bloomIndexer.AddChildIndexer(bloomTrieIndexer) + return chtIndexer, bloomIndexer, bloomTrieIndexer +} + func testRCL() RequestCostList { cl := make(RequestCostList, len(reqList)) for i, code := range reqList { @@ -134,9 +144,9 @@ func testRCL() RequestCostList { } // newTestProtocolManager creates a new protocol manager for testing purposes, -// with the given number of blocks already known, and potential notification -// channels for different events. -func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *core.BlockGen), peers *peerSet, odr *LesOdr, db ethdb.Database) (*ProtocolManager, error) { +// with the given number of blocks already known, potential notification +// channels for different events and relative chain indexers array. +func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database) (*ProtocolManager, error) { var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() @@ -155,16 +165,6 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain, _ = light.NewLightChain(odr, gspec.Config, engine) } else { blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) - - chtIndexer := light.NewChtIndexer(db, false, nil) - chtIndexer.Start(blockchain) - - bbtIndexer := light.NewBloomTrieIndexer(db, false, nil) - - bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks, light.HelperTrieProcessConfirmations) - bloomIndexer.AddChildIndexer(bbtIndexer) - bloomIndexer.Start(blockchain) - gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) @@ -172,7 +172,11 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain = blockchain } - pm, err := NewProtocolManager(gspec.Config, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) + indexConfig := light.TestServerIndexerConfig + if lightSync { + indexConfig = light.TestClientIndexerConfig + } + pm, err := NewProtocolManager(gspec.Config, indexConfig, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) if err != nil { return nil, err } @@ -193,11 +197,11 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor } // newTestProtocolManagerMust creates a new protocol manager for testing purposes, -// with the given number of blocks already known, and potential notification -// channels for different events. In case of an error, the constructor force- +// with the given number of blocks already known, potential notification +// channels for different events and relative chain indexers array. In case of an error, the constructor force- // fails the test. -func newTestProtocolManagerMust(t *testing.T, lightSync bool, blocks int, generator func(int, *core.BlockGen), peers *peerSet, odr *LesOdr, db ethdb.Database) *ProtocolManager { - pm, err := newTestProtocolManager(lightSync, blocks, generator, peers, odr, db) +func newTestProtocolManagerMust(t *testing.T, lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database) *ProtocolManager { + pm, err := newTestProtocolManager(lightSync, blocks, generator, odr, peers, db) if err != nil { t.Fatalf("Failed to create protocol manager: %v", err) } @@ -320,3 +324,122 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu func (p *testPeer) close() { p.app.Close() } + +// TestEntity represents a network entity for testing with necessary auxiliary fields. +type TestEntity struct { + db ethdb.Database + rPeer *peer + tPeer *testPeer + peers *peerSet + pm *ProtocolManager + // Indexers + chtIndexer *core.ChainIndexer + bloomIndexer *core.ChainIndexer + bloomTrieIndexer *core.ChainIndexer +} + +// newServerEnv creates a server testing environment with a connected test peer for testing purpose. +func newServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer)) (*TestEntity, func()) { + db := ethdb.NewMemDatabase() + cIndexer, bIndexer, btIndexer := testIndexers(db, nil, light.TestServerIndexerConfig) + + pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, nil, db) + peer, _ := newTestPeer(t, "peer", protocol, pm, true) + + cIndexer.Start(pm.blockchain.(*core.BlockChain)) + bIndexer.Start(pm.blockchain.(*core.BlockChain)) + + // Wait until indexers generate enough index data. + if waitIndexers != nil { + waitIndexers(cIndexer, bIndexer, btIndexer) + } + + return &TestEntity{ + db: db, + tPeer: peer, + pm: pm, + chtIndexer: cIndexer, + bloomIndexer: bIndexer, + bloomTrieIndexer: btIndexer, + }, func() { + peer.close() + // Note bloom trie indexer will be closed by it parent recursively. + cIndexer.Close() + bIndexer.Close() + } +} + +// newClientServerEnv creates a client/server arch environment with a connected les server and light client pair +// for testing purpose. +func newClientServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer), newPeer bool) (*TestEntity, *TestEntity, func()) { + db, ldb := ethdb.NewMemDatabase(), ethdb.NewMemDatabase() + peers, lPeers := newPeerSet(), newPeerSet() + + dist := newRequestDistributor(lPeers, make(chan struct{})) + rm := newRetrieveManager(lPeers, dist, nil) + odr := NewLesOdr(ldb, light.TestClientIndexerConfig, rm) + + cIndexer, bIndexer, btIndexer := testIndexers(db, nil, light.TestServerIndexerConfig) + lcIndexer, lbIndexer, lbtIndexer := testIndexers(ldb, odr, light.TestClientIndexerConfig) + odr.SetIndexers(lcIndexer, lbtIndexer, lbIndexer) + + pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, peers, db) + lpm := newTestProtocolManagerMust(t, true, 0, nil, odr, lPeers, ldb) + + startIndexers := func(clientMode bool, pm *ProtocolManager) { + if clientMode { + lcIndexer.Start(pm.blockchain.(*light.LightChain)) + lbIndexer.Start(pm.blockchain.(*light.LightChain)) + } else { + cIndexer.Start(pm.blockchain.(*core.BlockChain)) + bIndexer.Start(pm.blockchain.(*core.BlockChain)) + } + } + + startIndexers(false, pm) + startIndexers(true, lpm) + + // Execute wait until function if it is specified. + if waitIndexers != nil { + waitIndexers(cIndexer, bIndexer, btIndexer) + } + + var ( + peer, lPeer *peer + err1, err2 <-chan error + ) + if newPeer { + peer, err1, lPeer, err2 = newTestPeerPair("peer", protocol, pm, lpm) + select { + case <-time.After(time.Millisecond * 100): + case err := <-err1: + t.Fatalf("peer 1 handshake error: %v", err) + case err := <-err2: + t.Fatalf("peer 2 handshake error: %v", err) + } + } + + return &TestEntity{ + db: db, + pm: pm, + rPeer: peer, + peers: peers, + chtIndexer: cIndexer, + bloomIndexer: bIndexer, + bloomTrieIndexer: btIndexer, + }, &TestEntity{ + db: ldb, + pm: lpm, + rPeer: lPeer, + peers: lPeers, + chtIndexer: lcIndexer, + bloomIndexer: lbIndexer, + bloomTrieIndexer: lbtIndexer, + }, func() { + // Note bloom trie indexers will be closed by their parents recursively. + cIndexer.Close() + bIndexer.Close() + lcIndexer.Close() + lbIndexer.Close() + } +} diff --git a/les/odr.go b/les/odr.go index 2ad28d5d9f..9def05a676 100644 --- a/les/odr.go +++ b/les/odr.go @@ -28,16 +28,18 @@ import ( // LesOdr implements light.OdrBackend type LesOdr struct { db ethdb.Database + indexerConfig *light.IndexerConfig chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer retriever *retrieveManager stop chan struct{} } -func NewLesOdr(db ethdb.Database, retriever *retrieveManager) *LesOdr { +func NewLesOdr(db ethdb.Database, config *light.IndexerConfig, retriever *retrieveManager) *LesOdr { return &LesOdr{ - db: db, - retriever: retriever, - stop: make(chan struct{}), + db: db, + indexerConfig: config, + retriever: retriever, + stop: make(chan struct{}), } } @@ -73,6 +75,11 @@ func (odr *LesOdr) BloomIndexer() *core.ChainIndexer { return odr.bloomIndexer } +// IndexerConfig returns the indexer config. +func (odr *LesOdr) IndexerConfig() *light.IndexerConfig { + return odr.indexerConfig +} + const ( MsgBlockBodies = iota MsgCode diff --git a/les/odr_requests.go b/les/odr_requests.go index 075fcd92ca..9e9b2673f9 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -365,7 +365,7 @@ func (r *ChtRequest) CanSend(peer *peer) bool { peer.lock.RLock() defer peer.lock.RUnlock() - return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.CHTFrequencyClient + return peer.headInfo.Number >= r.Config.ChtConfirms && r.ChtNum <= (peer.headInfo.Number-r.Config.ChtConfirms)/r.Config.ChtSize } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -379,7 +379,21 @@ func (r *ChtRequest) Request(reqID uint64, peer *peer) error { Key: encNum[:], AuxReq: auxHeader, } - return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req}) + switch peer.version { + case lpv1: + var reqsV1 ChtReq + if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { + return fmt.Errorf("Request invalid in LES/1 mode") + } + blockNum := binary.BigEndian.Uint64(req.Key) + // convert HelperTrie request to old CHT request + reqsV1 = ChtReq{ChtNum: (req.TrieIdx + 1) * (r.Config.ChtSize / r.Config.PairChtSize), BlockNum: blockNum, FromLevel: req.FromLevel} + return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []ChtReq{reqsV1}) + case lpv2: + return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req}) + default: + panic(nil) + } } // Valid processes an ODR request reply message from the LES network @@ -484,7 +498,7 @@ func (r *BloomRequest) CanSend(peer *peer) bool { if peer.version < lpv2 { return false } - return peer.headInfo.Number >= light.HelperTrieConfirmations && r.BloomTrieNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.BloomTrieFrequency + return peer.headInfo.Number >= r.Config.BloomTrieConfirms && r.BloomTrieNum <= (peer.headInfo.Number-r.Config.BloomTrieConfirms)/r.Config.BloomTrieSize } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) diff --git a/les/odr_test.go b/les/odr_test.go index c7c25cbe4b..e6458adf56 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/params" @@ -160,36 +159,21 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai return res } +// testOdr tests odr requests whose validation guaranteed by block headers. func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { // Assemble the test environment - peers := newPeerSet() - dist := newRequestDistributor(peers, make(chan struct{})) - rm := newRetrieveManager(peers, dist, nil) - db := ethdb.NewMemDatabase() - ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, rm) - odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) - _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) - select { - case <-time.After(time.Millisecond * 100): - case err := <-err1: - t.Fatalf("peer 1 handshake error: %v", err) - case err := <-err2: - t.Fatalf("peer 1 handshake error: %v", err) - } - - lpm.synchronise(lpeer) + server, client, tearDown := newClientServerEnv(t, 4, protocol, nil, true) + defer tearDown() + client.pm.synchronise(client.rPeer) test := func(expFail uint64) { - for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := rawdb.ReadCanonicalHash(db, i) - b1 := fn(light.NoOdr, db, pm.chainConfig, pm.blockchain.(*core.BlockChain), nil, bhash) + for i := uint64(0); i <= server.pm.blockchain.CurrentHeader().Number.Uint64(); i++ { + bhash := rawdb.ReadCanonicalHash(server.db, i) + b1 := fn(light.NoOdr, server.db, server.pm.chainConfig, server.pm.blockchain.(*core.BlockChain), nil, bhash) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - b2 := fn(ctx, ldb, lpm.chainConfig, nil, lpm.blockchain.(*light.LightChain), bhash) + b2 := fn(ctx, client.db, client.pm.chainConfig, nil, client.pm.blockchain.(*light.LightChain), bhash) eq := bytes.Equal(b1, b2) exp := i < expFail @@ -201,21 +185,20 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { } } } - // temporarily remove peer to test odr fails // expect retrievals to fail (except genesis block) without a les peer - peers.Unregister(lpeer.id) + client.peers.Unregister(client.rPeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed test(expFail) // expect all retrievals to pass - peers.Register(lpeer) + client.peers.Register(client.rPeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed - lpeer.lock.Lock() - lpeer.hasBlock = func(common.Hash, uint64) bool { return true } - lpeer.lock.Unlock() + client.peers.lock.Lock() + client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true } + client.peers.lock.Unlock() test(5) // still expect all retrievals to pass, now data should be cached locally - peers.Unregister(lpeer.id) + client.peers.Unregister(client.rPeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed test(5) } diff --git a/les/peer.go b/les/peer.go index eb7452e276..70c863c2ff 100644 --- a/les/peer.go +++ b/les/peer.go @@ -19,7 +19,6 @@ package les import ( "crypto/ecdsa" - "encoding/binary" "errors" "fmt" "math/big" @@ -36,9 +35,10 @@ import ( ) var ( - errClosed = errors.New("peer set is closed") - errAlreadyRegistered = errors.New("peer is already registered") - errNotRegistered = errors.New("peer is not registered") + errClosed = errors.New("peer set is closed") + errAlreadyRegistered = errors.New("peer is already registered") + errNotRegistered = errors.New("peer is not registered") + errInvalidHelpTrieReq = errors.New("invalid help trie request") ) const maxResponseErrors = 50 // number of invalid responses tolerated (makes the protocol less brittle but still avoids spam) @@ -284,21 +284,21 @@ func (p *peer) RequestProofs(reqID, cost uint64, reqs []ProofReq) error { } // RequestHelperTrieProofs fetches a batch of HelperTrie merkle proofs from a remote node. -func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, reqs []HelperTrieReq) error { - p.Log().Debug("Fetching batch of HelperTrie proofs", "count", len(reqs)) +func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, data interface{}) error { switch p.version { case lpv1: - reqsV1 := make([]ChtReq, len(reqs)) - for i, req := range reqs { - if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { - return fmt.Errorf("Request invalid in LES/1 mode") - } - blockNum := binary.BigEndian.Uint64(req.Key) - // convert HelperTrie request to old CHT request - reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.CHTFrequencyClient / light.CHTFrequencyServer), BlockNum: blockNum, FromLevel: req.FromLevel} + reqs, ok := data.([]ChtReq) + if !ok { + return errInvalidHelpTrieReq } - return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqsV1) + p.Log().Debug("Fetching batch of header proofs", "count", len(reqs)) + return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqs) case lpv2: + reqs, ok := data.([]HelperTrieReq) + if !ok { + return errInvalidHelpTrieReq + } + p.Log().Debug("Fetching batch of HelperTrie proofs", "count", len(reqs)) return sendRequest(p.rw, GetHelperTrieProofsMsg, reqID, cost, reqs) default: panic(nil) diff --git a/les/request_test.go b/les/request_test.go index db576798b4..f02c2a3d76 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" ) @@ -84,35 +83,17 @@ func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrReq func testAccess(t *testing.T, protocol int, fn accessTestFn) { // Assemble the test environment - peers := newPeerSet() - dist := newRequestDistributor(peers, make(chan struct{})) - rm := newRetrieveManager(peers, dist, nil) - db := ethdb.NewMemDatabase() - ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, rm) - odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) - - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) - _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) - select { - case <-time.After(time.Millisecond * 100): - case err := <-err1: - t.Fatalf("peer 1 handshake error: %v", err) - case err := <-err2: - t.Fatalf("peer 1 handshake error: %v", err) - } - - lpm.synchronise(lpeer) + server, client, tearDown := newClientServerEnv(t, 4, protocol, nil, true) + defer tearDown() + client.pm.synchronise(client.rPeer) test := func(expFail uint64) { - for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := rawdb.ReadCanonicalHash(db, i) - if req := fn(ldb, bhash, i); req != nil { + for i := uint64(0); i <= server.pm.blockchain.CurrentHeader().Number.Uint64(); i++ { + bhash := rawdb.ReadCanonicalHash(server.db, i) + if req := fn(client.db, bhash, i); req != nil { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - - err := odr.Retrieve(ctx, req) + err := client.pm.odr.Retrieve(ctx, req) got := err == nil exp := i < expFail if exp && !got { @@ -126,16 +107,16 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { } // temporarily remove peer to test odr fails - peers.Unregister(lpeer.id) + client.peers.Unregister(client.rPeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed // expect retrievals to fail (except genesis block) without a les peer test(0) - peers.Register(lpeer) + client.peers.Register(client.rPeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed - lpeer.lock.Lock() - lpeer.hasBlock = func(common.Hash, uint64) bool { return true } - lpeer.lock.Unlock() + client.rPeer.lock.Lock() + client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true } + client.rPeer.lock.Unlock() // expect all retrievals to pass test(5) } diff --git a/les/server.go b/les/server.go index df98d1e3a8..2fa0456d69 100644 --- a/les/server.go +++ b/les/server.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discv5" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -50,7 +51,7 @@ type LesServer struct { func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync := make(chan struct{}) - pm, err := NewProtocolManager(eth.BlockChain().Config(), false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) + pm, err := NewProtocolManager(eth.BlockChain().Config(), light.DefaultServerIndexerConfig, false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) if err != nil { return nil, err } @@ -64,8 +65,9 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { lesCommons: lesCommons{ config: config, chainDb: eth.ChainDb(), - chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), - bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), + iConfig: light.DefaultServerIndexerConfig, + chtIndexer: light.NewChtIndexer(eth.ChainDb(), nil, params.CHTFrequencyServer, params.HelperTrieProcessConfirmations), + bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency), protocolManager: pm, }, quitSync: quitSync, @@ -75,14 +77,14 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { logger := log.New() chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility - chtV2SectionCount := chtV1SectionCount / (light.CHTFrequencyClient / light.CHTFrequencyServer) + chtV2SectionCount := chtV1SectionCount / (params.CHTFrequencyClient / params.CHTFrequencyServer) if chtV2SectionCount != 0 { // convert to LES/2 section chtLastSection := chtV2SectionCount - 1 // convert last LES/2 section index back to LES/1 index for chtIndexer.SectionHead - chtLastSectionV1 := (chtLastSection+1)*(light.CHTFrequencyClient/light.CHTFrequencyServer) - 1 + chtLastSectionV1 := (chtLastSection+1)*(params.CHTFrequencyClient/params.CHTFrequencyServer) - 1 chtSectionHead := srv.chtIndexer.SectionHead(chtLastSectionV1) - chtRoot := light.GetChtV2Root(pm.chainDb, chtLastSection, chtSectionHead) + chtRoot := light.GetChtRoot(pm.chainDb, chtLastSectionV1, chtSectionHead) logger.Info("Loaded CHT", "section", chtLastSection, "head", chtSectionHead, "root", chtRoot) } bloomTrieSectionCount, _, _ := srv.bloomTrieIndexer.Sections() diff --git a/light/lightchain.go b/light/lightchain.go index b5afe1f0e6..bd798eca24 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -48,6 +48,7 @@ var ( // interface. It only does header validation during chain insertion. type LightChain struct { hc *core.HeaderChain + indexerConfig *IndexerConfig chainDb ethdb.Database odr OdrBackend chainFeed event.Feed @@ -81,13 +82,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. blockCache, _ := lru.New(blockCacheLimit) bc := &LightChain{ - chainDb: odr.Database(), - odr: odr, - quit: make(chan struct{}), - bodyCache: bodyCache, - bodyRLPCache: bodyRLPCache, - blockCache: blockCache, - engine: engine, + chainDb: odr.Database(), + indexerConfig: odr.IndexerConfig(), + odr: odr, + quit: make(chan struct{}), + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + blockCache: blockCache, + engine: engine, } var err error bc.hc, err = core.NewHeaderChain(odr.Database(), config, bc.engine, bc.getProcInterrupt) @@ -128,7 +130,7 @@ func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) { if self.odr.BloomIndexer() != nil { self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } - log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*CHTFrequencyClient-1, "hash", cp.SectionHead) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*self.indexerConfig.ChtSize-1, "hash", cp.SectionHead) } func (self *LightChain) getProcInterrupt() bool { @@ -472,7 +474,7 @@ func (self *LightChain) SyncCht(ctx context.Context) bool { head := self.CurrentHeader().Number.Uint64() sections, _, _ := self.odr.ChtIndexer().Sections() - latest := sections*CHTFrequencyClient - 1 + latest := sections*self.indexerConfig.ChtSize - 1 if clique := self.hc.Config().Clique; clique != nil { latest -= latest % clique.Epoch // epoch snapshot for clique } diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 5f0baaf4cd..d45c0656d9 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -55,7 +55,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { db := ethdb.NewMemDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) - blockchain, _ := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFaker()) + blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker()) // Create and inject the requested chain if n == 0 { @@ -265,7 +265,8 @@ func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types. type dummyOdr struct { OdrBackend - db ethdb.Database + db ethdb.Database + indexerConfig *IndexerConfig } func (odr *dummyOdr) Database() ethdb.Database { @@ -276,6 +277,10 @@ func (odr *dummyOdr) Retrieve(ctx context.Context, req OdrRequest) error { return nil } +func (odr *dummyOdr) IndexerConfig() *IndexerConfig { + return odr.indexerConfig +} + // Tests that reorganizing a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. func TestReorgLongHeaders(t *testing.T) { diff --git a/light/odr.go b/light/odr.go index 83c64055a2..3cd8b2c040 100644 --- a/light/odr.go +++ b/light/odr.go @@ -44,6 +44,7 @@ type OdrBackend interface { BloomTrieIndexer() *core.ChainIndexer BloomIndexer() *core.ChainIndexer Retrieve(ctx context.Context, req OdrRequest) error + IndexerConfig() *IndexerConfig } // OdrRequest is an interface for retrieval requests @@ -136,6 +137,7 @@ func (req *ReceiptsRequest) StoreResult(db ethdb.Database) { // ChtRequest is the ODR request type for state/storage trie entries type ChtRequest struct { OdrRequest + Config *IndexerConfig ChtNum, BlockNum uint64 ChtRoot common.Hash Header *types.Header @@ -155,6 +157,7 @@ func (req *ChtRequest) StoreResult(db ethdb.Database) { // BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure type BloomRequest struct { OdrRequest + Config *IndexerConfig BloomTrieNum uint64 BitIdx uint SectionIdxList []uint64 @@ -166,7 +169,7 @@ type BloomRequest struct { // StoreResult stores the retrieved data in local database func (req *BloomRequest) StoreResult(db ethdb.Database) { for i, sectionIdx := range req.SectionIdxList { - sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*req.Config.BloomTrieSize-1) // if we don't have the canonical hash stored for this section head number, we'll still store it under // a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical // hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the diff --git a/light/odr_test.go b/light/odr_test.go index 3e7ac10118..eea5b1eab1 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -55,8 +55,9 @@ var ( type testOdr struct { OdrBackend - sdb, ldb ethdb.Database - disable bool + indexerConfig *IndexerConfig + sdb, ldb ethdb.Database + disable bool } func (odr *testOdr) Database() ethdb.Database { @@ -92,6 +93,10 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { return nil } +func (odr *testOdr) IndexerConfig() *IndexerConfig { + return odr.indexerConfig +} + type odrTestFn func(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) func TestOdrGetBlockLes1(t *testing.T) { testChainOdr(t, 1, odrGetBlock) } @@ -258,7 +263,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { t.Fatal(err) } - odr := &testOdr{sdb: sdb, ldb: ldb} + odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker()) if err != nil { t.Fatal(err) diff --git a/light/odr_util.go b/light/odr_util.go index 620af63835..9bc0f604b0 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -53,16 +53,16 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { chtCount-- if chtCount > 0 { - sectionHeadNum = chtCount*CHTFrequencyClient - 1 + sectionHeadNum = chtCount*odr.IndexerConfig().ChtSize - 1 sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1) canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } } } - if number >= chtCount*CHTFrequencyClient { + if number >= chtCount*odr.IndexerConfig().ChtSize { return nil, ErrNoTrustedCht } - r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number} + r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number, Config: odr.IndexerConfig()} if err := odr.Retrieve(ctx, r); err != nil { return nil, err } @@ -175,9 +175,9 @@ func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number // GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxList []uint64) ([][]byte, error) { - db := odr.Database() - result := make([][]byte, len(sectionIdxList)) var ( + db = odr.Database() + result = make([][]byte, len(sectionIdxList)) reqList []uint64 reqIdx []int ) @@ -193,7 +193,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { bloomTrieCount-- if bloomTrieCount > 0 { - sectionHeadNum = bloomTrieCount*BloomTrieFrequency - 1 + sectionHeadNum = bloomTrieCount*odr.IndexerConfig().BloomTrieSize - 1 sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1) canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } @@ -201,7 +201,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi } for i, sectionIdx := range sectionIdxList { - sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*odr.IndexerConfig().BloomSize-1) // if we don't have the canonical hash stored for this section head number, we'll still look for // an entry with a zero sectionHead (we store it with zero section head too if we don't know it // at the time of the retrieval) @@ -209,6 +209,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi if err == nil { result[i] = bloomBits } else { + // TODO(rjl493456442) Convert sectionIndex to BloomTrie relative index if sectionIdx >= bloomTrieCount { return nil, ErrNoTrustedBloomTrie } @@ -220,7 +221,8 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi return result, nil } - r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1, BitIdx: bitIdx, SectionIdxList: reqList} + r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1, + BitIdx: bitIdx, SectionIdxList: reqList, Config: odr.IndexerConfig()} if err := odr.Retrieve(ctx, r); err != nil { return nil, err } else { diff --git a/light/postprocess.go b/light/postprocess.go index f105d57b55..7b23e48b5b 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -36,20 +36,75 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -const ( - // CHTFrequencyClient is the block frequency for creating CHTs on the client side. - CHTFrequencyClient = 32768 +// IndexerConfig includes a set of configs for chain indexers. +type IndexerConfig struct { + // The block frequency for creating CHTs. + ChtSize uint64 - // CHTFrequencyServer is the block frequency for creating CHTs on the server side. - // Eventually this can be merged back with the client version, but that requires a - // full database upgrade, so that should be left for a suitable moment. - CHTFrequencyServer = 4096 + // A special auxiliary field represents client's chtsize for server config, otherwise represents server's chtsize. + PairChtSize uint64 - HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available - HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated + // The number of confirmations needed to generate/accept a canonical hash help trie. + ChtConfirms uint64 + + // The block frequency for creating new bloom bits. + BloomSize uint64 + + // The number of confirmation needed before a bloom section is considered probably final and its rotated bits + // are calculated. + BloomConfirms uint64 + + // The block frequency for creating BloomTrie. + BloomTrieSize uint64 + + // The number of confirmations needed to generate/accept a bloom trie. + BloomTrieConfirms uint64 +} + +var ( + // DefaultServerIndexerConfig wraps a set of configs as a default indexer config for server side. + DefaultServerIndexerConfig = &IndexerConfig{ + ChtSize: params.CHTFrequencyServer, + PairChtSize: params.CHTFrequencyClient, + ChtConfirms: params.HelperTrieProcessConfirmations, + BloomSize: params.BloomBitsBlocks, + BloomConfirms: params.BloomConfirms, + BloomTrieSize: params.BloomTrieFrequency, + BloomTrieConfirms: params.HelperTrieProcessConfirmations, + } + // DefaultClientIndexerConfig wraps a set of configs as a default indexer config for client side. + DefaultClientIndexerConfig = &IndexerConfig{ + ChtSize: params.CHTFrequencyClient, + PairChtSize: params.CHTFrequencyServer, + ChtConfirms: params.HelperTrieConfirmations, + BloomSize: params.BloomBitsBlocksClient, + BloomConfirms: params.HelperTrieConfirmations, + BloomTrieSize: params.BloomTrieFrequency, + BloomTrieConfirms: params.HelperTrieConfirmations, + } + // TestServerIndexerConfig wraps a set of configs as a test indexer config for server side. + TestServerIndexerConfig = &IndexerConfig{ + ChtSize: 256, + PairChtSize: 2048, + ChtConfirms: 16, + BloomSize: 256, + BloomConfirms: 16, + BloomTrieSize: 2048, + BloomTrieConfirms: 16, + } + // TestClientIndexerConfig wraps a set of configs as a test indexer config for client side. + TestClientIndexerConfig = &IndexerConfig{ + ChtSize: 2048, + PairChtSize: 256, + ChtConfirms: 128, + BloomSize: 2048, + BloomConfirms: 128, + BloomTrieSize: 2048, + BloomTrieConfirms: 128, + } ) -// TrustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with +// trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with // the appropriate section index and head hash. It is used to start light syncing from this checkpoint // and avoid downloading the entire header chain while still being able to securely access old headers/logs. type TrustedCheckpoint struct { @@ -84,9 +139,9 @@ var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ } var ( - ErrNoTrustedCht = errors.New("No trusted canonical hash trie") - ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") - ErrNoHeader = errors.New("Header not found") + ErrNoTrustedCht = errors.New("no trusted canonical hash trie") + ErrNoTrustedBloomTrie = errors.New("no trusted bloom trie") + ErrNoHeader = errors.New("header not found") chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash ChtTablePrefix = "cht-" ) @@ -97,8 +152,8 @@ type ChtNode struct { Td *big.Int } -// GetChtRoot reads the CHT root assoctiated to the given section from the database -// Note that sectionIdx is specified according to LES/1 CHT section size +// GetChtRoot reads the CHT root associated to the given section from the database +// Note that sectionIdx is specified according to LES/1 CHT section size. func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -106,21 +161,15 @@ func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) c return common.BytesToHash(data) } -// GetChtV2Root reads the CHT root assoctiated to the given section from the database -// Note that sectionIdx is specified according to LES/2 CHT section size -func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { - return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) -} - -// StoreChtRoot writes the CHT root assoctiated to the given section into the database -// Note that sectionIdx is specified according to LES/1 CHT section size +// StoreChtRoot writes the CHT root associated to the given section into the database +// Note that sectionIdx is specified according to LES/1 CHT section size. func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) } -// ChtIndexerBackend implements core.ChainIndexerBackend +// ChtIndexerBackend implements core.ChainIndexerBackend. type ChtIndexerBackend struct { diskdb, trieTable ethdb.Database odr OdrBackend @@ -130,33 +179,24 @@ type ChtIndexerBackend struct { trie *trie.Trie } -// NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewChtIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { - var sectionSize, confirmReq uint64 - if clientMode { - sectionSize = CHTFrequencyClient - confirmReq = HelperTrieConfirmations - } else { - sectionSize = CHTFrequencyServer - confirmReq = HelperTrieProcessConfirmations - } - idb := ethdb.NewTable(db, "chtIndex-") +// NewChtIndexer creates a Cht chain indexer +func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *core.ChainIndexer { trieTable := ethdb.NewTable(db, ChtTablePrefix) backend := &ChtIndexerBackend{ diskdb: db, odr: odr, trieTable: trieTable, triedb: trie.NewDatabase(trieTable), - sectionSize: sectionSize, + sectionSize: size, } - return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") + return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") } // fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the // ODR backend in order to be able to add new entries and calculate subsequent root hashes func (c *ChtIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { batch := c.trieTable.NewBatch() - r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1} + r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1, Config: c.odr.IndexerConfig()} for { err := c.odr.Retrieve(ctx, r) switch err { @@ -221,18 +261,13 @@ func (c *ChtIndexerBackend) Commit() error { } c.triedb.Commit(root, false) - if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { - log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) + if ((c.section+1)*c.sectionSize)%params.CHTFrequencyClient == 0 { + log.Info("Storing CHT", "section", c.section*c.sectionSize/params.CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil } -const ( - BloomTrieFrequency = 32768 - ethBloomBitsSection = 4096 -) - var ( bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash BloomTrieTablePrefix = "blt-" @@ -255,33 +290,31 @@ func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root // BloomTrieIndexerBackend implements core.ChainIndexerBackend type BloomTrieIndexerBackend struct { - diskdb, trieTable ethdb.Database - odr OdrBackend - triedb *trie.Database - section, parentSectionSize, bloomTrieRatio uint64 - trie *trie.Trie - sectionHeads []common.Hash + diskdb, trieTable ethdb.Database + triedb *trie.Database + odr OdrBackend + section uint64 + parentSize uint64 + size uint64 + bloomTrieRatio uint64 + trie *trie.Trie + sectionHeads []common.Hash } // NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewBloomTrieIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { +func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uint64) *core.ChainIndexer { trieTable := ethdb.NewTable(db, BloomTrieTablePrefix) backend := &BloomTrieIndexerBackend{ - diskdb: db, - odr: odr, - trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + diskdb: db, + odr: odr, + trieTable: trieTable, + triedb: trie.NewDatabase(trieTable), + parentSize: parentSize, + size: size, } - idb := ethdb.NewTable(db, "bltIndex-") - - if clientMode { - backend.parentSectionSize = BloomTrieFrequency - } else { - backend.parentSectionSize = ethBloomBitsSection - } - backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize + backend.bloomTrieRatio = size / parentSize backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) - return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, 0, time.Millisecond*100, "bloomtrie") + return core.NewChainIndexer(db, ethdb.NewTable(db, "bltIndex-"), backend, size, 0, time.Millisecond*100, "bloomtrie") } // fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the @@ -296,7 +329,7 @@ func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section for i := 0; i < 20; i++ { go func() { for bitIndex := range indexCh { - r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}} + r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}, Config: b.odr.IndexerConfig()} for { if err := b.odr.Retrieve(ctx, r); err == ErrNoPeers { // if there are no peers to serve, retry later @@ -351,9 +384,9 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las // Process implements core.ChainIndexerBackend func (b *BloomTrieIndexerBackend) Process(ctx context.Context, header *types.Header) error { - num := header.Number.Uint64() - b.section*BloomTrieFrequency - if (num+1)%b.parentSectionSize == 0 { - b.sectionHeads[num/b.parentSectionSize] = header.Hash() + num := header.Number.Uint64() - b.section*b.size + if (num+1)%b.parentSize == 0 { + b.sectionHeads[num/b.parentSize] = header.Hash() } return nil } @@ -372,7 +405,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { if err != nil { return err } - decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) + decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSize/8)) if err2 != nil { return err2 } @@ -397,6 +430,5 @@ func (b *BloomTrieIndexerBackend) Commit() error { sectionHead := b.sectionHeads[b.bloomTrieRatio-1] log.Info("Storing bloom trie", "section", b.section, "head", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression", float64(compSize)/float64(decompSize)) StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) - return nil } diff --git a/light/trie_test.go b/light/trie_test.go index 84c6f162fb..6bddfefe28 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -47,7 +47,7 @@ func TestNodeIterator(t *testing.T) { } ctx := context.Background() - odr := &testOdr{sdb: fulldb, ldb: lightdb} + odr := &testOdr{sdb: fulldb, ldb: lightdb, indexerConfig: TestClientIndexerConfig} head := blockchain.CurrentHeader() lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root) fullTrie, _ := state.NewDatabase(fulldb).OpenTrie(head.Root) diff --git a/light/txpool_test.go b/light/txpool_test.go index ccbd83a943..204347a6e4 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -94,7 +94,7 @@ func TestTxPool(t *testing.T) { panic(err) } - odr := &testOdr{sdb: sdb, ldb: ldb} + odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} relay := &testTxRelay{ send: make(chan int, 1), discard: make(chan int, 1), diff --git a/params/network_params.go b/params/network_params.go index 536a46d3df..f8731e8979 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -17,10 +17,38 @@ package params // These are network parameters that need to be constant between clients, but -// aren't necesarilly consensus related. +// aren't necessarily consensus related. const ( // BloomBitsBlocks is the number of blocks a single bloom bit section vector - // contains. + // contains on the server side. BloomBitsBlocks uint64 = 4096 + + // BloomBitsBlocksClient is the number of blocks a single bloom bit section vector + // contains on the light client side + BloomBitsBlocksClient uint64 = 32768 + + // BloomConfirms is the number of confirmation blocks before a bloom section is + // considered probably final and its rotated bits are calculated. + BloomConfirms = 256 + + // CHTFrequencyClient is the block frequency for creating CHTs on the client side. + CHTFrequencyClient = 32768 + + // CHTFrequencyServer is the block frequency for creating CHTs on the server side. + // Eventually this can be merged back with the client version, but that requires a + // full database upgrade, so that should be left for a suitable moment. + CHTFrequencyServer = 4096 + + // BloomTrieFrequency is the block frequency for creating BloomTrie on both + // server/client sides. + BloomTrieFrequency = 32768 + + // HelperTrieConfirmations is the number of confirmations before a client is expected + // to have the given HelperTrie available. + HelperTrieConfirmations = 2048 + + // HelperTrieProcessConfirmations is the number of confirmations before a HelperTrie + // is generated + HelperTrieProcessConfirmations = 256 ) From 63352bf4247f05d8ef255ff8c63290225c3bc671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 28 Aug 2018 09:31:34 +0200 Subject: [PATCH 074/126] core: safe indexer operation when syncing starts before the checkpoint (#17511) --- core/chain_indexer.go | 56 ++++++++++++++++++++++++++++++++++--------- light/lightchain.go | 6 ++--- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index b80b517d91..89ee75eb29 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -85,6 +85,9 @@ type ChainIndexer struct { knownSections uint64 // Number of sections known to be complete (block wise) cascadedHead uint64 // Block number of the last completed section cascaded to subindexers + checkpointSections uint64 // Number of sections covered by the checkpoint + checkpointHead common.Hash // Section head belonging to the checkpoint + throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources log log.Logger @@ -115,12 +118,19 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken return c } -// AddKnownSectionHead marks a new section head as known/processed if it is newer -// than the already known best section head -func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) { +// AddCheckpoint adds a checkpoint. Sections are never processed and the chain +// is not expected to be available before this point. The indexer assumes that +// the backend has sufficient information available to process subsequent sections. +// +// Note: knownSections == 0 and storedSections == checkpointSections until +// syncing reaches the checkpoint +func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) { c.lock.Lock() defer c.lock.Unlock() + c.checkpointSections = section + 1 + c.checkpointHead = shead + if section < c.storedSections { return } @@ -233,16 +243,23 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { // If a reorg happened, invalidate all sections until that point if reorg { // Revert the known section number to the reorg point - changed := head / c.sectionSize - if changed < c.knownSections { - c.knownSections = changed + known := head / c.sectionSize + stored := known + if known < c.checkpointSections { + known = 0 + } + if stored < c.checkpointSections { + stored = c.checkpointSections + } + if known < c.knownSections { + c.knownSections = known } // Revert the stored sections from the database to the reorg point - if changed < c.storedSections { - c.setValidSections(changed) + if stored < c.storedSections { + c.setValidSections(stored) } // Update the new head number to the finalized section end and notify children - head = changed * c.sectionSize + head = known * c.sectionSize if head < c.cascadedHead { c.cascadedHead = head @@ -256,7 +273,18 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { var sections uint64 if head >= c.confirmsReq { sections = (head + 1 - c.confirmsReq) / c.sectionSize + if sections < c.checkpointSections { + sections = 0 + } if sections > c.knownSections { + if c.knownSections < c.checkpointSections { + // syncing reached the checkpoint, verify section head + syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1) + if syncedHead != c.checkpointHead { + c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead) + return + } + } c.knownSections = sections select { @@ -401,8 +429,14 @@ func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) { c.children = append(c.children, indexer) // Cascade any pending updates to new children too - if c.storedSections > 0 { - indexer.newHead(c.storedSections*c.sectionSize-1, false) + sections := c.storedSections + if c.knownSections < sections { + // if a section is "stored" but not "known" then it is a checkpoint without + // available chain data so we should not cascade it yet + sections = c.knownSections + } + if sections > 0 { + indexer.newHead(sections*c.sectionSize-1, false) } } diff --git a/light/lightchain.go b/light/lightchain.go index bd798eca24..d40a4ee6c6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -121,14 +121,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) { if self.odr.ChtIndexer() != nil { StoreChtRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.CHTRoot) - self.odr.ChtIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) + self.odr.ChtIndexer().AddCheckpoint(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomTrieIndexer() != nil { StoreBloomTrieRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.BloomRoot) - self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) + self.odr.BloomTrieIndexer().AddCheckpoint(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomIndexer() != nil { - self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) + self.odr.BloomIndexer().AddCheckpoint(cp.SectionIdx, cp.SectionHead) } log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*self.indexerConfig.ChtSize-1, "hash", cp.SectionHead) } From c1c003e4ff36c22d67662ca661fc78cde850d401 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 28 Aug 2018 21:59:05 +0800 Subject: [PATCH 075/126] consensus, miner: stale block mining support (#17506) * consensus, miner: stale block supporting * consensus, miner: refactor seal signature * cmd, consensus, eth: add miner noverify flag * cmd, consensus, miner: polish --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 11 ++- consensus/clique/clique.go | 39 +++++---- consensus/consensus.go | 9 +- consensus/ethash/algorithm_test.go | 2 +- consensus/ethash/ethash.go | 33 +++---- consensus/ethash/ethash_test.go | 40 ++++----- consensus/ethash/sealer.go | 134 +++++++++++++++++++---------- consensus/ethash/sealer_test.go | 94 ++++++++++++++++++-- eth/api_tracer.go | 2 +- eth/backend.go | 8 +- eth/config.go | 1 + eth/gen_config.go | 7 +- les/backend.go | 2 +- miner/worker.go | 116 +++++++++++-------------- 16 files changed, 317 insertions(+), 183 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7b8a8244ee..0ed67d0d57 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -108,6 +108,7 @@ var ( utils.MinerExtraDataFlag, utils.MinerLegacyExtraDataFlag, utils.MinerRecommitIntervalFlag, + utils.MinerNoVerfiyFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 1e27d0ae8d..2f8260e0f6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -192,6 +192,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, + utils.MinerNoVerfiyFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b9a33ffe70..13430ad565 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -366,9 +366,13 @@ var ( } MinerRecommitIntervalFlag = cli.DurationFlag{ Name: "miner.recommit", - Usage: "Time interval to recreate the block being mined.", + Usage: "Time interval to recreate the block being mined", Value: eth.DefaultConfig.MinerRecommit, } + MinerNoVerfiyFlag = cli.BoolFlag{ + Name: "miner.noverify", + Usage: "Disable remote sealing verification", + } // Account settings UnlockedAccountFlag = cli.StringFlag{ Name: "unlock", @@ -1151,6 +1155,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { cfg.MinerRecommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } + if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) { + cfg.MinerNoverify = ctx.Bool(MinerNoVerfiyFlag.Name) + } if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name) @@ -1345,7 +1352,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir), DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem, DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk, - }, nil) + }, nil, false) } } if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 3730c91f62..5472909846 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -590,17 +590,17 @@ func (c *Clique) Authorize(signer common.Address, signFn SignerFn) { // Seal implements consensus.Engine, attempting to create a sealed block using // the local signing credentials. -func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) { +func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { header := block.Header() // Sealing the genesis block is not supported number := header.Number.Uint64() if number == 0 { - return nil, errUnknownBlock + return errUnknownBlock } // For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing) if c.config.Period == 0 && len(block.Transactions()) == 0 { - return nil, errWaitTransactions + return errWaitTransactions } // Don't hold the signer fields for the entire sealing procedure c.lock.RLock() @@ -610,10 +610,10 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch // Bail out if we're unauthorized to sign a block snap, err := c.snapshot(chain, number-1, header.ParentHash, nil) if err != nil { - return nil, err + return err } if _, authorized := snap.Signers[signer]; !authorized { - return nil, errUnauthorized + return errUnauthorized } // If we're amongst the recent signers, wait for the next block for seen, recent := range snap.Recents { @@ -621,8 +621,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch // Signer is among recents, only wait if the current block doesn't shift it out if limit := uint64(len(snap.Signers)/2 + 1); number < limit || seen > number-limit { log.Info("Signed recently, must wait for others") - <-stop - return nil, nil + return nil } } } @@ -635,21 +634,29 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle)) } - log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay)) - - select { - case <-stop: - return nil, nil - case <-time.After(delay): - } // Sign all the things! sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes()) if err != nil { - return nil, err + return err } copy(header.Extra[len(header.Extra)-extraSeal:], sighash) + // Wait until sealing is terminated or delay timeout. + log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay)) + go func() { + select { + case <-stop: + return + case <-time.After(delay): + } - return block.WithSeal(header), nil + select { + case results <- block.WithSeal(header): + default: + log.Warn("Sealing result is not read by miner", "sealhash", c.SealHash(header)) + } + }() + + return nil } // CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty diff --git a/consensus/consensus.go b/consensus/consensus.go index 27799f13c0..12ede7ff46 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -86,9 +86,12 @@ type Engine interface { Finalize(chain ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) - // Seal generates a new block for the given input block with the local miner's - // seal place on top. - Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) + // Seal generates a new sealing request for the given input block and pushes + // the result into the given channel. + // + // Note, the method returns immediately and will send the result async. More + // than one result may also be returned depending on the consensus algorothm. + Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error // SealHash returns the hash of a block prior to it being sealed. SealHash(header *types.Header) common.Hash diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index db22cccd0c..c58479e289 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { go func(idx int) { defer pend.Done() - ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil) + ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil, false) defer ethash.Close() if err := ethash.VerifySeal(nil, block.Header()); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d98c3371c5..b4819ca38b 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -50,7 +50,7 @@ var ( two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) // sharedEthash is a full instance that can be shared between multiple users. - sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil) + sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil, false) // algorithmRevision is the data structure version used for file naming. algorithmRevision = 23 @@ -405,6 +405,12 @@ type Config struct { PowMode Mode } +// sealTask wraps a seal block with relative result channel for remote sealer thread. +type sealTask struct { + block *types.Block + results chan<- *types.Block +} + // mineResult wraps the pow solution parameters for the specified block. type mineResult struct { nonce types.BlockNonce @@ -444,12 +450,11 @@ type Ethash struct { hashrate metrics.Meter // Meter tracking the average hashrate // Remote sealer related fields - workCh chan *types.Block // Notification channel to push new work to remote sealer - resultCh chan *types.Block // Channel used by mining threads to return result - fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work - submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result - fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. - submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate + workCh chan *sealTask // Notification channel to push new work and relative result channel to remote sealer + fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work + submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result + fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. + submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate // The fields below are hooks for testing shared *Ethash // Shared PoW verifier to avoid cache regeneration @@ -464,7 +469,7 @@ type Ethash struct { // New creates a full sized ethash PoW scheme and starts a background thread for // remote mining, also optionally notifying a batch of remote services of new work // packages. -func New(config Config, notify []string) *Ethash { +func New(config Config, notify []string, noverify bool) *Ethash { if config.CachesInMem <= 0 { log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) config.CachesInMem = 1 @@ -481,36 +486,34 @@ func New(config Config, notify []string) *Ethash { datasets: newlru("dataset", config.DatasetsInMem, newDataset), update: make(chan struct{}), hashrate: metrics.NewMeter(), - workCh: make(chan *types.Block), - resultCh: make(chan *types.Block), + workCh: make(chan *sealTask), fetchWorkCh: make(chan *sealWork), submitWorkCh: make(chan *mineResult), fetchRateCh: make(chan chan uint64), submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote(notify) + go ethash.remote(notify, noverify) return ethash } // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. -func NewTester(notify []string) *Ethash { +func NewTester(notify []string, noverify bool) *Ethash { ethash := &Ethash{ config: Config{PowMode: ModeTest}, caches: newlru("cache", 1, newCache), datasets: newlru("dataset", 1, newDataset), update: make(chan struct{}), hashrate: metrics.NewMeter(), - workCh: make(chan *types.Block), - resultCh: make(chan *types.Block), + workCh: make(chan *sealTask), fetchWorkCh: make(chan *sealWork), submitWorkCh: make(chan *mineResult), fetchRateCh: make(chan chan uint64), submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote(notify) + go ethash.remote(notify, noverify) return ethash } diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index b190d63d6b..8eded2ca81 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -34,17 +34,23 @@ import ( func TestTestMode(t *testing.T) { header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} - ethash := NewTester(nil) + ethash := NewTester(nil, false) defer ethash.Close() - block, err := ethash.Seal(nil, types.NewBlockWithHeader(header), nil) + results := make(chan *types.Block) + err := ethash.Seal(nil, types.NewBlockWithHeader(header), results, nil) if err != nil { t.Fatalf("failed to seal block: %v", err) } - header.Nonce = types.EncodeNonce(block.Nonce()) - header.MixDigest = block.MixDigest() - if err := ethash.VerifySeal(nil, header); err != nil { - t.Fatalf("unexpected verification error: %v", err) + select { + case block := <-results: + header.Nonce = types.EncodeNonce(block.Nonce()) + header.MixDigest = block.MixDigest() + if err := ethash.VerifySeal(nil, header); err != nil { + t.Fatalf("unexpected verification error: %v", err) + } + case <-time.NewTimer(time.Second).C: + t.Error("sealing result timeout") } } @@ -56,7 +62,7 @@ func TestCacheFileEvict(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpdir) - e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil) + e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false) defer e.Close() workers := 8 @@ -85,7 +91,7 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { } func TestRemoteSealer(t *testing.T) { - ethash := NewTester(nil) + ethash := NewTester(nil, false) defer ethash.Close() api := &API{ethash} @@ -97,7 +103,8 @@ func TestRemoteSealer(t *testing.T) { sealhash := ethash.SealHash(header) // Push new work. - ethash.Seal(nil, block, nil) + results := make(chan *types.Block) + ethash.Seal(nil, block, results, nil) var ( work [3]string @@ -114,20 +121,11 @@ func TestRemoteSealer(t *testing.T) { header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} block = types.NewBlockWithHeader(header) sealhash = ethash.SealHash(header) - ethash.Seal(nil, block, nil) + ethash.Seal(nil, block, results, nil) if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { t.Error("expect to return the latest pushed work") } - // Push block with higher block number. - newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} - newBlock := types.NewBlockWithHeader(newHead) - newSealhash := ethash.SealHash(newHead) - ethash.Seal(nil, newBlock, nil) - - if res := api.SubmitWork(types.BlockNonce{}, newSealhash, common.Hash{}); res { - t.Error("expect to return false when submit a stale solution") - } } func TestHashRate(t *testing.T) { @@ -136,7 +134,7 @@ func TestHashRate(t *testing.T) { expect uint64 ids = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")} ) - ethash := NewTester(nil) + ethash := NewTester(nil, false) defer ethash.Close() if tot := ethash.Hashrate(); tot != 0 { @@ -156,7 +154,7 @@ func TestHashRate(t *testing.T) { } func TestClosedRemoteSealer(t *testing.T) { - ethash := NewTester(nil) + ethash := NewTester(nil, false) time.Sleep(1 * time.Second) // ensure exit channel is listening ethash.Close() diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index a458c60f66..06c98a7811 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -35,6 +35,11 @@ import ( "github.com/ethereum/go-ethereum/log" ) +const ( + // staleThreshold is the maximum depth of the acceptable stale but valid ethash solution. + staleThreshold = 7 +) + var ( errNoMiningWork = errors.New("no mining work available yet") errInvalidSealResult = errors.New("invalid or stale proof-of-work solution") @@ -42,16 +47,21 @@ var ( // Seal implements consensus.Engine, attempting to find a nonce that satisfies // the block's difficulty requirements. -func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) { +func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { // If we're running a fake PoW, simply return a 0 nonce immediately if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake { header := block.Header() header.Nonce, header.MixDigest = types.BlockNonce{}, common.Hash{} - return block.WithSeal(header), nil + select { + case results <- block.WithSeal(header): + default: + log.Warn("Sealing result is not read by miner", "mode", "fake", "sealhash", ethash.SealHash(block.Header())) + } + return nil } // If we're running a shared PoW, delegate sealing to it if ethash.shared != nil { - return ethash.shared.Seal(chain, block, stop) + return ethash.shared.Seal(chain, block, results, stop) } // Create a runner and the multiple search threads it directs abort := make(chan struct{}) @@ -62,7 +72,7 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) if err != nil { ethash.lock.Unlock() - return nil, err + return err } ethash.rand = rand.New(rand.NewSource(seed.Int64())) } @@ -75,34 +85,45 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop } // Push new work to remote sealer if ethash.workCh != nil { - ethash.workCh <- block + ethash.workCh <- &sealTask{block: block, results: results} } - var pend sync.WaitGroup + var ( + pend sync.WaitGroup + locals = make(chan *types.Block) + ) for i := 0; i < threads; i++ { pend.Add(1) go func(id int, nonce uint64) { defer pend.Done() - ethash.mine(block, id, nonce, abort, ethash.resultCh) + ethash.mine(block, id, nonce, abort, locals) }(i, uint64(ethash.rand.Int63())) } // Wait until sealing is terminated or a nonce is found - var result *types.Block - select { - case <-stop: - // Outside abort, stop all miner threads - close(abort) - case result = <-ethash.resultCh: - // One of the threads found a block, abort all others - close(abort) - case <-ethash.update: - // Thread count was changed on user request, restart - close(abort) + go func() { + var result *types.Block + select { + case <-stop: + // Outside abort, stop all miner threads + close(abort) + case result = <-locals: + // One of the threads found a block, abort all others + select { + case results <- result: + default: + log.Warn("Sealing result is not read by miner", "mode", "local", "sealhash", ethash.SealHash(block.Header())) + } + close(abort) + case <-ethash.update: + // Thread count was changed on user request, restart + close(abort) + if err := ethash.Seal(chain, block, results, stop); err != nil { + log.Error("Failed to restart sealing after update", "err", err) + } + } + // Wait for all miners to terminate and return the block pend.Wait() - return ethash.Seal(chain, block, stop) - } - // Wait for all miners to terminate and return the block - pend.Wait() - return result, nil + }() + return nil } // mine is the actual proof-of-work miner that searches for a nonce starting from @@ -165,11 +186,12 @@ search: } // remote is a standalone goroutine to handle remote mining related stuff. -func (ethash *Ethash) remote(notify []string) { +func (ethash *Ethash) remote(notify []string, noverify bool) { var ( works = make(map[common.Hash]*types.Block) rates = make(map[common.Hash]hashrate) + results chan<- *types.Block currentBlock *types.Block currentWork [3]string @@ -226,11 +248,15 @@ func (ethash *Ethash) remote(notify []string) { // submitWork verifies the submitted pow solution, returning // whether the solution was accepted or not (not can be both a bad pow as well as // any other error, like no pending work or stale mining result). - submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, hash common.Hash) bool { + submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, sealhash common.Hash) bool { + if currentBlock == nil { + log.Error("Pending work without block", "sealhash", sealhash) + return false + } // Make sure the work submitted is present - block := works[hash] + block := works[sealhash] if block == nil { - log.Info("Work submitted but none pending", "hash", hash) + log.Warn("Work submitted but none pending", "sealhash", sealhash, "curnumber", currentBlock.NumberU64()) return false } // Verify the correctness of submitted result. @@ -239,26 +265,36 @@ func (ethash *Ethash) remote(notify []string) { header.MixDigest = mixDigest start := time.Now() - if err := ethash.verifySeal(nil, header, true); err != nil { - log.Warn("Invalid proof-of-work submitted", "hash", hash, "elapsed", time.Since(start), "err", err) - return false + if !noverify { + if err := ethash.verifySeal(nil, header, true); err != nil { + log.Warn("Invalid proof-of-work submitted", "sealhash", sealhash, "elapsed", time.Since(start), "err", err) + return false + } } - // Make sure the result channel is created. - if ethash.resultCh == nil { + // Make sure the result channel is assigned. + if results == nil { log.Warn("Ethash result channel is empty, submitted mining result is rejected") return false } - log.Trace("Verified correct proof-of-work", "hash", hash, "elapsed", time.Since(start)) + log.Trace("Verified correct proof-of-work", "sealhash", sealhash, "elapsed", time.Since(start)) // Solutions seems to be valid, return to the miner and notify acceptance. - select { - case ethash.resultCh <- block.WithSeal(header): - delete(works, hash) - return true - default: - log.Info("Work submitted is stale", "hash", hash) - return false + solution := block.WithSeal(header) + + // The submitted solution is within the scope of acceptance. + if solution.NumberU64()+staleThreshold > currentBlock.NumberU64() { + select { + case results <- solution: + log.Debug("Work submitted is acceptable", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) + return true + default: + log.Warn("Sealing result is not read by miner", "mode", "remote", "sealhash", sealhash) + return false + } } + // The submitted block is too old to accept, drop it. + log.Warn("Work submitted is too old", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) + return false } ticker := time.NewTicker(5 * time.Second) @@ -266,14 +302,12 @@ func (ethash *Ethash) remote(notify []string) { for { select { - case block := <-ethash.workCh: - if currentBlock != nil && block.ParentHash() != currentBlock.ParentHash() { - // Start new round mining, throw out all previous work. - works = make(map[common.Hash]*types.Block) - } + case work := <-ethash.workCh: // Update current work with new received block. // Note same work can be past twice, happens when changing CPU threads. - makeWork(block) + results = work.results + + makeWork(work.block) // Notify and requested URLs of the new work availability notifyWork() @@ -315,6 +349,14 @@ func (ethash *Ethash) remote(notify []string) { delete(rates, id) } } + // Clear stale pending blocks + if currentBlock != nil { + for hash, block := range works { + if block.NumberU64()+staleThreshold <= currentBlock.NumberU64() { + delete(works, hash) + } + } + } case errc := <-ethash.exitCh: // Exit remote loop if ethash is closed and return relevant error. diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index d1b66f9cf3..31d18b67c7 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -41,14 +41,14 @@ func TestRemoteNotify(t *testing.T) { go server.Serve(listener) // Create the custom ethash engine - ethash := NewTester([]string{"http://" + listener.Addr().String()}) + ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) defer ethash.Close() // Stream a work task and ensure the notification bubbles out header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) - ethash.Seal(nil, block, nil) + ethash.Seal(nil, block, nil, nil) select { case work := <-sink: if want := ethash.SealHash(header).Hex(); work[0] != want { @@ -66,7 +66,7 @@ func TestRemoteNotify(t *testing.T) { } } -// Tests that pushing work packages fast to the miner doesn't cause any daa race +// Tests that pushing work packages fast to the miner doesn't cause any data race // issues in the notifications. func TestRemoteMultiNotify(t *testing.T) { // Start a simple webserver to capture notifications @@ -95,7 +95,7 @@ func TestRemoteMultiNotify(t *testing.T) { go server.Serve(listener) // Create the custom ethash engine - ethash := NewTester([]string{"http://" + listener.Addr().String()}) + ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) defer ethash.Close() // Stream a lot of work task and ensure all the notifications bubble out @@ -103,7 +103,7 @@ func TestRemoteMultiNotify(t *testing.T) { header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) - ethash.Seal(nil, block, nil) + ethash.Seal(nil, block, nil, nil) } for i := 0; i < cap(sink); i++ { select { @@ -113,3 +113,87 @@ func TestRemoteMultiNotify(t *testing.T) { } } } + +// Tests whether stale solutions are correctly processed. +func TestStaleSubmission(t *testing.T) { + ethash := NewTester(nil, true) + defer ethash.Close() + api := &API{ethash} + + fakeNonce, fakeDigest := types.BlockNonce{0x01, 0x02, 0x03}, common.HexToHash("deadbeef") + + testcases := []struct { + headers []*types.Header + submitIndex int + submitRes bool + }{ + // Case1: submit solution for the latest mining package + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xa}), Number: big.NewInt(1), Difficulty: big.NewInt(100000000)}, + }, + 0, + true, + }, + // Case2: submit solution for the previous package but have same parent. + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xb}), Number: big.NewInt(2), Difficulty: big.NewInt(100000000)}, + {ParentHash: common.BytesToHash([]byte{0xb}), Number: big.NewInt(2), Difficulty: big.NewInt(100000001)}, + }, + 0, + true, + }, + // Case3: submit stale but acceptable solution + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xc}), Number: big.NewInt(3), Difficulty: big.NewInt(100000000)}, + {ParentHash: common.BytesToHash([]byte{0xd}), Number: big.NewInt(9), Difficulty: big.NewInt(100000000)}, + }, + 0, + true, + }, + // Case4: submit very old solution + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xe}), Number: big.NewInt(10), Difficulty: big.NewInt(100000000)}, + {ParentHash: common.BytesToHash([]byte{0xf}), Number: big.NewInt(17), Difficulty: big.NewInt(100000000)}, + }, + 0, + false, + }, + } + results := make(chan *types.Block, 16) + + for id, c := range testcases { + for _, h := range c.headers { + ethash.Seal(nil, types.NewBlockWithHeader(h), results, nil) + } + if res := api.SubmitWork(fakeNonce, ethash.SealHash(c.headers[c.submitIndex]), fakeDigest); res != c.submitRes { + t.Errorf("case %d submit result mismatch, want %t, get %t", id+1, c.submitRes, res) + } + if !c.submitRes { + continue + } + select { + case res := <-results: + if res.Header().Nonce != fakeNonce { + t.Errorf("case %d block nonce mismatch, want %s, get %s", id+1, fakeNonce, res.Header().Nonce) + } + if res.Header().MixDigest != fakeDigest { + t.Errorf("case %d block digest mismatch, want %s, get %s", id+1, fakeDigest, res.Header().MixDigest) + } + if res.Header().Difficulty.Uint64() != c.headers[c.submitIndex].Difficulty.Uint64() { + t.Errorf("case %d block difficulty mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Difficulty, res.Header().Difficulty) + } + if res.Header().Number.Uint64() != c.headers[c.submitIndex].Number.Uint64() { + t.Errorf("case %d block number mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Number.Uint64(), res.Header().Number.Uint64()) + } + if res.Header().ParentHash != c.headers[c.submitIndex].ParentHash { + t.Errorf("case %d block parent hash mismatch, want %s, get %s", id+1, c.headers[c.submitIndex].ParentHash.Hex(), res.Header().ParentHash.Hex()) + } + case <-time.NewTimer(time.Second).C: + t.Errorf("case %d fetch ethash result timeout", id+1) + } + } +} diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 704a6cdbaa..0a8b9a9942 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -294,7 +294,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl failed = err break } - // Reference the trie twice, once for us, once for the trancer + // Reference the trie twice, once for us, once for the tracer database.TrieDB().Reference(root, common.Hash{}) if number >= origin { database.TrieDB().Reference(root, common.Hash{}) diff --git a/eth/backend.go b/eth/backend.go index da7e0b2cd5..3032e1a6d3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -130,7 +130,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { chainConfig: chainConfig, eventMux: ctx.EventMux, accountManager: ctx.AccountManager, - engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb), + engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, config.MinerNoverify, chainDb), shutdownChan: make(chan bool), networkID: config.NetworkId, gasPrice: config.MinerGasPrice, @@ -216,7 +216,7 @@ func CreateDB(ctx *node.ServiceContext, config *Config, name string) (ethdb.Data } // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service -func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, db ethdb.Database) consensus.Engine { +func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up if chainConfig.Clique != nil { return clique.New(chainConfig.Clique, db) @@ -228,7 +228,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainCo return ethash.NewFaker() case ethash.ModeTest: log.Warn("Ethash used in test mode") - return ethash.NewTester(nil) + return ethash.NewTester(nil, noverify) case ethash.ModeShared: log.Warn("Ethash used in shared mode") return ethash.NewShared() @@ -240,7 +240,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainCo DatasetDir: config.DatasetDir, DatasetsInMem: config.DatasetsInMem, DatasetsOnDisk: config.DatasetsOnDisk, - }, notify) + }, notify, noverify) engine.SetThreads(-1) // Disable CPU mining return engine } diff --git a/eth/config.go b/eth/config.go index 5b2eca585c..517d7d8f3d 100644 --- a/eth/config.go +++ b/eth/config.go @@ -101,6 +101,7 @@ type Config struct { MinerExtraData []byte `toml:",omitempty"` MinerGasPrice *big.Int MinerRecommit time.Duration + MinerNoverify bool // Ethash options Ethash ethash.Config diff --git a/eth/gen_config.go b/eth/gen_config.go index a72e35bcd1..df4ffeb11f 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -35,6 +35,7 @@ func (c Config) MarshalTOML() (interface{}, error) { MinerExtraData hexutil.Bytes `toml:",omitempty"` MinerGasPrice *big.Int MinerRecommit time.Duration + MinerNoverify bool Ethash ethash.Config TxPool core.TxPoolConfig GPO gasprice.Config @@ -58,6 +59,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.MinerExtraData = c.MinerExtraData enc.MinerGasPrice = c.MinerGasPrice enc.MinerRecommit = c.MinerRecommit + enc.MinerNoverify = c.MinerNoverify enc.Ethash = c.Ethash enc.TxPool = c.TxPool enc.GPO = c.GPO @@ -81,11 +83,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieCache *int TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` - MinerThreads *int `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData *hexutil.Bytes `toml:",omitempty"` MinerGasPrice *big.Int MinerRecommit *time.Duration + MinerNoverify *bool Ethash *ethash.Config TxPool *core.TxPoolConfig GPO *gasprice.Config @@ -144,6 +146,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.MinerRecommit != nil { c.MinerRecommit = *dec.MinerRecommit } + if dec.MinerNoverify != nil { + c.MinerNoverify = *dec.MinerNoverify + } if dec.Ethash != nil { c.Ethash = *dec.Ethash } diff --git a/les/backend.go b/les/backend.go index 75049da085..a3474a6830 100644 --- a/les/backend.go +++ b/les/backend.go @@ -102,7 +102,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { peers: peers, reqDist: newRequestDistributor(peers, quitSync), accountManager: ctx.AccountManager, - engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), + engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, false, chainDb), shutdownChan: make(chan bool), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), diff --git a/miner/worker.go b/miner/worker.go index 1a881799d5..ca68da6e94 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -73,7 +73,7 @@ const ( // increasing upper limit or decreasing lower limit so that the limit can be reachable. intervalAdjustBias = 200 * 1000.0 * 1000.0 - // staleThreshold is the maximum distance of the acceptable stale block. + // staleThreshold is the maximum depth of the acceptable stale block. staleThreshold = 7 ) @@ -139,7 +139,7 @@ type worker struct { // Channels newWorkCh chan *newWorkReq taskCh chan *task - resultCh chan *task + resultCh chan *types.Block startCh chan struct{} exitCh chan struct{} resubmitIntervalCh chan time.Duration @@ -186,7 +186,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), newWorkCh: make(chan *newWorkReq), taskCh: make(chan *task), - resultCh: make(chan *task, resultQueueSize), + resultCh: make(chan *types.Block, resultQueueSize), exitCh: make(chan struct{}), startCh: make(chan struct{}, 1), resubmitIntervalCh: make(chan time.Duration), @@ -269,18 +269,10 @@ func (w *worker) isRunning() bool { return atomic.LoadInt32(&w.running) == 1 } -// close terminates all background threads maintained by the worker and cleans up buffered channels. +// close terminates all background threads maintained by the worker. // Note the worker does not support being closed multiple times. func (w *worker) close() { close(w.exitCh) - // Clean up buffered channels - for empty := false; !empty; { - select { - case <-w.resultCh: - default: - empty = true - } - } } // newWorkLoop is a standalone goroutine to submit new mining work upon received events. @@ -471,42 +463,6 @@ func (w *worker) mainLoop() { } } -// seal pushes a sealing task to consensus engine and submits the result. -func (w *worker) seal(t *task, stop <-chan struct{}) { - if w.skipSealHook != nil && w.skipSealHook(t) { - return - } - // The reason for caching task first is: - // A previous sealing action will be canceled by subsequent actions, - // however, remote miner may submit a result based on the cancelled task. - // So we should only submit the pending state corresponding to the seal result. - // TODO(rjl493456442) Replace the seal-wait logic structure - w.pendingMu.Lock() - w.pendingTasks[w.engine.SealHash(t.block.Header())] = t - w.pendingMu.Unlock() - - if block, err := w.engine.Seal(w.chain, t.block, stop); block != nil { - sealhash := w.engine.SealHash(block.Header()) - w.pendingMu.RLock() - task, exist := w.pendingTasks[sealhash] - w.pendingMu.RUnlock() - if !exist { - log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash()) - return - } - // Assemble sealing result - task.block = block - log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash(), - "elapsed", common.PrettyDuration(time.Since(task.createdAt))) - select { - case w.resultCh <- task: - case <-w.exitCh: - } - } else if err != nil { - log.Warn("Block sealing failed", "err", err) - } -} - // taskLoop is a standalone goroutine to fetch sealing task from the generator and // push them to consensus engine. func (w *worker) taskLoop() { @@ -533,10 +489,20 @@ func (w *worker) taskLoop() { if sealHash == prev { continue } + // Interrupt previous sealing operation interrupt() - stopCh = make(chan struct{}) - prev = sealHash - go w.seal(task, stopCh) + stopCh, prev = make(chan struct{}), sealHash + + if w.skipSealHook != nil && w.skipSealHook(task) { + continue + } + w.pendingMu.Lock() + w.pendingTasks[w.engine.SealHash(task.block.Header())] = task + w.pendingMu.Unlock() + + if err := w.engine.Seal(w.chain, task.block, w.resultCh, stopCh); err != nil { + log.Warn("Block sealing failed", "err", err) + } case <-w.exitCh: interrupt() return @@ -549,38 +515,54 @@ func (w *worker) taskLoop() { func (w *worker) resultLoop() { for { select { - case result := <-w.resultCh: + case block := <-w.resultCh: // Short circuit when receiving empty result. - if result == nil { + if block == nil { continue } // Short circuit when receiving duplicate result caused by resubmitting. - block := result.block if w.chain.HasBlock(block.Hash(), block.NumberU64()) { continue } - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - for _, r := range result.receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } + var ( + sealhash = w.engine.SealHash(block.Header()) + hash = block.Hash() + ) + w.pendingMu.RLock() + task, exist := w.pendingTasks[sealhash] + w.pendingMu.RUnlock() + if !exist { + log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) + continue } - for _, log := range result.state.Logs() { - log.BlockHash = block.Hash() + // Different block could share same sealhash, deep copy here to prevent write-write conflict. + var ( + receipts = make([]*types.Receipt, len(task.receipts)) + logs []*types.Log + ) + for i, receipt := range task.receipts { + receipts[i] = new(types.Receipt) + *receipts[i] = *receipt + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, log := range receipt.Logs { + log.BlockHash = hash + } + logs = append(logs, receipt.Logs...) } // Commit block and state to database. - stat, err := w.chain.WriteBlockWithState(block, result.receipts, result.state) + stat, err := w.chain.WriteBlockWithState(block, receipts, task.state) if err != nil { log.Error("Failed writing block to chain", "err", err) continue } + log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", hash, + "elapsed", common.PrettyDuration(time.Since(task.createdAt))) + // Broadcast the block and announce chain insertion event w.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - logs = result.state.Logs() - ) + + var events []interface{} switch stat { case core.CanonStatTy: events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) From e8f229b82ef99213f8f84b8a71f752b236024494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 29 Aug 2018 12:21:12 +0300 Subject: [PATCH 076/126] cmd, core, eth, miner, params: configurable gas floor and ceil --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 1 + cmd/puppeth/module_node.go | 11 +++++++++-- cmd/puppeth/wizard_node.go | 6 +++++- cmd/utils/flags.go | 27 ++++++++++++++++----------- core/bench_test.go | 3 ++- core/block_validator.go | 22 ++++++++++++++-------- core/chain_makers.go | 2 +- eth/backend.go | 2 +- eth/config.go | 6 +++++- eth/gasprice/gasprice.go | 2 +- eth/gen_config.go | 12 ++++++++++++ internal/ethapi/api.go | 2 +- miner/miner.go | 4 ++-- miner/stress_clique.go | 2 ++ miner/stress_ethash.go | 2 ++ miner/worker.go | 9 +++++++-- miner/worker_test.go | 2 +- params/denomination.go | 16 +++++----------- params/protocol_params.go | 4 ---- 20 files changed, 88 insertions(+), 49 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0ed67d0d57..4d7e98698c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -101,6 +101,7 @@ var ( utils.MinerNotifyFlag, utils.MinerGasTargetFlag, utils.MinerLegacyGasTargetFlag, + utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, utils.MinerLegacyGasPriceFlag, utils.MinerEtherbaseFlag, @@ -236,7 +237,6 @@ func init() { // Start system runtime metrics collection go metrics.CollectProcessMetrics(3 * time.Second) - utils.SetupNetwork(ctx) return nil } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 2f8260e0f6..a674eca4f1 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -189,6 +189,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerNotifyFlag, utils.MinerGasPriceFlag, utils.MinerGasTargetFlag, + utils.MinerGasLimitFlag, utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 8ad41555e1..038152a3e4 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -68,6 +68,7 @@ services: - STATS_NAME={{.Ethstats}} - MINER_NAME={{.Etherbase}} - GAS_TARGET={{.GasTarget}} + - GAS_LIMIT={{.GasLimit}} - GAS_PRICE={{.GasPrice}} logging: driver: "json-file" @@ -104,6 +105,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "Ethstats": config.ethstats, "Etherbase": config.etherbase, "GasTarget": uint64(1000000 * config.gasTarget), + "GasLimit": uint64(1000000 * config.gasLimit), "GasPrice": uint64(1000000000 * config.gasPrice), "Unlock": config.keyJSON != "", }) @@ -122,6 +124,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], "Etherbase": config.etherbase, "GasTarget": config.gasTarget, + "GasLimit": config.gasLimit, "GasPrice": config.gasPrice, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -160,6 +163,7 @@ type nodeInfos struct { keyJSON string keyPass string gasTarget float64 + gasLimit float64 gasPrice float64 } @@ -175,8 +179,9 @@ func (info *nodeInfos) Report() map[string]string { } if info.gasTarget > 0 { // Miner or signer node - report["Gas limit (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget) report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice) + report["Gas floor (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget) + report["Gas ceil (target maximum)"] = fmt.Sprintf("%0.3f MGas", info.gasLimit) if info.etherbase != "" { // Ethash proof-of-work miner @@ -217,6 +222,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"]) lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"]) gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64) + gasLimit, _ := strconv.ParseFloat(infos.envvars["GAS_LIMIT"], 64) gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64) // Container available, retrieve its node ID and its genesis json @@ -256,6 +262,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) keyJSON: keyJSON, keyPass: keyPass, gasTarget: gasTarget, + gasLimit: gasLimit, gasPrice: gasPrice, } stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port) diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 9b22da4460..49b10a023a 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -50,7 +50,7 @@ func (w *wizard) deployNode(boot bool) { if boot { infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256} } else { - infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18} + infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 7.5, gasLimit: 10, gasPrice: 1} } } existed := err == nil @@ -152,6 +152,10 @@ func (w *wizard) deployNode(boot bool) { fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget) infos.gasTarget = w.readDefaultFloat(infos.gasTarget) + fmt.Println() + fmt.Printf("What gas limit should full blocks target (MGas)? (default = %0.3f)\n", infos.gasLimit) + infos.gasLimit = w.readDefaultFloat(infos.gasLimit) + fmt.Println() fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice) infos.gasPrice = w.readDefaultFloat(infos.gasPrice) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 13430ad565..495bfe13e0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -329,12 +329,17 @@ var ( MinerGasTargetFlag = cli.Uint64Flag{ Name: "miner.gastarget", Usage: "Target gas floor for mined blocks", - Value: params.GenesisGasLimit, + Value: eth.DefaultConfig.MinerGasFloor, } MinerLegacyGasTargetFlag = cli.Uint64Flag{ Name: "targetgaslimit", Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)", - Value: params.GenesisGasLimit, + Value: eth.DefaultConfig.MinerGasFloor, + } + MinerGasLimitFlag = cli.Uint64Flag{ + Name: "miner.gaslimit", + Usage: "Target gas ceiling for mined blocks", + Value: eth.DefaultConfig.MinerGasCeil, } MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", @@ -1146,6 +1151,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { cfg.MinerExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } + if ctx.GlobalIsSet(MinerLegacyGasTargetFlag.Name) { + cfg.MinerGasFloor = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name) + } + if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { + cfg.MinerGasFloor = ctx.GlobalUint64(MinerGasTargetFlag.Name) + } + if ctx.GlobalIsSet(MinerGasLimitFlag.Name) { + cfg.MinerGasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name) + } if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { cfg.MinerGasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) } @@ -1270,15 +1284,6 @@ func RegisterEthStatsService(stack *node.Node, url string) { } } -// SetupNetwork configures the system for either the main net or some test network. -func SetupNetwork(ctx *cli.Context) { - // TODO(fjl): move target gas limit into config - params.TargetGasLimit = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name) - if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { - params.TargetGasLimit = ctx.GlobalUint64(MinerGasTargetFlag.Name) - } -} - func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/core/bench_test.go b/core/bench_test.go index 748aebe407..8d95456e9f 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -111,7 +111,8 @@ func init() { func genTxRing(naccounts int) func(int, *BlockGen) { from := 0 return func(i int, gen *BlockGen) { - gas := CalcGasLimit(gen.PrevBlock(i - 1)) + block := gen.PrevBlock(i - 1) + gas := CalcGasLimit(block, block.GasLimit(), block.GasLimit()) for { gas -= params.TxGas if gas < params.TxGas { diff --git a/core/block_validator.go b/core/block_validator.go index ecd6a89bc2..1329f62427 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -101,9 +101,11 @@ func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *stat return nil } -// CalcGasLimit computes the gas limit of the next block after parent. -// This is miner strategy, not consensus protocol. -func CalcGasLimit(parent *types.Block) uint64 { +// CalcGasLimit computes the gas limit of the next block after parent. It aims +// to keep the baseline gas above the provided floor, and increase it towards the +// ceil if the blocks are full. If the ceil is exceeded, it will always decrease +// the gas allowance. +func CalcGasLimit(parent *types.Block, gasFloor, gasCeil uint64) uint64 { // contrib = (parentGasUsed * 3 / 2) / 1024 contrib := (parent.GasUsed() + parent.GasUsed()/2) / params.GasLimitBoundDivisor @@ -121,12 +123,16 @@ func CalcGasLimit(parent *types.Block) uint64 { if limit < params.MinGasLimit { limit = params.MinGasLimit } - // however, if we're now below the target (TargetGasLimit) we increase the - // limit as much as we can (parentGasLimit / 1024 -1) - if limit < params.TargetGasLimit { + // If we're outside our allowed gas range, we try to hone towards them + if limit < gasFloor { limit = parent.GasLimit() + decay - if limit > params.TargetGasLimit { - limit = params.TargetGasLimit + if limit > gasFloor { + limit = gasFloor + } + } else if limit > gasCeil { + limit = parent.GasLimit() - decay + if limit < gasCeil { + limit = gasCeil } } return limit diff --git a/core/chain_makers.go b/core/chain_makers.go index c1e4b4264a..de0fc6be9e 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -240,7 +240,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S Difficulty: parent.Difficulty(), UncleHash: parent.UncleHash(), }), - GasLimit: CalcGasLimit(parent), + GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), Number: new(big.Int).Add(parent.Number(), common.Big1), Time: time, } diff --git a/eth/backend.go b/eth/backend.go index 3032e1a6d3..9926225f24 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -173,7 +173,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine, config.MinerRecommit) + eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine, config.MinerRecommit, config.MinerGasFloor, config.MinerGasCeil) eth.miner.SetExtra(makeExtraData(config.MinerExtraData)) eth.APIBackend = &EthAPIBackend{eth, nil} diff --git a/eth/config.go b/eth/config.go index 517d7d8f3d..f1a402e370 100644 --- a/eth/config.go +++ b/eth/config.go @@ -48,7 +48,9 @@ var DefaultConfig = Config{ DatabaseCache: 768, TrieCache: 256, TrieTimeout: 60 * time.Minute, - MinerGasPrice: big.NewInt(18 * params.Shannon), + MinerGasFloor: 8000000, + MinerGasCeil: 8000000, + MinerGasPrice: big.NewInt(params.GWei), MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, @@ -99,6 +101,8 @@ type Config struct { Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData []byte `toml:",omitempty"` + MinerGasFloor uint64 + MinerGasCeil uint64 MinerGasPrice *big.Int MinerRecommit time.Duration MinerNoverify bool diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 54325692c6..3b8db78a1c 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var maxPrice = big.NewInt(500 * params.Shannon) +var maxPrice = big.NewInt(500 * params.GWei) type Config struct { Blocks int diff --git a/eth/gen_config.go b/eth/gen_config.go index df4ffeb11f..d401a917d2 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -33,6 +33,8 @@ func (c Config) MarshalTOML() (interface{}, error) { Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData hexutil.Bytes `toml:",omitempty"` + MinerGasFloor uint64 + MinerGasCeil uint64 MinerGasPrice *big.Int MinerRecommit time.Duration MinerNoverify bool @@ -57,6 +59,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Etherbase = c.Etherbase enc.MinerNotify = c.MinerNotify enc.MinerExtraData = c.MinerExtraData + enc.MinerGasFloor = c.MinerGasFloor + enc.MinerGasCeil = c.MinerGasCeil enc.MinerGasPrice = c.MinerGasPrice enc.MinerRecommit = c.MinerRecommit enc.MinerNoverify = c.MinerNoverify @@ -85,6 +89,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Etherbase *common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData *hexutil.Bytes `toml:",omitempty"` + MinerGasFloor *uint64 + MinerGasCeil *uint64 MinerGasPrice *big.Int MinerRecommit *time.Duration MinerNoverify *bool @@ -140,6 +146,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.MinerExtraData != nil { c.MinerExtraData = *dec.MinerExtraData } + if dec.MinerGasFloor != nil { + c.MinerGasFloor = *dec.MinerGasFloor + } + if dec.MinerGasCeil != nil { + c.MinerGasCeil = *dec.MinerGasCeil + } if dec.MinerGasPrice != nil { c.MinerGasPrice = dec.MinerGasPrice } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a465a59046..f7379e228c 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -47,7 +47,7 @@ import ( ) const ( - defaultGasPrice = 50 * params.Shannon + defaultGasPrice = params.GWei ) // PublicEthereumAPI provides an API to access Ethereum related information. diff --git a/miner/miner.go b/miner/miner.go index c5a0c9d62a..7f194db261 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -52,13 +52,13 @@ type Miner struct { shouldStart int32 // should start indicates whether we should start after sync } -func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, recommit time.Duration) *Miner { +func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, recommit time.Duration, gasFloor, gasCeil uint64) *Miner { miner := &Miner{ eth: eth, mux: mux, engine: engine, exitCh: make(chan struct{}), - worker: newWorker(config, engine, eth, mux, recommit), + worker: newWorker(config, engine, eth, mux, recommit, gasFloor, gasCeil), canStart: 1, } go miner.update() diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 8545b9413a..8961091d56 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -206,6 +206,8 @@ func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { DatabaseHandles: 256, TxPool: core.DefaultTxPoolConfig, GPO: eth.DefaultConfig.GPO, + MinerGasFloor: genesis.GasLimit * 9 / 10, + MinerGasCeil: genesis.GasLimit * 11 / 10, MinerGasPrice: big.NewInt(1), MinerRecommit: time.Second, }) diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 4ed8aeee4f..5ed11d73a5 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -186,6 +186,8 @@ func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { TxPool: core.DefaultTxPoolConfig, GPO: eth.DefaultConfig.GPO, Ethash: eth.DefaultConfig.Ethash, + MinerGasFloor: genesis.GasLimit * 9 / 10, + MinerGasCeil: genesis.GasLimit * 11 / 10, MinerGasPrice: big.NewInt(1), MinerRecommit: time.Second, }) diff --git a/miner/worker.go b/miner/worker.go index ca68da6e94..5348cb3f1e 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -127,6 +127,9 @@ type worker struct { eth Backend chain *core.BlockChain + gasFloor uint64 + gasCeil uint64 + // Subscriptions mux *event.TypeMux txsCh chan core.NewTxsEvent @@ -171,13 +174,15 @@ type worker struct { resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, recommit time.Duration) *worker { +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, recommit time.Duration, gasFloor, gasCeil uint64) *worker { worker := &worker{ config: config, engine: engine, eth: eth, mux: mux, chain: eth.BlockChain(), + gasFloor: gasFloor, + gasCeil: gasCeil, possibleUncles: make(map[common.Hash]*types.Block), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), pendingTasks: make(map[common.Hash]*task), @@ -807,7 +812,7 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { header := &types.Header{ ParentHash: parent.Hash(), Number: num.Add(num, common.Big1), - GasLimit: core.CalcGasLimit(parent), + GasLimit: core.CalcGasLimit(parent, w.gasFloor, w.gasCeil), Extra: w.extra, Time: big.NewInt(tstamp), } diff --git a/miner/worker_test.go b/miner/worker_test.go index 16708c18c6..9f9a3b4a5e 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -119,7 +119,7 @@ func (b *testWorkerBackend) PostChainEvents(events []interface{}) { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine) backend.txPool.AddLocals(pendingTxs) - w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second) + w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second, params.GenesisGasLimit, params.GenesisGasLimit) w.setEtherbase(testBankAddress) return w, backend } diff --git a/params/denomination.go b/params/denomination.go index 1a309827da..fb4da7f412 100644 --- a/params/denomination.go +++ b/params/denomination.go @@ -17,18 +17,12 @@ package params // These are the multipliers for ether denominations. -// Example: To get the wei value of an amount in 'douglas', use +// Example: To get the wei value of an amount in 'gwei', use // -// new(big.Int).Mul(value, big.NewInt(params.Douglas)) +// new(big.Int).Mul(value, big.NewInt(params.GWei)) // const ( - Wei = 1 - Ada = 1e3 - Babbage = 1e6 - Shannon = 1e9 - Szabo = 1e12 - Finney = 1e15 - Ether = 1e18 - Einstein = 1e21 - Douglas = 1e42 + Wei = 1 + GWei = 1e9 + Ether = 1e18 ) diff --git a/params/protocol_params.go b/params/protocol_params.go index 46624ac9a2..4b53b3320a 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -18,10 +18,6 @@ package params import "math/big" -var ( - TargetGasLimit = GenesisGasLimit // The artificial target -) - const ( GasLimitBoundDivisor uint64 = 1024 // The bound divisor of the gas limit, used in update calculations. MinGasLimit uint64 = 5000 // Minimum the gas limit may ever be. From f751c6ed47db49e0032dfcd0f4c06ce72f3616bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 28 Aug 2018 16:48:20 +0300 Subject: [PATCH 077/126] miner: track uncles more aggressively --- miner/worker.go | 32 ++++++++--------- miner/worker_test.go | 82 ++++++++++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index ca68da6e94..99f1f8dcf6 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -18,7 +18,7 @@ package miner import ( "bytes" - "fmt" + "errors" "math/big" "sync" "sync/atomic" @@ -615,13 +615,16 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { func (w *worker) commitUncle(env *environment, uncle *types.Header) error { hash := uncle.Hash() if env.uncles.Contains(hash) { - return fmt.Errorf("uncle not unique") + return errors.New("uncle not unique") + } + if env.header.ParentHash == uncle.ParentHash { + return errors.New("uncle is sibling") } if !env.ancestors.Contains(uncle.ParentHash) { - return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) + return errors.New("uncle's parent unknown") } if env.family.Contains(hash) { - return fmt.Errorf("uncle already in family (%x)", hash) + return errors.New("uncle already included") } env.uncles.Add(uncle.Hash()) return nil @@ -847,29 +850,24 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { if w.config.DAOForkSupport && w.config.DAOForkBlock != nil && w.config.DAOForkBlock.Cmp(header.Number) == 0 { misc.ApplyDAOHardFork(env.state) } - - // compute uncles for the new block. - var ( - uncles []*types.Header - badUncles []common.Hash - ) + // Accumulate the uncles for the current block + for hash, uncle := range w.possibleUncles { + if uncle.NumberU64()+staleThreshold <= header.Number.Uint64() { + delete(w.possibleUncles, hash) + } + } + uncles := make([]*types.Header, 0, 2) for hash, uncle := range w.possibleUncles { if len(uncles) == 2 { break } if err := w.commitUncle(env, uncle.Header()); err != nil { - log.Trace("Bad uncle found and will be removed", "hash", hash) - log.Trace(fmt.Sprint(uncle)) - - badUncles = append(badUncles, hash) + log.Trace("Possible uncle rejected", "hash", hash, "reason", err) } else { log.Debug("Committing new uncle to block", "hash", hash) uncles = append(uncles, uncle.Header()) } } - for _, hash := range badUncles { - delete(w.possibleUncles, hash) - } if !noempty { // Create an empty block based on temporary copied state for sealing in advance without waiting block diff --git a/miner/worker_test.go b/miner/worker_test.go index 16708c18c6..40a57d1376 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -45,8 +45,8 @@ var ( testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) testBankFunds = big.NewInt(1000000000000000000) - acc1Key, _ = crypto.GenerateKey() - acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) + testUserKey, _ = crypto.GenerateKey() + testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) // Test transactions pendingTxs []*types.Transaction @@ -62,9 +62,9 @@ func init() { Period: 10, Epoch: 30000, } - tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + tx1, _ := types.SignTx(types.NewTransaction(0, testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) pendingTxs = append(pendingTxs, tx1) - tx2, _ := types.SignTx(types.NewTransaction(1, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + tx2, _ := types.SignTx(types.NewTransaction(1, testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) newTxs = append(newTxs, tx2) } @@ -77,7 +77,7 @@ type testWorkerBackend struct { uncleBlock *types.Block } -func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend { +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, n int) *testWorkerBackend { var ( db = ethdb.NewMemDatabase() gspec = core.Genesis{ @@ -92,14 +92,28 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine copy(gspec.ExtraData[32:], testBankAddress[:]) case *ethash.Ethash: default: - t.Fatal("unexpect consensus engine type") + t.Fatalf("unexpected consensus engine type: %T", engine) } genesis := gspec.MustCommit(db) chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) - blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, 1, func(i int, gen *core.BlockGen) { - gen.SetCoinbase(acc1Addr) + + // Generate a small n-block chain and an uncle block for it + if n > 0 { + blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, n, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(testBankAddress) + }) + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("failed to insert origin chain: %v", err) + } + } + parent := genesis + if n > 0 { + parent = chain.GetBlockByHash(chain.CurrentBlock().ParentHash()) + } + blocks, _ := core.GenerateChain(chainConfig, parent, engine, db, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(testUserAddress) }) return &testWorkerBackend{ @@ -116,8 +130,8 @@ func (b *testWorkerBackend) PostChainEvents(events []interface{}) { b.chain.PostChainEvents(events, nil) } -func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { - backend := newTestWorkerBackend(t, chainConfig, engine) +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, blocks int) (*worker, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine, blocks) backend.txPool.AddLocals(pendingTxs) w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second) w.setEtherbase(testBankAddress) @@ -134,24 +148,24 @@ func TestPendingStateAndBlockClique(t *testing.T) { func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, b := newTestWorker(t, chainConfig, engine) + w, b := newTestWorker(t, chainConfig, engine, 0) defer w.close() // Ensure snapshot has been updated. time.Sleep(100 * time.Millisecond) block, state := w.pending() if block.NumberU64() != 1 { - t.Errorf("block number mismatch, has %d, want %d", block.NumberU64(), 1) + t.Errorf("block number mismatch: have %d, want %d", block.NumberU64(), 1) } - if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(1000)) != 0 { - t.Errorf("account balance mismatch, has %d, want %d", balance, 1000) + if balance := state.GetBalance(testUserAddress); balance.Cmp(big.NewInt(1000)) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", balance, 1000) } b.txPool.AddLocals(newTxs) // Ensure the new tx events has been processed time.Sleep(100 * time.Millisecond) block, state = w.pending() - if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(2000)) != 0 { - t.Errorf("account balance mismatch, has %d, want %d", balance, 2000) + if balance := state.GetBalance(testUserAddress); balance.Cmp(big.NewInt(2000)) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", balance, 2000) } } @@ -165,7 +179,7 @@ func TestEmptyWorkClique(t *testing.T) { func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, _ := newTestWorker(t, chainConfig, engine) + w, _ := newTestWorker(t, chainConfig, engine, 0) defer w.close() var ( @@ -179,10 +193,10 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens receiptLen, balance = 1, big.NewInt(1000) } if len(task.receipts) != receiptLen { - t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + t.Errorf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) } - if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { - t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) } } @@ -219,19 +233,19 @@ func TestStreamUncleBlock(t *testing.T) { ethash := ethash.NewFaker() defer ethash.Close() - w, b := newTestWorker(t, ethashChainConfig, ethash) + w, b := newTestWorker(t, ethashChainConfig, ethash, 1) defer w.close() var taskCh = make(chan struct{}) taskIndex := 0 w.newTaskHook = func(task *task) { - if task.block.NumberU64() == 1 { + if task.block.NumberU64() == 2 { if taskIndex == 2 { - has := task.block.Header().UncleHash + have := task.block.Header().UncleHash want := types.CalcUncleHash([]*types.Header{b.uncleBlock.Header()}) - if has != want { - t.Errorf("uncle hash mismatch, has %s, want %s", has.Hex(), want.Hex()) + if have != want { + t.Errorf("uncle hash mismatch: have %s, want %s", have.Hex(), want.Hex()) } } taskCh <- struct{}{} @@ -248,12 +262,12 @@ func TestStreamUncleBlock(t *testing.T) { // Ensure worker has finished initialization for { b := w.pendingBlock() - if b != nil && b.NumberU64() == 1 { + if b != nil && b.NumberU64() == 2 { break } } - w.start() + // Ignore the first two works for i := 0; i < 2; i += 1 { select { @@ -282,7 +296,7 @@ func TestRegenerateMiningBlockClique(t *testing.T) { func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, b := newTestWorker(t, chainConfig, engine) + w, b := newTestWorker(t, chainConfig, engine, 0) defer w.close() var taskCh = make(chan struct{}) @@ -293,10 +307,10 @@ func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, en if taskIndex == 2 { receiptLen, balance := 2, big.NewInt(2000) if len(task.receipts) != receiptLen { - t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + t.Errorf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) } - if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { - t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) } } taskCh <- struct{}{} @@ -347,7 +361,7 @@ func TestAdjustIntervalClique(t *testing.T) { func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, _ := newTestWorker(t, chainConfig, engine) + w, _ := newTestWorker(t, chainConfig, engine, 0) defer w.close() w.skipSealHook = func(task *task) bool { @@ -387,10 +401,10 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co // Check interval if minInterval != wantMinInterval { - t.Errorf("resubmit min interval mismatch want %s has %s", wantMinInterval, minInterval) + t.Errorf("resubmit min interval mismatch: have %v, want %v ", minInterval, wantMinInterval) } if recommitInterval != wantRecommitInterval { - t.Errorf("resubmit interval mismatch want %s has %s", wantRecommitInterval, recommitInterval) + t.Errorf("resubmit interval mismatch: have %v, want %v", recommitInterval, wantRecommitInterval) } result = append(result, float64(recommitInterval.Nanoseconds())) index += 1 From 957496811627d6ef33b9b4232e4d42353d376f1d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 29 Aug 2018 12:52:21 +0200 Subject: [PATCH 078/126] cmd/swarm: disable ACT tests on windows (#17536) --- cmd/swarm/access_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 163eb2b4d6..7a8bcf9d3d 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -13,6 +13,9 @@ // // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . + +// +build !windows + package main import ( From 75ae5af62a8dbbc4fc547d2b3b4cce192d783b2d Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Wed, 29 Aug 2018 12:56:13 +0200 Subject: [PATCH 079/126] whisper: fix loop in expire() (#17532) --- whisper/whisperv5/whisper.go | 2 +- whisper/whisperv5/whisper_test.go | 33 +++++++++++++++++++------------ whisper/whisperv6/whisper.go | 2 +- whisper/whisperv6/whisper_test.go | 33 +++++++++++++++++++------------ 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 8e0662327a..4655458214 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -717,7 +717,7 @@ func (w *Whisper) expire() { w.stats.messagesCleared++ w.stats.memoryCleared += sz w.stats.memoryUsed -= sz - return true + return false }) w.expirations[expiry].Clear() delete(w.expirations, expiry) diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index 8af085292a..a7bd17e4d8 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -487,27 +487,34 @@ func TestExpiry(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.TTL = 1 - msg, err := NewSentMessage(params) - if err != nil { - t.Fatalf("failed to create new message with seed %d: %s.", seed, err) - } - env, err := msg.Wrap(params) - if err != nil { - t.Fatalf("failed Wrap with seed %d: %s.", seed, err) - } - err = w.Send(env) - if err != nil { - t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + messagesCount := 5 + + // Send a few messages one after another. Due to low PoW and expiration buckets + // with one second resolution, it covers a case when there are multiple items + // in a single expiration bucket. + for i := 0; i < messagesCount; i++ { + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + err = w.Send(env) + if err != nil { + t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + } } // wait till received or timeout var received, expired bool for j := 0; j < 20; j++ { time.Sleep(100 * time.Millisecond) - if len(w.Envelopes()) > 0 { + if len(w.Envelopes()) == messagesCount { received = true break } diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 34c1256a6f..be633e7ff6 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -900,7 +900,7 @@ func (whisper *Whisper) expire() { whisper.stats.messagesCleared++ whisper.stats.memoryCleared += sz whisper.stats.memoryUsed -= sz - return true + return false }) whisper.expirations[expiry].Clear() delete(whisper.expirations, expiry) diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 7fe256309e..895bb2b969 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -465,27 +465,34 @@ func TestExpiry(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.TTL = 1 - msg, err := NewSentMessage(params) - if err != nil { - t.Fatalf("failed to create new message with seed %d: %s.", seed, err) - } - env, err := msg.Wrap(params) - if err != nil { - t.Fatalf("failed Wrap with seed %d: %s.", seed, err) - } - err = w.Send(env) - if err != nil { - t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + messagesCount := 5 + + // Send a few messages one after another. Due to low PoW and expiration buckets + // with one second resolution, it covers a case when there are multiple items + // in a single expiration bucket. + for i := 0; i < messagesCount; i++ { + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + err = w.Send(env) + if err != nil { + t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + } } // wait till received or timeout var received, expired bool for j := 0; j < 20; j++ { time.Sleep(100 * time.Millisecond) - if len(w.Envelopes()) > 0 { + if len(w.Envelopes()) == messagesCount { received = true break } From f0242ee76d5d6f54fcd68732cce5c98c7c085101 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 29 Aug 2018 22:31:59 +0800 Subject: [PATCH 080/126] miner: keep the timestamp for resubmitted mining block (#17547) --- miner/worker.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 629ce5bd24..3500ca4c23 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -111,6 +111,7 @@ const ( type newWorkReq struct { interrupt *int32 noempty bool + timestamp int64 } // intervalAdjust represents a resubmitting interval adjustment. @@ -285,6 +286,7 @@ func (w *worker) newWorkLoop(recommit time.Duration) { var ( interrupt *int32 minRecommit = recommit // minimal resubmit interval specified by user. + timestamp int64 // timestamp for each round of mining. ) timer := time.NewTimer(0) @@ -296,7 +298,7 @@ func (w *worker) newWorkLoop(recommit time.Duration) { atomic.StoreInt32(interrupt, s) } interrupt = new(int32) - w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty} + w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty, timestamp: timestamp} timer.Reset(recommit) atomic.StoreInt32(&w.newTxs, 0) } @@ -336,10 +338,12 @@ func (w *worker) newWorkLoop(recommit time.Duration) { select { case <-w.startCh: clearPending(w.chain.CurrentBlock().NumberU64()) + timestamp = time.Now().Unix() commit(false, commitInterruptNewHead) case head := <-w.chainHeadCh: clearPending(head.Block.NumberU64()) + timestamp = time.Now().Unix() commit(false, commitInterruptNewHead) case <-timer.C: @@ -398,7 +402,7 @@ func (w *worker) mainLoop() { for { select { case req := <-w.newWorkCh: - w.commitNewWork(req.interrupt, req.noempty) + w.commitNewWork(req.interrupt, req.noempty, req.timestamp) case ev := <-w.chainSideCh: if _, exist := w.possibleUncles[ev.Block.Hash()]; exist { @@ -450,7 +454,7 @@ func (w *worker) mainLoop() { } else { // If we're mining, but nothing is being processed, wake on new transactions if w.config.Clique != nil && w.config.Clique.Period == 0 { - w.commitNewWork(nil, false) + w.commitNewWork(nil, false, time.Now().Unix()) } } atomic.AddInt32(&w.newTxs, int32(len(ev.Txs))) @@ -793,20 +797,19 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin } // commitNewWork generates several new sealing tasks based on the parent block. -func (w *worker) commitNewWork(interrupt *int32, noempty bool) { +func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) { w.mu.RLock() defer w.mu.RUnlock() tstart := time.Now() parent := w.chain.CurrentBlock() - tstamp := tstart.Unix() - if parent.Time().Cmp(new(big.Int).SetInt64(tstamp)) >= 0 { - tstamp = parent.Time().Int64() + 1 + if parent.Time().Cmp(new(big.Int).SetInt64(timestamp)) >= 0 { + timestamp = parent.Time().Int64() + 1 } // this will ensure we're not going off too far in the future - if now := time.Now().Unix(); tstamp > now+1 { - wait := time.Duration(tstamp-now) * time.Second + if now := time.Now().Unix(); timestamp > now+1 { + wait := time.Duration(timestamp-now) * time.Second log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) time.Sleep(wait) } @@ -817,7 +820,7 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { Number: num.Add(num, common.Big1), GasLimit: core.CalcGasLimit(parent, w.gasFloor, w.gasCeil), Extra: w.extra, - Time: big.NewInt(tstamp), + Time: big.NewInt(timestamp), } // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) if w.isRunning() { From 89451f7c382ad2185987ee369f16416f89c28a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 29 Aug 2018 18:24:32 +0300 Subject: [PATCH 081/126] params, swarm: release geth v1.8.15 and swarm 0.3.3 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index f65236a85c..12f4111507 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 15 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 15 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 5672e19f81..4cd095fb7c 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 62e94895da80838ec694b4ba19eeac80c9dc88c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 29 Aug 2018 18:27:43 +0300 Subject: [PATCH 082/126] params, swarm: begin geth v1.8.16 and swarm v0.3.4 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 12f4111507..a178af8f5f 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 15 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 16 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 4cd095fb7c..a13ca64afb 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 4 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5c0954afffe223ea61ea42dbc74ec7dbdc3663a1 Mon Sep 17 00:00:00 2001 From: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> Date: Mon, 3 Sep 2018 22:47:20 +0800 Subject: [PATCH 083/126] p2p/discv5: make idx bounds checking more sound (#17571) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index b93c93d648..a6cabf0803 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -1228,7 +1228,7 @@ func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) { if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash { return nil, errors.New("topic hash mismatch") } - if int(data.Idx) < 0 || int(data.Idx) >= len(data.Topics) { + if data.Idx >= uint(len(data.Topics)) { return nil, errors.New("topic index out of range") } return pongpkt.data.(*pong), nil From 992b77992f5da7c0301c1b4f8203e8f0d3dfb8f4 Mon Sep 17 00:00:00 2001 From: ult-bobonovski Date: Mon, 3 Sep 2018 22:49:00 +0800 Subject: [PATCH 084/126] consensus: fix comment typo (#17562) --- consensus/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 12ede7ff46..487b07be77 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -90,7 +90,7 @@ type Engine interface { // the result into the given channel. // // Note, the method returns immediately and will send the result async. More - // than one result may also be returned depending on the consensus algorothm. + // than one result may also be returned depending on the consensus algorithm. Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error // SealHash returns the hash of a block prior to it being sealed. From e1c64a7d89a7cc6ae8867ee030d3c8ead677c272 Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Mon, 3 Sep 2018 22:52:32 +0800 Subject: [PATCH 085/126] params: fix typo (#17552) --- params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/config.go b/params/config.go index 70a1edead4..629720550a 100644 --- a/params/config.go +++ b/params/config.go @@ -327,7 +327,7 @@ func (err *ConfigCompatError) Error() string { return fmt.Sprintf("mismatching %s in database (have %d, want %d, rewindto %d)", err.What, err.StoredConfig, err.NewConfig, err.RewindTo) } -// Rules wraps ChainConfig and is merely syntatic sugar or can be used for functions +// Rules wraps ChainConfig and is merely syntactic sugar or can be used for functions // that do not have or require information about the block. // // Rules is a one time interface meaning that it shouldn't be used in between transition From c9a0b36a5f89797672facdb2983600e22ecf6417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=ADsli=20Kristj=C3=A1nsson?= Date: Mon, 3 Sep 2018 14:56:30 +0000 Subject: [PATCH 086/126] rpc: reset client write deadline after write (#17549) This fixes an issue with websocket ping frame handling. --- rpc/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/client.go b/rpc/client.go index a2ef2ed6b6..d96189a2d8 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -487,6 +487,7 @@ func (c *Client) write(ctx context.Context, msg interface{}) error { } c.writeConn.SetWriteDeadline(deadline) err := json.NewEncoder(c.writeConn).Encode(msg) + c.writeConn.SetWriteDeadline(time.Time{}) if err != nil { c.writeConn = nil } From cc2b39bbd18a8f67ac9ceca9ee8eabfc72d0b14b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 3 Sep 2018 16:59:23 +0200 Subject: [PATCH 087/126] consensus/ethash: increase timeout in test (#17526) This is an attempt to fix the flaky consensus/ethash tests under macOS. --- consensus/ethash/sealer_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 31d18b67c7..436359af7c 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -40,6 +40,18 @@ func TestRemoteNotify(t *testing.T) { go server.Serve(listener) + // Wait for server to start listening + var tries int + for tries = 0; tries < 10; tries++ { + conn, _ := net.DialTimeout("tcp", listener.Addr().String(), 1*time.Second) + if conn != nil { + break + } + } + if tries == 10 { + t.Fatal("tcp listener not ready for more than 10 seconds") + } + // Create the custom ethash engine ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) defer ethash.Close() @@ -61,7 +73,7 @@ func TestRemoteNotify(t *testing.T) { if want := common.BytesToHash(target.Bytes()).Hex(); work[2] != want { t.Errorf("work packet target mismatch: have %s, want %s", work[2], want) } - case <-time.After(time.Second): + case <-time.After(3 * time.Second): t.Fatalf("notification timed out") } } @@ -108,7 +120,7 @@ func TestRemoteMultiNotify(t *testing.T) { for i := 0; i < cap(sink); i++ { select { case <-sink: - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Fatalf("notification %d timed out", i) } } From 6fc84946203929143c51010e0a10d9fa61fb9b92 Mon Sep 17 00:00:00 2001 From: Eugene Valeyev Date: Mon, 3 Sep 2018 18:30:47 +0300 Subject: [PATCH 088/126] mobile: add whisper client (#15922) --- mobile/ethclient.go | 8 +- mobile/shhclient.go | 195 ++++++++++++++++++++++++++++++++++++++++++++ mobile/types.go | 93 +++++++++++++++++++++ 3 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 mobile/shhclient.go diff --git a/mobile/ethclient.go b/mobile/ethclient.go index 66399c6b56..662125c4ad 100644 --- a/mobile/ethclient.go +++ b/mobile/ethclient.go @@ -138,7 +138,9 @@ func (ec *EthereumClient) SubscribeNewHead(ctx *Context, handler NewHeadHandler, handler.OnNewHead(&Header{header}) case err := <-rawSub.Err(): - handler.OnError(err.Error()) + if err != nil { + handler.OnError(err.Error()) + } return } } @@ -227,7 +229,9 @@ func (ec *EthereumClient) SubscribeFilterLogs(ctx *Context, query *FilterQuery, handler.OnFilterLogs(&Log{&log}) case err := <-rawSub.Err(): - handler.OnError(err.Error()) + if err != nil { + handler.OnError(err.Error()) + } return } } diff --git a/mobile/shhclient.go b/mobile/shhclient.go new file mode 100644 index 0000000000..a069c9bd40 --- /dev/null +++ b/mobile/shhclient.go @@ -0,0 +1,195 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a wrapper for the Whisper client. + +package geth + +import ( + "github.com/ethereum/go-ethereum/whisper/shhclient" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" +) + +// WhisperClient provides access to the Ethereum APIs. +type WhisperClient struct { + client *shhclient.Client +} + +// NewWhisperClient connects a client to the given URL. +func NewWhisperClient(rawurl string) (client *WhisperClient, _ error) { + rawClient, err := shhclient.Dial(rawurl) + return &WhisperClient{rawClient}, err +} + +// GetVersion returns the Whisper sub-protocol version. +func (wc *WhisperClient) GetVersion(ctx *Context) (version string, _ error) { + return wc.client.Version(ctx.context) +} + +// Info returns diagnostic information about the whisper node. +func (wc *WhisperClient) GetInfo(ctx *Context) (info *Info, _ error) { + rawInfo, err := wc.client.Info(ctx.context) + return &Info{&rawInfo}, err +} + +// SetMaxMessageSize sets the maximal message size allowed by this node. Incoming +// and outgoing messages with a larger size will be rejected. Whisper message size +// can never exceed the limit imposed by the underlying P2P protocol (10 Mb). +func (wc *WhisperClient) SetMaxMessageSize(ctx *Context, size int32) error { + return wc.client.SetMaxMessageSize(ctx.context, uint32(size)) +} + +// SetMinimumPoW (experimental) sets the minimal PoW required by this node. +// This experimental function was introduced for the future dynamic adjustment of +// PoW requirement. If the node is overwhelmed with messages, it should raise the +// PoW requirement and notify the peers. The new value should be set relative to +// the old value (e.g. double). The old value could be obtained via shh_info call. +func (wc *WhisperClient) SetMinimumPoW(ctx *Context, pow float64) error { + return wc.client.SetMinimumPoW(ctx.context, pow) +} + +// Marks specific peer trusted, which will allow it to send historic (expired) messages. +// Note This function is not adding new nodes, the node needs to exists as a peer. +func (wc *WhisperClient) MarkTrustedPeer(ctx *Context, enode string) error { + return wc.client.MarkTrustedPeer(ctx.context, enode) +} + +// NewKeyPair generates a new public and private key pair for message decryption and encryption. +// It returns an identifier that can be used to refer to the key. +func (wc *WhisperClient) NewKeyPair(ctx *Context) (string, error) { + return wc.client.NewKeyPair(ctx.context) +} + +// AddPrivateKey stored the key pair, and returns its ID. +func (wc *WhisperClient) AddPrivateKey(ctx *Context, key []byte) (string, error) { + return wc.client.AddPrivateKey(ctx.context, key) +} + +// DeleteKeyPair delete the specifies key. +func (wc *WhisperClient) DeleteKeyPair(ctx *Context, id string) (string, error) { + return wc.client.DeleteKeyPair(ctx.context, id) +} + +// HasKeyPair returns an indication if the node has a private key or +// key pair matching the given ID. +func (wc *WhisperClient) HasKeyPair(ctx *Context, id string) (bool, error) { + return wc.client.HasKeyPair(ctx.context, id) +} + +// GetPublicKey return the public key for a key ID. +func (wc *WhisperClient) GetPublicKey(ctx *Context, id string) ([]byte, error) { + return wc.client.PublicKey(ctx.context, id) +} + +// GetPrivateKey return the private key for a key ID. +func (wc *WhisperClient) GetPrivateKey(ctx *Context, id string) ([]byte, error) { + return wc.client.PrivateKey(ctx.context, id) +} + +// NewSymmetricKey generates a random symmetric key and returns its identifier. +// Can be used encrypting and decrypting messages where the key is known to both parties. +func (wc *WhisperClient) NewSymmetricKey(ctx *Context) (string, error) { + return wc.client.NewSymmetricKey(ctx.context) +} + +// AddSymmetricKey stores the key, and returns its identifier. +func (wc *WhisperClient) AddSymmetricKey(ctx *Context, key []byte) (string, error) { + return wc.client.AddSymmetricKey(ctx.context, key) +} + +// GenerateSymmetricKeyFromPassword generates the key from password, stores it, and returns its identifier. +func (wc *WhisperClient) GenerateSymmetricKeyFromPassword(ctx *Context, passwd string) (string, error) { + return wc.client.GenerateSymmetricKeyFromPassword(ctx.context, passwd) +} + +// HasSymmetricKey returns an indication if the key associated with the given id is stored in the node. +func (wc *WhisperClient) HasSymmetricKey(ctx *Context, id string) (bool, error) { + return wc.client.HasSymmetricKey(ctx.context, id) +} + +// GetSymmetricKey returns the symmetric key associated with the given identifier. +func (wc *WhisperClient) GetSymmetricKey(ctx *Context, id string) ([]byte, error) { + return wc.client.GetSymmetricKey(ctx.context, id) +} + +// DeleteSymmetricKey deletes the symmetric key associated with the given identifier. +func (wc *WhisperClient) DeleteSymmetricKey(ctx *Context, id string) error { + return wc.client.DeleteSymmetricKey(ctx.context, id) +} + +// Post a message onto the network. +func (wc *WhisperClient) Post(ctx *Context, message *NewMessage) (string, error) { + return wc.client.Post(ctx.context, *message.newMessage) +} + +// NewHeadHandler is a client-side subscription callback to invoke on events and +// subscription failure. +type NewMessageHandler interface { + OnNewMessage(message *Message) + OnError(failure string) +} + +// SubscribeMessages subscribes to messages that match the given criteria. This method +// is only supported on bi-directional connections such as websockets and IPC. +// NewMessageFilter uses polling and is supported over HTTP. +func (wc *WhisperClient) SubscribeMessages(ctx *Context, criteria *Criteria, handler NewMessageHandler, buffer int) (*Subscription, error) { + // Subscribe to the event internally + ch := make(chan *whisper.Message, buffer) + rawSub, err := wc.client.SubscribeMessages(ctx.context, *criteria.criteria, ch) + if err != nil { + return nil, err + } + // Start up a dispatcher to feed into the callback + go func() { + for { + select { + case message := <-ch: + handler.OnNewMessage(&Message{message}) + + case err := <-rawSub.Err(): + if err != nil { + handler.OnError(err.Error()) + } + return + } + } + }() + return &Subscription{rawSub}, nil +} + +// NewMessageFilter creates a filter within the node. This filter can be used to poll +// for new messages (see FilterMessages) that satisfy the given criteria. A filter can +// timeout when it was polled for in whisper.filterTimeout. +func (wc *WhisperClient) NewMessageFilter(ctx *Context, criteria *Criteria) (string, error) { + return wc.client.NewMessageFilter(ctx.context, *criteria.criteria) +} + +// DeleteMessageFilter removes the filter associated with the given id. +func (wc *WhisperClient) DeleteMessageFilter(ctx *Context, id string) error { + return wc.client.DeleteMessageFilter(ctx.context, id) +} + +// GetFilterMessages retrieves all messages that are received between the last call to +// this function and match the criteria that where given when the filter was created. +func (wc *WhisperClient) GetFilterMessages(ctx *Context, id string) (*Messages, error) { + rawFilterMessages, err := wc.client.FilterMessages(ctx.context, id) + if err != nil { + return nil, err + } + res := make([]*whisper.Message, len(rawFilterMessages)) + copy(res, rawFilterMessages) + return &Messages{res}, nil +} diff --git a/mobile/types.go b/mobile/types.go index b2780f3076..443d07ea93 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) // A Nonce is a 64-bit hash which proves (combined with the mix-hash) that @@ -334,3 +335,95 @@ func (r *Receipt) GetLogs() *Logs { return &Logs{r.receipt.Logs} } func (r *Receipt) GetTxHash() *Hash { return &Hash{r.receipt.TxHash} } func (r *Receipt) GetContractAddress() *Address { return &Address{r.receipt.ContractAddress} } func (r *Receipt) GetGasUsed() int64 { return int64(r.receipt.GasUsed) } + +// Info represents a diagnostic information about the whisper node. +type Info struct { + info *whisper.Info +} + +// NewMessage represents a new whisper message that is posted through the RPC. +type NewMessage struct { + newMessage *whisper.NewMessage +} + +func NewNewMessage() *NewMessage { + nm := &NewMessage{ + newMessage: new(whisper.NewMessage), + } + return nm +} + +func (nm *NewMessage) GetSymKeyID() string { return nm.newMessage.SymKeyID } +func (nm *NewMessage) SetSymKeyID(symKeyID string) { nm.newMessage.SymKeyID = symKeyID } +func (nm *NewMessage) GetPublicKey() []byte { return nm.newMessage.PublicKey } +func (nm *NewMessage) SetPublicKey(publicKey []byte) { + nm.newMessage.PublicKey = common.CopyBytes(publicKey) +} +func (nm *NewMessage) GetSig() string { return nm.newMessage.Sig } +func (nm *NewMessage) SetSig(sig string) { nm.newMessage.Sig = sig } +func (nm *NewMessage) GetTTL() int64 { return int64(nm.newMessage.TTL) } +func (nm *NewMessage) SetTTL(ttl int64) { nm.newMessage.TTL = uint32(ttl) } +func (nm *NewMessage) GetPayload() []byte { return nm.newMessage.Payload } +func (nm *NewMessage) SetPayload(payload []byte) { nm.newMessage.Payload = common.CopyBytes(payload) } +func (nm *NewMessage) GetPowTime() int64 { return int64(nm.newMessage.PowTime) } +func (nm *NewMessage) SetPowTime(powTime int64) { nm.newMessage.PowTime = uint32(powTime) } +func (nm *NewMessage) GetPowTarget() float64 { return nm.newMessage.PowTarget } +func (nm *NewMessage) SetPowTarget(powTarget float64) { nm.newMessage.PowTarget = powTarget } +func (nm *NewMessage) GetTargetPeer() string { return nm.newMessage.TargetPeer } +func (nm *NewMessage) SetTargetPeer(targetPeer string) { nm.newMessage.TargetPeer = targetPeer } +func (nm *NewMessage) GetTopic() []byte { return nm.newMessage.Topic[:] } +func (nm *NewMessage) SetTopic(topic []byte) { nm.newMessage.Topic = whisper.BytesToTopic(topic) } + +// Message represents a whisper message. +type Message struct { + message *whisper.Message +} + +func (m *Message) GetSig() []byte { return m.message.Sig } +func (m *Message) GetTTL() int64 { return int64(m.message.TTL) } +func (m *Message) GetTimestamp() int64 { return int64(m.message.Timestamp) } +func (m *Message) GetPayload() []byte { return m.message.Payload } +func (m *Message) GetPoW() float64 { return m.message.PoW } +func (m *Message) GetHash() []byte { return m.message.Hash } +func (m *Message) GetDst() []byte { return m.message.Dst } + +// Messages represents an array of messages. +type Messages struct { + messages []*whisper.Message +} + +// Size returns the number of messages in the slice. +func (m *Messages) Size() int { + return len(m.messages) +} + +// Get returns the message at the given index from the slice. +func (m *Messages) Get(index int) (message *Message, _ error) { + if index < 0 || index >= len(m.messages) { + return nil, errors.New("index out of bounds") + } + return &Message{m.messages[index]}, nil +} + +// Criteria holds various filter options for inbound messages. +type Criteria struct { + criteria *whisper.Criteria +} + +func NewCriteria(topic []byte) *Criteria { + c := &Criteria{ + criteria: new(whisper.Criteria), + } + encodedTopic := whisper.BytesToTopic(topic) + c.criteria.Topics = []whisper.TopicType{encodedTopic} + return c +} + +func (c *Criteria) GetSymKeyID() string { return c.criteria.SymKeyID } +func (c *Criteria) SetSymKeyID(symKeyID string) { c.criteria.SymKeyID = symKeyID } +func (c *Criteria) GetPrivateKeyID() string { return c.criteria.PrivateKeyID } +func (c *Criteria) SetPrivateKeyID(privateKeyID string) { c.criteria.PrivateKeyID = privateKeyID } +func (c *Criteria) GetSig() []byte { return c.criteria.Sig } +func (c *Criteria) SetSig(sig []byte) { c.criteria.Sig = common.CopyBytes(sig) } +func (c *Criteria) GetMinPow() float64 { return c.criteria.MinPow } +func (c *Criteria) SetMinPow(pow float64) { c.criteria.MinPow = pow } From 6a33954731658667056466bf7573ed1c397f4750 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Mon, 3 Sep 2018 23:33:21 +0800 Subject: [PATCH 089/126] core, eth, trie: use common/prque (#17508) --- core/blockchain.go | 6 +- core/tx_pool.go | 6 +- eth/downloader/queue.go | 34 +++--- eth/fetcher/fetcher.go | 8 +- trie/sync.go | 6 +- vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE | 25 ---- .../gopkg.in/karalabe/cookiejar.v2/README.md | 109 ------------------ .../cookiejar.v2/collections/prque/prque.go | 66 ----------- .../cookiejar.v2/collections/prque/sstack.go | 91 --------------- vendor/vendor.json | 6 - 10 files changed, 30 insertions(+), 327 deletions(-) delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/README.md delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go diff --git a/core/blockchain.go b/core/blockchain.go index 0461da7fd9..63f60ca280 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -43,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) var ( @@ -151,7 +151,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par chainConfig: chainConfig, cacheConfig: cacheConfig, db: db, - triegc: prque.New(), + triegc: prque.New(nil), stateCache: state.NewDatabase(db), quit: make(chan struct{}), bodyCache: bodyCache, @@ -915,7 +915,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } else { // Full but not archive node, do proper garbage collection triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive - bc.triegc.Push(root, -float32(block.NumberU64())) + bc.triegc.Push(root, -int64(block.NumberU64())) if current := block.NumberU64(); current > triesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk diff --git a/core/tx_pool.go b/core/tx_pool.go index 46ae2759bc..a0a6ff8510 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -26,13 +26,13 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) const ( @@ -987,11 +987,11 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { if pending > pool.config.GlobalSlots { pendingBeforeCap := pending // Assemble a spam order to penalize large transactors first - spammers := prque.New() + spammers := prque.New(nil) for addr, list := range pool.pending { // Only evict transactions from high rollers if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { - spammers.Push(addr, float32(list.Len())) + spammers.Push(addr, int64(list.Len())) } } // Gradually drop transactions from offenders diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 8529535ba7..a0e8a6d48c 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -26,10 +26,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) var ( @@ -105,11 +105,11 @@ func newQueue() *queue { headerPendPool: make(map[string]*fetchRequest), headerContCh: make(chan bool), blockTaskPool: make(map[common.Hash]*types.Header), - blockTaskQueue: prque.New(), + blockTaskQueue: prque.New(nil), blockPendPool: make(map[string]*fetchRequest), blockDonePool: make(map[common.Hash]struct{}), receiptTaskPool: make(map[common.Hash]*types.Header), - receiptTaskQueue: prque.New(), + receiptTaskQueue: prque.New(nil), receiptPendPool: make(map[string]*fetchRequest), receiptDonePool: make(map[common.Hash]struct{}), resultCache: make([]*fetchResult, blockCacheItems), @@ -277,7 +277,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { } // Schedule all the header retrieval tasks for the skeleton assembly q.headerTaskPool = make(map[uint64]*types.Header) - q.headerTaskQueue = prque.New() + q.headerTaskQueue = prque.New(nil) q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) q.headerProced = 0 @@ -288,7 +288,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { index := from + uint64(i*MaxHeaderFetch) q.headerTaskPool[index] = header - q.headerTaskQueue.Push(index, -float32(index)) + q.headerTaskQueue.Push(index, -int64(index)) } } @@ -334,11 +334,11 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { } // Queue the header for content retrieval q.blockTaskPool[hash] = header - q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) if q.mode == FastSync { q.receiptTaskPool[hash] = header - q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) } inserts = append(inserts, header) q.headerHead = hash @@ -436,7 +436,7 @@ func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { } // Merge all the skipped batches back for _, from := range skip { - q.headerTaskQueue.Push(from, -float32(from)) + q.headerTaskQueue.Push(from, -int64(from)) } // Assemble and return the block download request if send == 0 { @@ -542,7 +542,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common } // Merge all the skipped headers back for _, header := range skip { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } if progress { // Wake WaitResults, resultCache was modified @@ -585,10 +585,10 @@ func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool m defer q.lock.Unlock() if request.From > 0 { - taskQueue.Push(request.From, -float32(request.From)) + taskQueue.Push(request.From, -int64(request.From)) } for _, header := range request.Headers { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } delete(pendPool, request.Peer.id) } @@ -602,13 +602,13 @@ func (q *queue) Revoke(peerID string) { if request, ok := q.blockPendPool[peerID]; ok { for _, header := range request.Headers { - q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) } delete(q.blockPendPool, peerID) } if request, ok := q.receiptPendPool[peerID]; ok { for _, header := range request.Headers { - q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) } delete(q.receiptPendPool, peerID) } @@ -657,10 +657,10 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, // Return any non satisfied requests to the pool if request.From > 0 { - taskQueue.Push(request.From, -float32(request.From)) + taskQueue.Push(request.From, -int64(request.From)) } for _, header := range request.Headers { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) @@ -731,7 +731,7 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh } miss[request.From] = struct{}{} - q.headerTaskQueue.Push(request.From, -float32(request.From)) + q.headerTaskQueue.Push(request.From, -int64(request.From)) return 0, errors.New("delivery not accepted") } // Clean up a successful fetch and try to deliver any sub-results @@ -854,7 +854,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ // Return all failed or missing fetches to the queue for _, header := range request.Headers { if header != nil { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } } // Wake up WaitResults diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 277f14b81a..f0b5e8064b 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -23,10 +23,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) const ( @@ -160,7 +160,7 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc fetching: make(map[common.Hash]*announce), fetched: make(map[common.Hash][]*announce), completing: make(map[common.Hash]*announce), - queue: prque.New(), + queue: prque.New(nil), queues: make(map[string]int), queued: make(map[common.Hash]*inject), getBlock: getBlock, @@ -299,7 +299,7 @@ func (f *Fetcher) loop() { // If too high up the chain or phase, continue later number := op.block.NumberU64() if number > height+1 { - f.queue.Push(op, -float32(number)) + f.queue.Push(op, -int64(number)) if f.queueChangeHook != nil { f.queueChangeHook(hash, true) } @@ -624,7 +624,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { } f.queues[peer] = count f.queued[hash] = op - f.queue.Push(op, -float32(block.NumberU64())) + f.queue.Push(op, -int64(block.NumberU64())) if f.queueChangeHook != nil { f.queueChangeHook(op.block.Hash(), true) } diff --git a/trie/sync.go b/trie/sync.go index 88d6eb7799..67dff5a8b6 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/ethdb" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) // ErrNotRequested is returned by the trie sync when it's requested to process a @@ -84,7 +84,7 @@ func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) * database: database, membatch: newSyncMemBatch(), requests: make(map[common.Hash]*request), - queue: prque.New(), + queue: prque.New(nil), } ts.AddSubTrie(root, 0, common.Hash{}, callback) return ts @@ -242,7 +242,7 @@ func (s *Sync) schedule(req *request) { return } // Schedule the request for future retrieval - s.queue.Push(req.hash, float32(req.depth)) + s.queue.Push(req.hash, int64(req.depth)) s.requests[req.hash] = req } diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE b/vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE deleted file mode 100755 index 467d60878d..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2014 Péter Szilágyi. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Alternatively, the CookieJar toolbox may be used in accordance with the terms -and conditions contained in a signed written agreement between you and the -author(s). diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/README.md b/vendor/gopkg.in/karalabe/cookiejar.v2/README.md deleted file mode 100755 index 44c420b60e..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/README.md +++ /dev/null @@ -1,109 +0,0 @@ - CookieJar - A contestant's toolbox -====================================== - -CookieJar is a small collection of common algorithms, data structures and library extensions that were deemed handy for computing competitions at one point or another. - -This toolbox is a work in progress for the time being. It may be lacking, and it may change drastically between commits (although every effort is made not to). You're welcome to use it, but it's your head on the line :) - - Installation ----------------- - -To get the package, execute: - - go get gopkg.in/karalabe/cookiejar.v2 - -To import this package, add the following line to your code: - - import "gopkg.in/karalabe/cookiejar.v2" - -For more details, see the [package documentation](http://godoc.org/gopkg.in/karalabe/cookiejar.v2). - - Contents ------------- - -Algorithms: - - Graph - - [Breadth First Search](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/graph/bfs) - - [Depth First Search](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/graph/dfs) - -Data structures: - - [Bag](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/bag) - - [Deque](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/deque) - - [Graph](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/graph) - - [Priority Queue](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/prque) - - [Queue](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/queue) - - [Set](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/set) - - [Stack](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/stack) - -Extensions: - - [fmt](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/fmtext) - - `Scan` and `Fscan` for `int`, `float64`, `string` and lines - - [math](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/mathext) - - `Abs` for `int` - - `Min` and `Max` for `int`, `big.Int` and `big.Rat` - - `Sign` for `int` and `float64` - - [os](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/osext) - - `Open` and `Create` without error codes - - [sort](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/sortext) - - `Sort` and `Search` for `big.Int` and `big.Rat` - - `Unique` for any `sort.Interface` - -Below are the performance results for the data structures and the complexity analysis for the algorithms. - - Performance ---------------- - -Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz: -``` -- bag - - BenchmarkInsert 309 ns/op - - BenchmarkRemove 197 ns/op - - BenchmarkDo 28.1 ns/op -- deque - - BenchmarkPush 25.4 ns/op - - BenchmarkPop 6.72 ns/op -- prque - - BenchmarkPush 171 ns/op - - BenchmarkPop 947 ns/op -- queue - - BenchmarkPush 23.0 ns/op - - BenchmarkPop 5.92 ns/op -- set - - BenchmarkInsert 259 ns/op - - BenchmarkRemove 115 ns/op - - BenchmarkDo 20.9 ns/op -- stack - - BenchmarkPush 16.4 ns/op - - BenchmarkPop 6.45 ns/op -``` - - Complexity --------------- - -| Algorithm | Time complexity | Space complexity | -|:---------:|:---------------:|:----------------:| -| graph/bfs | O(E) | O(V) | -| graph/dfs | O(E) | O(E) | - - Here be dragons :) ----------------------- - -``` - . _///_, - . / ` ' '> - ) o' __/_'> - ( / _/ )_\'> - ' "__/ /_/\_> - ____/_/_/_/ - /,---, _/ / - "" /_/_/_/ - /_(_(_(_ \ - ( \_\_\\_ )\ - \'__\_\_\_\__ ).\ - //____|___\__) )_/ - | _ \'___'_( /' - \_ (-'\'___'_\ __,'_' - __) \ \\___(_ __/.__,' - ,((,-,__\ '", __\_/. __,' - '"./_._._-' -``` diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go b/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go deleted file mode 100755 index 5c1967c658..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go +++ /dev/null @@ -1,66 +0,0 @@ -// CookieJar - A contestant's algorithm toolbox -// Copyright (c) 2013 Peter Szilagyi. All rights reserved. -// -// CookieJar is dual licensed: use of this source code is governed by a BSD -// license that can be found in the LICENSE file. Alternatively, the CookieJar -// toolbox may be used in accordance with the terms and conditions contained -// in a signed written agreement between you and the author(s). - -// Package prque implements a priority queue data structure supporting arbitrary -// value types and float priorities. -// -// The reasoning behind using floats for the priorities vs. ints or interfaces -// was larger flexibility without sacrificing too much performance or code -// complexity. -// -// If you would like to use a min-priority queue, simply negate the priorities. -// -// Internally the queue is based on the standard heap package working on a -// sortable version of the block based stack. -package prque - -import ( - "container/heap" -) - -// Priority queue data structure. -type Prque struct { - cont *sstack -} - -// Creates a new priority queue. -func New() *Prque { - return &Prque{newSstack()} -} - -// Pushes a value with a given priority into the queue, expanding if necessary. -func (p *Prque) Push(data interface{}, priority float32) { - heap.Push(p.cont, &item{data, priority}) -} - -// Pops the value with the greates priority off the stack and returns it. -// Currently no shrinking is done. -func (p *Prque) Pop() (interface{}, float32) { - item := heap.Pop(p.cont).(*item) - return item.value, item.priority -} - -// Pops only the item from the queue, dropping the associated priority value. -func (p *Prque) PopItem() interface{} { - return heap.Pop(p.cont).(*item).value -} - -// Checks whether the priority queue is empty. -func (p *Prque) Empty() bool { - return p.cont.Len() == 0 -} - -// Returns the number of element in the priority queue. -func (p *Prque) Size() int { - return p.cont.Len() -} - -// Clears the contents of the priority queue. -func (p *Prque) Reset() { - *p = *New() -} diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go b/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go deleted file mode 100755 index 9f393196ef..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go +++ /dev/null @@ -1,91 +0,0 @@ -// CookieJar - A contestant's algorithm toolbox -// Copyright (c) 2013 Peter Szilagyi. All rights reserved. -// -// CookieJar is dual licensed: use of this source code is governed by a BSD -// license that can be found in the LICENSE file. Alternatively, the CookieJar -// toolbox may be used in accordance with the terms and conditions contained -// in a signed written agreement between you and the author(s). - -package prque - -// The size of a block of data -const blockSize = 4096 - -// A prioritized item in the sorted stack. -type item struct { - value interface{} - priority float32 -} - -// Internal sortable stack data structure. Implements the Push and Pop ops for -// the stack (heap) functionality and the Len, Less and Swap methods for the -// sortability requirements of the heaps. -type sstack struct { - size int - capacity int - offset int - - blocks [][]*item - active []*item -} - -// Creates a new, empty stack. -func newSstack() *sstack { - result := new(sstack) - result.active = make([]*item, blockSize) - result.blocks = [][]*item{result.active} - result.capacity = blockSize - return result -} - -// Pushes a value onto the stack, expanding it if necessary. Required by -// heap.Interface. -func (s *sstack) Push(data interface{}) { - if s.size == s.capacity { - s.active = make([]*item, blockSize) - s.blocks = append(s.blocks, s.active) - s.capacity += blockSize - s.offset = 0 - } else if s.offset == blockSize { - s.active = s.blocks[s.size/blockSize] - s.offset = 0 - } - s.active[s.offset] = data.(*item) - s.offset++ - s.size++ -} - -// Pops a value off the stack and returns it. Currently no shrinking is done. -// Required by heap.Interface. -func (s *sstack) Pop() (res interface{}) { - s.size-- - s.offset-- - if s.offset < 0 { - s.offset = blockSize - 1 - s.active = s.blocks[s.size/blockSize] - } - res, s.active[s.offset] = s.active[s.offset], nil - return -} - -// Returns the length of the stack. Required by sort.Interface. -func (s *sstack) Len() int { - return s.size -} - -// Compares the priority of two elements of the stack (higher is first). -// Required by sort.Interface. -func (s *sstack) Less(i, j int) bool { - return s.blocks[i/blockSize][i%blockSize].priority > s.blocks[j/blockSize][j%blockSize].priority -} - -// Swaps two elements in the stack. Required by sort.Interface. -func (s *sstack) Swap(i, j int) { - ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize - s.blocks[ib][io], s.blocks[jb][jo] = s.blocks[jb][jo], s.blocks[ib][io] -} - -// Resets the stack, effectively clearing its contents. -func (s *sstack) Reset() { - *s = *newSstack() -} diff --git a/vendor/vendor.json b/vendor/vendor.json index fea2c804ea..6e5610460a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -891,12 +891,6 @@ "revision": "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec", "revisionTime": "2016-12-08T18:13:25Z" }, - { - "checksumSHA1": "DQXNV0EivoHm4q+bkdahYXrjjfE=", - "path": "gopkg.in/karalabe/cookiejar.v2/collections/prque", - "revision": "8dcd6a7f4951f6ff3ee9cbb919a06d8925822e57", - "revisionTime": "2015-07-24T13:16:13Z" - }, { "checksumSHA1": "0xgs8lwcWLUffemlj+SsgKlxvDU=", "path": "gopkg.in/natefinch/npipe.v2", From 32f28a9360d26a661d55915915f12fd3c70f012b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 4 Sep 2018 10:49:18 +0200 Subject: [PATCH 090/126] core/vm, tests: update tests, enable constantinople statetests, fix SAR opcode (#17538) This commit does a few things at once: - Updates the tests to contain the latest data from ethereum/tests repo. - Enables Constantinople state tests. This is needed to be able to fuzz-test the evm with constantinople rules. - Fixes the error in opSAR that we've known about for some time. I was kind of saving it to see if we hit upon it with the random test generator, but it's difficult to both enable the tests and have the bug there -- we don't want to forget about it, so maybe it's better to just fix it. --- core/vm/instructions.go | 2 +- tests/block_test.go | 6 +++--- tests/init_test.go | 10 ++++++++++ tests/state_test.go | 3 --- tests/state_test_util.go | 13 ++++++++++++- tests/testdata | 2 +- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index b7742e6551..ca9e775ac5 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -355,7 +355,7 @@ func opSAR(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory * defer interpreter.intPool.put(shift) // First operand back into the pool if shift.Cmp(common.Big256) >= 0 { - if value.Sign() > 0 { + if value.Sign() >= 0 { value.SetUint64(0) } else { value.SetInt64(-1) diff --git a/tests/block_test.go b/tests/block_test.go index 669d3ca08a..c911199299 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -30,11 +30,11 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcForgedTest/bcForkUncle\.json`) bt.skipLoad(`^bcMultiChainTest/(ChainAtoChainB_blockorder|CallContractFromNotBestBlock)`) bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) - // Constantinople is not implemented yet. - bt.skipLoad(`(?i)(constantinople)`) + // This test is broken + bt.fails(`blockhashNonConstArg_Constantinople`, "Broken test") // Still failing tests - bt.skipLoad(`^bcWalletTest.*_Byzantium$`) + // bt.skipLoad(`^bcWalletTest.*_Byzantium$`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { if err := bt.checkFailure(t, name, test.Run()); err != nil { diff --git a/tests/init_test.go b/tests/init_test.go index 26e919d24b..90a74448a8 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -91,6 +91,7 @@ type testMatcher struct { failpat []testFailure skiploadpat []*regexp.Regexp skipshortpat []*regexp.Regexp + whitelistpat *regexp.Regexp } type testConfig struct { @@ -121,6 +122,10 @@ func (tm *testMatcher) fails(pattern string, reason string) { tm.failpat = append(tm.failpat, testFailure{regexp.MustCompile(pattern), reason}) } +func (tm *testMatcher) whitelist(pattern string) { + tm.whitelistpat = regexp.MustCompile(pattern) +} + // config defines chain config for tests matching the pattern. func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) { tm.configpat = append(tm.configpat, testConfig{regexp.MustCompile(pattern), cfg}) @@ -208,6 +213,11 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte if r, _ := tm.findSkip(name); r != "" { t.Skip(r) } + if tm.whitelistpat != nil { + if !tm.whitelistpat.MatchString(name) { + t.Skip("Skipped by whitelist") + } + } t.Parallel() // Load the file as map[string]. diff --git a/tests/state_test.go b/tests/state_test.go index adec4feb2b..b61a1ca285 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -44,9 +44,6 @@ func TestState(t *testing.T) { key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key t.Run(key, func(t *testing.T) { - if subtest.Fork == "Constantinople" { - t.Skip("constantinople not supported yet") - } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { _, err := test.Run(subtest, vmconfig) return st.checkFailure(t, name, err) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 84581fae18..5d2251e529 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -146,7 +146,18 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - root, _ := statedb.Commit(config.IsEIP158(block.Number())) + // Commit block + statedb.Commit(config.IsEIP158(block.Number())) + // Add 0-value mining reward. This only makes a difference in the cases + // where + // - the coinbase suicided, or + // - there are only 'bad' transactions, which aren't executed. In those cases, + // the coinbase gets no txfee, so isn't created, and thus needs to be touched + statedb.AddBalance(block.Coinbase(), new(big.Int)) + // And _now_ get the state root + root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) + // N.B: We need to do this in a two-step process, because the first Commit takes care + // of suicides, and we need to touch the coinbase _after_ it has potentially suicided. if root != common.Hash(post.Root) { return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } diff --git a/tests/testdata b/tests/testdata index 2bb0c3da3b..ad2184adca 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 2bb0c3da3bbb15c528bcef2a7e5ac4bd73f81f87 +Subproject commit ad2184adca367c0b68c65b44519dba16e1d0b9e2 From 003e031994327ab24900aa95e0f516a13e97f76f Mon Sep 17 00:00:00 2001 From: dipingxian2 <39109351+dipingxian2@users.noreply.github.com> Date: Tue, 4 Sep 2018 19:16:49 +0800 Subject: [PATCH 091/126] cmd/faucet: remove trailing newline in password (#17558) Fixes #17557 --- cmd/faucet/faucet.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 6799060275..cfe4e45f11 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -157,7 +157,8 @@ func main() { if blob, err = ioutil.ReadFile(*accPassFlag); err != nil { log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err) } - pass := string(blob) + // Delete trailing newline in password + pass := strings.TrimSuffix(string(blob), "\n") ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP) if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil { From 84084df26c29c3e35075a0624ca7f5bf204b01e8 Mon Sep 17 00:00:00 2001 From: Evangelos Pappas Date: Tue, 4 Sep 2018 12:45:27 +0100 Subject: [PATCH 092/126] cmd/ethkey: fix the README to match updated commands (#17332) --- cmd/ethkey/README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/ethkey/README.md b/cmd/ethkey/README.md index cf72ba43d7..48d3c9e9b7 100644 --- a/cmd/ethkey/README.md +++ b/cmd/ethkey/README.md @@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag; make sure to use this feature with great caution! -### `ethkey sign ` +### `ethkey signmessage ` Sign the message with a keyfile. It is possible to refer to a file containing the message. +To sign a message contained in a file, use the `--msgfile` flag. -### `ethkey verify
` +### `ethkey verifymessage
` Verify the signature of the message. It is possible to refer to a file containing the message. +To sign a message contained in a file, use the --msgfile flag. + + +### `ethkey changepassphrase ` + +Change the passphrase of a keyfile. +use the `--newpasswordfile` to point to the new password file. ## Passphrases For every command that uses a keyfile, you will be prompted to provide the passphrase for decrypting the keyfile. To avoid this message, it is possible -to pass the passphrase by using the `--passphrase` flag pointing to a file that +to pass the passphrase by using the `--passwordfile` flag pointing to a file that contains the passphrase. + +## JSON + +In case you need to output the result in a JSON format, you shall by using the `--json` flag. From 3e8184006189b57e2d998313ef0fa75429b0c5bc Mon Sep 17 00:00:00 2001 From: ligi Date: Tue, 4 Sep 2018 14:12:16 +0200 Subject: [PATCH 093/126] common: fix typo (#17582) Fixes #17581 --- common/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/types.go b/common/types.go index 71fe5c95cd..a4b9995267 100644 --- a/common/types.go +++ b/common/types.go @@ -34,7 +34,7 @@ import ( const ( // HashLength is the expected length of the hash HashLength = 32 - // AddressLength is the expected length of the adddress + // AddressLength is the expected length of the address AddressLength = 20 ) From 661aa4dc20bdd3ac7536fe8d7c41273c7433a1d8 Mon Sep 17 00:00:00 2001 From: Richard Littauer Date: Tue, 4 Sep 2018 08:13:50 -0400 Subject: [PATCH 094/126] .github: slight edits to No Response template (#17475) The template was not grammatical to me as it was. I have edited the language a tiny bit to make the close comment more clear and less abrasive. --- .github/no-response.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/no-response.yml b/.github/no-response.yml index a6227159d1..b6e96efdc6 100644 --- a/.github/no-response.yml +++ b/.github/no-response.yml @@ -7,5 +7,5 @@ closeComment: > This issue has been automatically closed because there has been no response to our request for more information from the original author. With only the information that is currently in the issue, we don't have enough information - to take action. Please reach out if you have or find the answers we need so - that we can investigate further. + to take action. Please reach out if you have more relevant information or + answers to our questions so that we can investigate further. From beee7a52e08f6e948d4cca9de8ff59d5b8305917 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 4 Sep 2018 14:29:00 +0200 Subject: [PATCH 095/126] cmd/swarm: added scaling test for ACT manifests (#17496) --- cmd/swarm/access_test.go | 43 ++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 7a8bcf9d3d..ed589f9f47 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -34,12 +34,15 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" ) +var DefaultCurve = crypto.S256() + // TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded @@ -359,11 +362,22 @@ func TestAccessPK(t *testing.T) { } } +// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) +func TestAccessACT(t *testing.T) { + testAccessACT(t, 0) +} + +// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest) +func TestAccessACTScale(t *testing.T) { + testAccessACT(t, 1000) +} + // TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data // set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through -// one of the nodes then disappears. -func TestAccessACT(t *testing.T) { +// one of the nodes then disappears. If `bogusEntries` is bigger than 0, the test will generate the number of bogus act entries +// to test what happens at scale +func testAccessACT(t *testing.T, bogusEntries int) { // Setup Swarm and upload a test file to it cluster := newTestCluster(t, 3) defer cluster.Shutdown() @@ -415,19 +429,36 @@ func TestAccessACT(t *testing.T) { grantees = append(grantees, hex.EncodeToString(granteePubKey)) } - granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv") + if bogusEntries > 0 { + bogusGrantees := []string{} + + for i := 0; i < bogusEntries; i++ { + prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil) + if err != nil { + t.Fatal(err) + } + bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey))) + } + r2 := gorand.New(gorand.NewSource(time.Now().UnixNano())) + for i := 0; i < len(grantees); i++ { + insertAtIdx := r2.Intn(len(bogusGrantees)) + bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...) + } + grantees = bogusGrantees + } + + granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list") if err != nil { t.Fatal(err) } + defer granteesPubkeyListFile.Close() + defer os.Remove(granteesPubkeyListFile.Name()) _, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) if err != nil { t.Fatal(err) } - defer granteesPubkeyListFile.Close() - defer os.Remove(granteesPubkeyListFile.Name()) - publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") if err != nil { t.Fatal(err) From 42bd67bd6fd599a5583452f011152a0c26c9dff4 Mon Sep 17 00:00:00 2001 From: Diep Pham Date: Tue, 4 Sep 2018 22:53:28 +0700 Subject: [PATCH 096/126] accounts/abi: fix unpacking of negative int256 (#17583) --- accounts/abi/unpack.go | 28 +++++++++++++++++++++++++--- accounts/abi/unpack_test.go | 5 +++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 793d515adf..d5875140cc 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -25,8 +25,17 @@ import ( "github.com/ethereum/go-ethereum/common" ) +var ( + maxUint256 = big.NewInt(0).Add( + big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil), + big.NewInt(-1)) + maxInt256 = big.NewInt(0).Add( + big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil), + big.NewInt(-1)) +) + // reads the integer based on its kind -func readInteger(kind reflect.Kind, b []byte) interface{} { +func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} { switch kind { case reflect.Uint8: return b[len(b)-1] @@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} { case reflect.Int64: return int64(binary.BigEndian.Uint64(b[len(b)-8:])) default: - return new(big.Int).SetBytes(b) + // the only case lefts for integer is int256/uint256. + // big.SetBytes can't tell if a number is negative, positive on itself. + // On EVM, if the returned number > max int256, it is negative. + ret := new(big.Int).SetBytes(b) + if typ == UintTy { + return ret + } + + if ret.Cmp(maxInt256) > 0 { + ret.Add(maxUint256, big.NewInt(0).Neg(ret)) + ret.Add(ret, big.NewInt(1)) + ret.Neg(ret) + } + return ret } } @@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { case StringTy: // variable arrays are written at the end of the return bytes return string(output[begin : begin+end]), nil case IntTy, UintTy: - return readInteger(t.Kind, returnOutput), nil + return readInteger(t.T, t.Kind, returnOutput), nil case BoolTy: return readBool(returnOutput) case AddressTy: diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index bdbab10b4f..97552b90cf 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -117,6 +117,11 @@ var unpackTests = []unpackTest{ enc: "0000000000000000000000000000000000000000000000000000000000000001", want: big.NewInt(1), }, + { + def: `[{"type": "int256"}]`, + enc: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + want: big.NewInt(-1), + }, { def: `[{"type": "address"}]`, enc: "0000000000000000000000000100000000000000000000000000000000000000", From cf33d8b83ce78d1e79cd8c43a21070b2050d5c7e Mon Sep 17 00:00:00 2001 From: Hyung-Kyu Hqueue Choi Date: Wed, 5 Sep 2018 17:29:51 +0900 Subject: [PATCH 097/126] core: fix typo in comment (#17586) --- core/state_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state_processor.go b/core/state_processor.go index 1a91a57ab4..503a35d16a 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -110,7 +110,7 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo *usedGas += gas // Create a new receipt for the transaction, storing the intermediate root and gas used by the tx - // based on the eip phase, we're passing wether the root touch-delete accounts. + // based on the eip phase, we're passing whether the root touch-delete accounts. receipt := types.NewReceipt(root, failed, *usedGas) receipt.TxHash = tx.Hash() receipt.GasUsed = gas From 8711e2b6366109912057e8fb20add325a1051a4e Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 5 Sep 2018 11:57:45 +0300 Subject: [PATCH 098/126] whisper: add light mode check to handshake (#16725) --- cmd/geth/config.go | 3 ++ cmd/geth/main.go | 1 + cmd/utils/flags.go | 7 ++++ whisper/whisperv6/api.go | 8 ++--- whisper/whisperv6/config.go | 10 +++--- whisper/whisperv6/peer.go | 10 +++++- whisper/whisperv6/peer_test.go | 61 ++++++++++++++++++++++++++++++++++ whisper/whisperv6/whisper.go | 44 +++++++++++++++++++----- 8 files changed, 126 insertions(+), 18 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index e6bd4d5bef..b0749d2329 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -168,6 +168,9 @@ func makeFullNode(ctx *cli.Context) *node.Node { if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) { cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name) } + if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) { + cfg.Shh.RestrictConnectionBetweenLightClients = true + } utils.RegisterShhService(stack, &cfg.Shh) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4d7e98698c..134d5a4c01 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -151,6 +151,7 @@ var ( utils.WhisperEnabledFlag, utils.WhisperMaxMessageSizeFlag, utils.WhisperMinPOWFlag, + utils.WhisperRestrictConnectionBetweenLightClientsFlag, } metricsFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 495bfe13e0..f431621baf 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -567,6 +567,10 @@ var ( Usage: "Minimum POW accepted", Value: whisper.DefaultMinimumPoW, } + WhisperRestrictConnectionBetweenLightClientsFlag = cli.BoolFlag{ + Name: "shh.restrict-light", + Usage: "Restrict connection between two whisper light clients", + } // Metrics flags MetricsEnabledFlag = cli.BoolFlag{ @@ -1099,6 +1103,9 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { if ctx.GlobalIsSet(WhisperMinPOWFlag.Name) { cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(WhisperMinPOWFlag.Name) } + if ctx.GlobalIsSet(WhisperRestrictConnectionBetweenLightClientsFlag.Name) { + cfg.RestrictConnectionBetweenLightClients = true + } } // SetEthConfig applies eth-related command line flags to the config. diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index d729e79c35..e1dab7b226 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -195,14 +195,14 @@ func (api *PublicWhisperAPI) DeleteSymKey(ctx context.Context, id string) bool { // MakeLightClient turns the node into light client, which does not forward // any incoming messages, and sends only messages originated in this node. func (api *PublicWhisperAPI) MakeLightClient(ctx context.Context) bool { - api.w.lightClient = true - return api.w.lightClient + api.w.SetLightClientMode(true) + return api.w.LightClientMode() } // CancelLightClient cancels light client mode. func (api *PublicWhisperAPI) CancelLightClient(ctx context.Context) bool { - api.w.lightClient = false - return !api.w.lightClient + api.w.SetLightClientMode(false) + return !api.w.LightClientMode() } //go:generate gencodec -type NewMessage -field-override newMessageOverride -out gen_newmessage_json.go diff --git a/whisper/whisperv6/config.go b/whisper/whisperv6/config.go index 61419de007..38eb9551cc 100644 --- a/whisper/whisperv6/config.go +++ b/whisper/whisperv6/config.go @@ -18,12 +18,14 @@ package whisperv6 // Config represents the configuration state of a whisper node. type Config struct { - MaxMessageSize uint32 `toml:",omitempty"` - MinimumAcceptedPOW float64 `toml:",omitempty"` + MaxMessageSize uint32 `toml:",omitempty"` + MinimumAcceptedPOW float64 `toml:",omitempty"` + RestrictConnectionBetweenLightClients bool `toml:",omitempty"` } // DefaultConfig represents (shocker!) the default configuration. var DefaultConfig = Config{ - MaxMessageSize: DefaultMaxMessageSize, - MinimumAcceptedPOW: DefaultMinimumPoW, + MaxMessageSize: DefaultMaxMessageSize, + MinimumAcceptedPOW: DefaultMinimumPoW, + RestrictConnectionBetweenLightClients: true, } diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 79cc212708..621d512081 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -79,11 +79,14 @@ func (peer *Peer) stop() { func (peer *Peer) handshake() error { // Send the handshake status message asynchronously errc := make(chan error, 1) + isLightNode := peer.host.LightClientMode() + isRestrictedLightNodeConnection := peer.host.LightClientModeConnectionRestricted() go func() { pow := peer.host.MinPow() powConverted := math.Float64bits(pow) bloom := peer.host.BloomFilter() - errc <- p2p.SendItems(peer.ws, statusCode, ProtocolVersion, powConverted, bloom) + + errc <- p2p.SendItems(peer.ws, statusCode, ProtocolVersion, powConverted, bloom, isLightNode) }() // Fetch the remote status packet and verify protocol match @@ -127,6 +130,11 @@ func (peer *Peer) handshake() error { } } + isRemotePeerLightNode, err := s.Bool() + if isRemotePeerLightNode && isLightNode && isRestrictedLightNodeConnection { + return fmt.Errorf("peer [%x] is useless: two light client communication restricted", peer.ID()) + } + if err := <-errc; err != nil { return fmt.Errorf("peer [%x] failed to send status packet: %v", peer.ID(), err) } diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 0c9b380901..fe31922cb2 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/nat" + "github.com/ethereum/go-ethereum/rlp" ) var keys = []string{ @@ -507,3 +508,63 @@ func waitForServersToStart(t *testing.T) { } t.Fatalf("Failed to start all the servers, running: %d", started) } + +//two generic whisper node handshake +func TestPeerHandshakeWithTwoFullNode(t *testing.T) { + w1 := Whisper{} + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize), false}}) + err := p1.handshake() + if err != nil { + t.Fatal() + } +} + +//two generic whisper node handshake. one don't send light flag +func TestHandshakeWithOldVersionWithoutLightModeFlag(t *testing.T) { + w1 := Whisper{} + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize)}}) + err := p1.handshake() + if err != nil { + t.Fatal() + } +} + +//two light nodes handshake. restriction disabled +func TestTwoLightPeerHandshakeRestrictionOff(t *testing.T) { + w1 := Whisper{} + w1.settings.Store(restrictConnectionBetweenLightClientsIdx, false) + w1.SetLightClientMode(true) + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize), true}}) + err := p1.handshake() + if err != nil { + t.FailNow() + } +} + +//two light nodes handshake. restriction enabled +func TestTwoLightPeerHandshakeError(t *testing.T) { + w1 := Whisper{} + w1.settings.Store(restrictConnectionBetweenLightClientsIdx, true) + w1.SetLightClientMode(true) + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize), true}}) + err := p1.handshake() + if err == nil { + t.FailNow() + } +} + +type rwStub struct { + payload []interface{} +} + +func (stub *rwStub) ReadMsg() (p2p.Msg, error) { + size, r, err := rlp.EncodeToReader(stub.payload) + if err != nil { + return p2p.Msg{}, err + } + return p2p.Msg{Code: statusCode, Size: uint32(size), Payload: r}, nil +} + +func (stub *rwStub) WriteMsg(m p2p.Msg) error { + return nil +} diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index be633e7ff6..eb713f84ee 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -49,12 +49,14 @@ type Statistics struct { } const ( - maxMsgSizeIdx = iota // Maximal message length allowed by the whisper node - overflowIdx // Indicator of message queue overflow - minPowIdx // Minimal PoW required by the whisper node - minPowToleranceIdx // Minimal PoW tolerated by the whisper node for a limited time - bloomFilterIdx // Bloom filter for topics of interest for this node - bloomFilterToleranceIdx // Bloom filter tolerated by the whisper node for a limited time + maxMsgSizeIdx = iota // Maximal message length allowed by the whisper node + overflowIdx // Indicator of message queue overflow + minPowIdx // Minimal PoW required by the whisper node + minPowToleranceIdx // Minimal PoW tolerated by the whisper node for a limited time + bloomFilterIdx // Bloom filter for topics of interest for this node + bloomFilterToleranceIdx // Bloom filter tolerated by the whisper node for a limited time + lightClientModeIdx // Light client mode. (does not forward any messages) + restrictConnectionBetweenLightClientsIdx // Restrict connection between two light clients ) // Whisper represents a dark communication interface through the Ethereum @@ -82,8 +84,6 @@ type Whisper struct { syncAllowance int // maximum time in seconds allowed to process the whisper-related messages - lightClient bool // indicates is this node is pure light client (does not forward any messages) - statsMu sync.Mutex // guard stats stats Statistics // Statistics of whisper node @@ -113,6 +113,7 @@ func New(cfg *Config) *Whisper { whisper.settings.Store(minPowIdx, cfg.MinimumAcceptedPOW) whisper.settings.Store(maxMsgSizeIdx, cfg.MaxMessageSize) whisper.settings.Store(overflowIdx, false) + whisper.settings.Store(restrictConnectionBetweenLightClientsIdx, cfg.RestrictConnectionBetweenLightClients) // p2p whisper sub protocol handler whisper.protocol = p2p.Protocol{ @@ -276,6 +277,31 @@ func (whisper *Whisper) SetMinimumPowTest(val float64) { whisper.settings.Store(minPowToleranceIdx, val) } +//SetLightClientMode makes node light client (does not forward any messages) +func (whisper *Whisper) SetLightClientMode(v bool) { + whisper.settings.Store(lightClientModeIdx, v) +} + +//LightClientMode indicates is this node is light client (does not forward any messages) +func (whisper *Whisper) LightClientMode() bool { + val, exist := whisper.settings.Load(lightClientModeIdx) + if !exist || val == nil { + return false + } + v, ok := val.(bool) + return v && ok +} + +//LightClientModeConnectionRestricted indicates that connection to light client in light client mode not allowed +func (whisper *Whisper) LightClientModeConnectionRestricted() bool { + val, exist := whisper.settings.Load(restrictConnectionBetweenLightClientsIdx) + if !exist || val == nil { + return false + } + v, ok := val.(bool) + return v && ok +} + func (whisper *Whisper) notifyPeersAboutPowRequirementChange(pow float64) { arr := whisper.getPeers() for _, p := range arr { @@ -672,7 +698,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { trouble := false for _, env := range envelopes { - cached, err := whisper.add(env, whisper.lightClient) + cached, err := whisper.add(env, whisper.LightClientMode()) if err != nil { trouble = true log.Error("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) From 5918b88a8f8be34f39b8df94d146f77ac9b4fbfc Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 5 Sep 2018 11:33:07 +0200 Subject: [PATCH 099/126] cmd/swarm: added publisher key assertion to act tests (#17471) --- cmd/swarm/access_test.go | 44 ++++++++++++++++++++++++++++++++++++---- cmd/swarm/main.go | 24 ++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index ed589f9f47..384d256306 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -145,7 +145,9 @@ func TestAccessPassword(t *testing.T) { if a.KdfParams == nil { t.Fatal("manifest access kdf params is nil") } - + if a.Publisher != "" { + t.Fatal("should be empty") + } client := swarm.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) @@ -222,7 +224,7 @@ func TestAccessPassword(t *testing.T) { // the test will fail if the proxy's given private key is not granted on the ACT. func TestAccessPK(t *testing.T) { // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 1) + cluster := newTestCluster(t, 2) defer cluster.Shutdown() // create a tmp file @@ -302,6 +304,20 @@ func TestAccessPK(t *testing.T) { t.Fatalf("stdout not matched") } + //get the public key from the publisher directory + publicKeyFromDataDir := runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "print-keys", + "--compressed", + ) + _, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+") + publicKeyFromDataDir.ExpectExit() + pkComp := strings.Split(publicKeyString[0], "=")[1] var m api.Manifest err = json.Unmarshal([]byte(matches[0]), &m) @@ -335,7 +351,9 @@ func TestAccessPK(t *testing.T) { if a.KdfParams != nil { t.Fatal("manifest access kdf params should be nil") } - + if a.Publisher != pkComp { + t.Fatal("publisher key did not match") + } client := swarm.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) @@ -499,6 +517,22 @@ func testAccessACT(t *testing.T, bogusEntries int) { if len(matches) == 0 { t.Fatalf("stdout not matched") } + + //get the public key from the publisher directory + publicKeyFromDataDir := runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "print-keys", + "--compressed", + ) + _, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+") + publicKeyFromDataDir.ExpectExit() + pkComp := strings.Split(publicKeyString[0], "=")[1] + hash := matches[0] m, _, err := client.DownloadManifest(hash) if err != nil { @@ -531,7 +565,9 @@ func testAccessACT(t *testing.T, bogusEntries int) { if a.KdfParams != nil { t.Fatal("manifest access kdf params should be nil") } - + if a.Publisher != pkComp { + t.Fatal("publisher key did not match") + } httpClient := &http.Client{} // all nodes except the skipped node should be able to decrypt the content diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 637ae06e96..e654409371 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -18,6 +18,7 @@ package main import ( "crypto/ecdsa" + "encoding/hex" "fmt" "io/ioutil" "os" @@ -208,6 +209,10 @@ var ( Name: "data", Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x", } + SwarmCompressedFlag = cli.BoolFlag{ + Name: "compressed", + Usage: "Prints encryption keys in compressed form", + } ) //declare a few constant error messages, useful for later error check comparisons in test @@ -252,6 +257,14 @@ func init() { Usage: "Print version numbers", Description: "The output of this command is supposed to be machine-readable", }, + { + Action: keys, + CustomHelpTemplate: helpTemplate, + Name: "print-keys", + Flags: []cli.Flag{SwarmCompressedFlag}, + Usage: "Print public key information", + Description: "The output of this command is supposed to be machine-readable", + }, { Action: upload, CustomHelpTemplate: helpTemplate, @@ -580,6 +593,17 @@ func main() { } } +func keys(ctx *cli.Context) error { + privateKey := getPrivKey(ctx) + pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey)) + pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + if !ctx.Bool(SwarmCompressedFlag.Name) { + fmt.Println(fmt.Sprintf("publicKey=%s", pub)) + } + fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed)) + return nil +} + func version(ctx *cli.Context) error { fmt.Println(strings.Title(clientIdentifier)) fmt.Println("Version:", sv.VersionWithMeta) From 4c15ffffdd0aaf3b834be88ff7e7d9ea938c45b4 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 6 Sep 2018 12:08:39 +0200 Subject: [PATCH 100/126] swarm/api/http: added a regression test for resolver bug from #17483 (#17502) --- swarm/api/client/client_test.go | 14 ++-- swarm/api/http/response_test.go | 10 +-- swarm/api/http/server_test.go | 143 ++++++++++++++++++++++++++++++-- swarm/testutil/http.go | 4 +- 4 files changed, 148 insertions(+), 23 deletions(-) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 2212f5c4c3..f9312d48fe 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -47,7 +47,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) { } func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -88,7 +88,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) { } func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -186,7 +186,7 @@ func newTestDirectory(t *testing.T) string { // TestClientUploadDownloadDirectory tests uploading and downloading a // directory of files to a swarm manifest func TestClientUploadDownloadDirectory(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -252,7 +252,7 @@ func TestClientFileListEncrypted(t *testing.T) { } func testClientFileList(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -310,7 +310,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { // TestClientMultipartUpload tests uploading files to swarm using a multipart // upload func TestClientMultipartUpload(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // define an uploader which uploads testDirFiles with some data @@ -376,7 +376,7 @@ func TestClientCreateResourceMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() @@ -439,7 +439,7 @@ func TestClientCreateUpdateResource(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() diff --git a/swarm/api/http/response_test.go b/swarm/api/http/response_test.go index 2e24bda6d1..50a704be6d 100644 --- a/swarm/api/http/response_test.go +++ b/swarm/api/http/response_test.go @@ -29,7 +29,7 @@ import ( ) func TestError(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -55,7 +55,7 @@ func TestError(t *testing.T) { } func Test404Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -81,7 +81,7 @@ func Test404Page(t *testing.T) { } func Test500Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -106,7 +106,7 @@ func Test500Page(t *testing.T) { } } func Test500PageWith0xHashPrefix(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -136,7 +136,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) { } func TestJsonResponse(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 7934e37eba..891deb8817 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "io/ioutil" + "math/big" "mime/multipart" "net/http" "os" @@ -36,6 +37,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" @@ -114,7 +116,7 @@ func TestBzzResourceMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // add the data our multihash aliased manifest will point to @@ -208,7 +210,7 @@ func TestBzzResourceMultihash(t *testing.T) { // Test resource updates using the raw update methods func TestBzzResource(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) signer, _ := newTestSigner() defer srv.Close() @@ -467,7 +469,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { addr := [3]storage.Address{} - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for i, mf := range testmanifest { @@ -702,7 +704,7 @@ func TestBzzTar(t *testing.T) { } func testBzzTar(encrypted bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} @@ -827,7 +829,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) { } func testBzzRootRedirect(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // create a manifest with some data at the root path @@ -882,7 +884,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { } func TestMethodsNotAllowed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() databytes := "bar" for _, c := range []struct { @@ -941,7 +943,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string } func TestGet(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for _, testCase := range []struct { @@ -1025,7 +1027,7 @@ func TestGet(t *testing.T) { } func TestModify(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() swarmClient := swarm.NewClient(srv.URL) @@ -1126,7 +1128,7 @@ func TestMultiPartUpload(t *testing.T) { // POST /bzz:/ Content-Type: multipart/form-data verbose := false // Setup Swarm - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() url := fmt.Sprintf("%s/bzz:/", srv.URL) @@ -1153,3 +1155,126 @@ func TestMultiPartUpload(t *testing.T) { t.Fatalf("expected POST multipart/form-data to return a 64 char manifest but the answer was %d chars long", len(body)) } } + +// TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver +func TestBzzGetFileWithResolver(t *testing.T) { + resolver := newTestResolveValidator("") + srv := testutil.NewTestSwarmServer(t, serverFunc, resolver) + defer srv.Close() + fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"} + fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} + + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + + for i, v := range fileNames { + size := len(fileContents[i]) + hdr := &tar.Header{ + Name: v, + Mode: 0644, + Size: int64(size), + ModTime: time.Now(), + Xattrs: map[string]string{ + "user.swarm.content-type": "text/plain", + }, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + // copy the file into the tar stream + n, err := io.WriteString(tw, fileContents[i]) + if err != nil { + t.Fatal(err) + } else if n != size { + t.Fatal("size mismatch") + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + //post tar stream + url := srv.URL + "/bzz:/" + + req, err := http.NewRequest("POST", url, buf) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Content-Type", "application/x-tar") + client := &http.Client{} + serverResponse, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + if serverResponse.StatusCode != http.StatusOK { + t.Fatalf("err %s", serverResponse.Status) + } + swarmHash, err := ioutil.ReadAll(serverResponse.Body) + serverResponse.Body.Close() + if err != nil { + t.Fatal(err) + } + // set the resolved hash to be the swarm hash of what we've just uploaded + hash := common.HexToHash(string(swarmHash)) + resolver.hash = &hash + for _, v := range []struct { + addr string + path string + expectedStatusCode int + }{ + { + addr: string(swarmHash), + path: fileNames[0], + expectedStatusCode: http.StatusOK, + }, + { + addr: "somebogusensname", + path: fileNames[0], + expectedStatusCode: http.StatusOK, + }, + } { + req, err := http.NewRequest("GET", fmt.Sprintf(srv.URL+"/bzz:/%s/%s", v.addr, v.path), nil) + if err != nil { + t.Fatal(err) + } + serverResponse, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer serverResponse.Body.Close() + if serverResponse.StatusCode != v.expectedStatusCode { + t.Fatalf("expected %d, got %d", v.expectedStatusCode, serverResponse.StatusCode) + } + } +} + +// testResolver implements the Resolver interface and either returns the given +// hash if it is set, or returns a "name not found" error +type testResolveValidator struct { + hash *common.Hash +} + +func newTestResolveValidator(addr string) *testResolveValidator { + r := &testResolveValidator{} + if addr != "" { + hash := common.HexToHash(addr) + r.hash = &hash + } + return r +} + +func (t *testResolveValidator) Resolve(addr string) (common.Hash, error) { + if t.hash == nil { + return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr) + } + return *t.hash, nil +} + +func (t *testResolveValidator) Owner(node [32]byte) (addr common.Address, err error) { + return +} +func (t *testResolveValidator) HeaderByNumber(context.Context, *big.Int) (header *types.Header, err error) { + return +} diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 7fd60fcc3d..0748230329 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -45,7 +45,7 @@ func (f *fakeTimeProvider) Now() mru.Timestamp { return mru.Timestamp{Time: f.currentTime} } -func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *TestSwarmServer { +func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, resolver api.Resolver) *TestSwarmServer { dir, err := ioutil.TempDir("", "swarm-storage-test") if err != nil { t.Fatal(err) @@ -77,7 +77,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *Tes t.Fatal(err) } - a := api.NewAPI(fileStore, nil, rh.Handler, nil) + a := api.NewAPI(fileStore, resolver, rh.Handler, nil) srv := httptest.NewServer(serverFunc(a)) return &TestSwarmServer{ Server: srv, From 580145e96db848cb8e2f8bb8f0621bcacbc9521c Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 6 Sep 2018 12:11:38 +0200 Subject: [PATCH 101/126] swarm/storage: added metrics for db entry count (#17589) --- swarm/api/http/server_test.go | 39 +++++++++++++++++------------------ swarm/storage/ldbstore.go | 6 ++++++ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 891deb8817..2b6a97ebdf 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -893,14 +893,14 @@ func TestMethodsNotAllowed(t *testing.T) { }{ { url: fmt.Sprintf("%s/bzz-list:/", srv.URL), - code: 405, + code: http.StatusMethodNotAllowed, }, { url: fmt.Sprintf("%s/bzz-hash:/", srv.URL), - code: 405, + code: http.StatusMethodNotAllowed, }, { url: fmt.Sprintf("%s/bzz-immutable:/", srv.URL), - code: 405, + code: http.StatusMethodNotAllowed, }, } { res, _ := http.Post(c.url, "text/plain", bytes.NewReader([]byte(databytes))) @@ -958,7 +958,7 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/", srv.URL), method: "GET", headers: map[string]string{"Accept": "text/html"}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution", verbose: false, }, @@ -966,7 +966,7 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/", srv.URL), method: "GET", headers: map[string]string{"Accept": "application/json"}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", verbose: false, }, @@ -974,7 +974,7 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/robots.txt", srv.URL), method: "GET", headers: map[string]string{"Accept": "text/html"}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "User-agent: *\nDisallow: /", verbose: false, }, @@ -982,38 +982,37 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/nonexistent_path", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/bzz:asdf/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/tbz2/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/bzz-rack:/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/bzz-ls", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, - }, - } { + }} { t.Run("GET "+testCase.uri, func(t *testing.T) { res, body := httpDo(testCase.method, testCase.uri, nil, testCase.headers, testCase.verbose, t) if res.StatusCode != testCase.expectedStatusCode { @@ -1060,7 +1059,7 @@ func TestModify(t *testing.T) { uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "DELETE", headers: map[string]string{}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "8b634aea26eec353ac0ecbec20c94f44d6f8d11f38d4578a4c207a84c74ef731", verbose: false, }, @@ -1068,21 +1067,21 @@ func TestModify(t *testing.T) { uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "PUT", headers: map[string]string{}, - expectedStatusCode: 405, + expectedStatusCode: http.StatusMethodNotAllowed, verbose: false, }, { uri: fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, hash), method: "PUT", headers: map[string]string{}, - expectedStatusCode: 405, + expectedStatusCode: http.StatusMethodNotAllowed, verbose: false, }, { uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "PATCH", headers: map[string]string{}, - expectedStatusCode: 405, + expectedStatusCode: http.StatusMethodNotAllowed, verbose: false, }, { @@ -1090,7 +1089,7 @@ func TestModify(t *testing.T) { method: "POST", headers: map[string]string{}, requestBody: []byte("POSTdata"), - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseHeaders: map[string]string{"Content-Length": "64"}, verbose: false, }, @@ -1099,7 +1098,7 @@ func TestModify(t *testing.T) { method: "POST", headers: map[string]string{}, requestBody: []byte("POSTdata"), - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseHeaders: map[string]string{"Content-Length": "128"}, verbose: false, }, @@ -1148,7 +1147,7 @@ func TestMultiPartUpload(t *testing.T) { } res, body := httpDo("POST", url, buf, headers, verbose, t) - if res.StatusCode != 200 { + if res.StatusCode != http.StatusOK { t.Fatalf("expected POST multipart/form-data to return 200, but it returned %d", res.StatusCode) } if len(body) != 64 { diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index b95aa13b09..bd3501ea26 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -48,6 +48,10 @@ const ( maxGCitems = 5000 // max number of items to be gc'd per call to collectGarbage() ) +var ( + dbEntryCount = metrics.NewRegisteredCounter("ldbstore.entryCnt", nil) +) + var ( keyIndex = byte(0) keyOldData = byte(1) @@ -495,6 +499,7 @@ func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { batch.Delete(idxKey) batch.Delete(getDataKey(idx, po)) s.entryCnt-- + dbEntryCount.Dec(1) s.bucketCnt[po]-- cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt @@ -566,6 +571,7 @@ func (s *LDBStore) doPut(chunk *Chunk, index *dpaDBIndex, po uint8) { index.Idx = s.dataIdx s.bucketCnt[po] = s.dataIdx s.entryCnt++ + dbEntryCount.Inc(1) s.dataIdx++ cntKey := make([]byte, 2) From 70d31fb27842b047582a6557529b2234de1a4a8d Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 7 Sep 2018 09:56:05 +0200 Subject: [PATCH 102/126] cmd/swarm: added password to ACT (#17598) --- cmd/swarm/access.go | 49 +++++++---- cmd/swarm/access_test.go | 176 +++++++++++++++------------------------ cmd/swarm/main.go | 1 + swarm/api/act.go | 149 ++++++++++++++++++++++++--------- swarm/testutil/file.go | 44 ++++++++++ 5 files changed, 250 insertions(+), 169 deletions(-) create mode 100644 swarm/testutil/file.go diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index 12cfbfc1a4..1e69526ec3 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -51,7 +51,7 @@ func accessNewPass(ctx *cli.Context) { password = getPassPhrase("", 0, makePasswordList(ctx)) dryRun = ctx.Bool(SwarmDryRunFlag.Name) ) - accessKey, ae, err = api.DoPasswordNew(ctx, password, salt) + accessKey, ae, err = api.DoPassword(ctx, password, salt) if err != nil { utils.Fatalf("error getting session key: %v", err) } @@ -85,7 +85,7 @@ func accessNewPK(ctx *cli.Context) { granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name) dryRun = ctx.Bool(SwarmDryRunFlag.Name) ) - sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt) + sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt) if err != nil { utils.Fatalf("error getting session key: %v", err) } @@ -110,23 +110,38 @@ func accessNewACT(ctx *cli.Context) { } var ( - ae *api.AccessEntry - actManifest *api.Manifest - accessKey []byte - err error - ref = args[0] - grantees = []string{} - actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) - privateKey = getPrivKey(ctx) - dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ae *api.AccessEntry + actManifest *api.Manifest + accessKey []byte + err error + ref = args[0] + pkGrantees = []string{} + passGrantees = []string{} + pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) + passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name) + privateKey = getPrivKey(ctx) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) ) - - bytes, err := ioutil.ReadFile(actFilename) - if err != nil { - utils.Fatalf("had an error reading the grantee public key list") + if pkGranteesFilename == "" && passGranteesFilename == "" { + utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)") } - grantees = strings.Split(string(bytes), "\n") - accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees) + + if pkGranteesFilename != "" { + bytes, err := ioutil.ReadFile(pkGranteesFilename) + if err != nil { + utils.Fatalf("had an error reading the grantee public key list") + } + pkGrantees = strings.Split(string(bytes), "\n") + } + + if passGranteesFilename != "" { + bytes, err := ioutil.ReadFile(passGranteesFilename) + if err != nil { + utils.Fatalf("could not read password filename: %v", err) + } + passGrantees = strings.Split(string(bytes), "\n") + } + accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees) if err != nil { utils.Fatalf("error generating ACT manifest: %v", err) } diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 384d256306..106e6dd912 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -28,7 +28,6 @@ import ( gorand "math/rand" "net/http" "os" - "path/filepath" "strings" "testing" "time" @@ -39,6 +38,12 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +const ( + hashRegexp = `[a-f\d]{128}` + data = "notsorandomdata" ) var DefaultCurve = crypto.S256() @@ -53,23 +58,8 @@ func TestAccessPassword(t *testing.T) { defer cluster.Shutdown() proxyNode := cluster.Nodes[0] - // create a tmp file - tmp, err := ioutil.TempDir("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) - - // write data to file - data := "notsorandomdata" - dataFilename := filepath.Join(tmp, "data.txt") - - err = ioutil.WriteFile(dataFilename, []byte(data), 0666) - if err != nil { - t.Fatal(err) - } - - hashRegexp := `[a-f\d]{128}` + dataFilename := testutil.TempFileWithContent(t, data) + defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, @@ -86,14 +76,14 @@ func TestAccessPassword(t *testing.T) { } ref := matches[0] - - password := "smth" - passwordFilename := filepath.Join(tmp, "password.txt") - - err = ioutil.WriteFile(passwordFilename, []byte(password), 0666) + tmp, err := ioutil.TempDir("", "swarm-test") if err != nil { t.Fatal(err) } + defer os.RemoveAll(tmp) + password := "smth" + passwordFilename := testutil.TempFileWithContent(t, "smth") + defer os.RemoveAll(passwordFilename) up = runSwarm(t, "access", @@ -193,12 +183,8 @@ func TestAccessPassword(t *testing.T) { t.Errorf("expected decrypted data %q, got %q", data, string(d)) } - wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt") - - err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666) - if err != nil { - t.Fatal(err) - } + wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng") + defer os.RemoveAll(wrongPasswordFilename) //download file with 'swarm down' with wrong password up = runSwarm(t, @@ -227,22 +213,8 @@ func TestAccessPK(t *testing.T) { cluster := newTestCluster(t, 2) defer cluster.Shutdown() - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer tmp.Close() - defer os.Remove(tmp.Name()) - - // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - - hashRegexp := `[a-f\d]{128}` + dataFilename := testutil.TempFileWithContent(t, data) + defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, @@ -250,7 +222,7 @@ func TestAccessPK(t *testing.T) { cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()) + dataFilename) _, matches := up.ExpectRegexp(hashRegexp) up.ExpectExit() @@ -259,7 +231,6 @@ func TestAccessPK(t *testing.T) { } ref := matches[0] - pk := cluster.Nodes[0].PrivateKey granteePubKey := crypto.CompressPubkey(&pk.PublicKey) @@ -268,22 +239,15 @@ func TestAccessPK(t *testing.T) { t.Fatal(err) } - passFile, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer passFile.Close() - defer os.Remove(passFile.Name()) - _, err = io.WriteString(passFile, testPassphrase) - if err != nil { - t.Fatal(err) - } + passwordFilename := testutil.TempFileWithContent(t, testPassphrase) + defer os.RemoveAll(passwordFilename) + _, publisherAccount := getTestAccount(t, publisherDir) up = runSwarm(t, "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "--bzzapi", @@ -309,7 +273,7 @@ func TestAccessPK(t *testing.T) { "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "print-keys", @@ -390,37 +354,24 @@ func TestAccessACTScale(t *testing.T) { testAccessACT(t, 1000) } -// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control +// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data -// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through -// one of the nodes then disappears. If `bogusEntries` is bigger than 0, the test will generate the number of bogus act entries -// to test what happens at scale +// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access. +// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails. +// the publisher uploads through one of the nodes then disappears. func testAccessACT(t *testing.T, bogusEntries int) { // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 3) + const clusterSize = 3 + cluster := newTestCluster(t, clusterSize) defer cluster.Shutdown() var uploadThroughNode = cluster.Nodes[0] client := swarm.NewClient(uploadThroughNode.URL) r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) - nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`) - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer tmp.Close() - defer os.Remove(tmp.Name()) - - // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - - hashRegexp := `[a-f\d]{128}` + nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`) + dataFilename := testutil.TempFileWithContent(t, data) + defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, @@ -428,7 +379,7 @@ func testAccessACT(t *testing.T, bogusEntries int) { cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()) + dataFilename) _, matches := up.ExpectRegexp(hashRegexp) up.ExpectExit() @@ -464,41 +415,25 @@ func testAccessACT(t *testing.T, bogusEntries int) { } grantees = bogusGrantees } - - granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list") - if err != nil { - t.Fatal(err) - } - defer granteesPubkeyListFile.Close() - defer os.Remove(granteesPubkeyListFile.Name()) - - _, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) - if err != nil { - t.Fatal(err) - } + granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n")) + defer os.RemoveAll(granteesPubkeyListFile) publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") if err != nil { t.Fatal(err) } + defer os.RemoveAll(publisherDir) - passFile, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer passFile.Close() - defer os.Remove(passFile.Name()) - _, err = io.WriteString(passFile, testPassphrase) - if err != nil { - t.Fatal(err) - } - + passwordFilename := testutil.TempFileWithContent(t, testPassphrase) + defer os.RemoveAll(passwordFilename) + actPasswordFilename := testutil.TempFileWithContent(t, "smth") + defer os.RemoveAll(actPasswordFilename) _, publisherAccount := getTestAccount(t, publisherDir) up = runSwarm(t, "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "--bzzapi", @@ -507,7 +442,9 @@ func testAccessACT(t *testing.T, bogusEntries int) { "new", "act", "--grant-keys", - granteesPubkeyListFile.Name(), + granteesPubkeyListFile, + "--password", + actPasswordFilename, ref, ) @@ -523,7 +460,7 @@ func testAccessACT(t *testing.T, bogusEntries int) { "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "print-keys", @@ -562,9 +499,7 @@ func testAccessACT(t *testing.T, bogusEntries int) { if len(a.Salt) < 32 { t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) } - if a.KdfParams != nil { - t.Fatal("manifest access kdf params should be nil") - } + if a.Publisher != pkComp { t.Fatal("publisher key did not match") } @@ -588,6 +523,25 @@ func testAccessACT(t *testing.T, bogusEntries int) { t.Fatalf("should be a 401") } + // try downloading using a password instead, using the unauthorized node + passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1) + response, err = httpClient.Get(passwordUrl) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatal("should be a 200") + } + + // now try with the wrong password, expect 401 + passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1) + response, err = httpClient.Get(passwordUrl) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusUnauthorized { + t.Fatal("should be a 401") + } continue } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index e654409371..c93344c420 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -319,6 +319,7 @@ func init() { Flags: []cli.Flag{ SwarmAccessGrantKeysFlag, SwarmDryRunFlag, + utils.PasswordFileFlag, }, Name: "act", Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", diff --git a/swarm/api/act.go b/swarm/api/act.go index b1a5947831..52d9098271 100644 --- a/swarm/api/act.go +++ b/swarm/api/act.go @@ -102,6 +102,7 @@ const AccessTypePass = AccessType("pass") const AccessTypePK = AccessType("pk") const AccessTypeACT = AccessType("act") +// NewAccessEntryPassword creates a manifest AccessEntry in order to create an ACT protected by a password func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) { if len(salt) != 32 { return nil, fmt.Errorf("salt should be 32 bytes long") @@ -113,6 +114,7 @@ func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, er }, nil } +// NewAccessEntryPK creates a manifest AccessEntry in order to create an ACT protected by a pair of Elliptic Curve keys func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) { if len(publisher) != 66 { return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher)) @@ -127,6 +129,7 @@ func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) { }, nil } +// NewAccessEntryACT creates a manifest AccessEntry in order to create an ACT protected by a combination of EC keys and passwords func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) { if len(salt) != 32 { return nil, fmt.Errorf("salt should be 32 bytes long") @@ -140,15 +143,19 @@ func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, Publisher: publisher, Salt: salt, Act: act, + KdfParams: DefaultKdfParams, }, nil } +// NOOPDecrypt is a generic decrypt function that is passed into the API in places where real ACT decryption capabilities are +// either unwanted, or alternatively, cannot be implemented in the immediate scope func NOOPDecrypt(*ManifestEntry) error { return nil } var DefaultKdfParams = NewKdfParams(262144, 1, 8) +// NewKdfParams returns a KdfParams struct with the given scrypt params func NewKdfParams(n, p, r int) *KdfParams { return &KdfParams{ @@ -161,15 +168,20 @@ func NewKdfParams(n, p, r int) *KdfParams { // NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt // and kdf parameters in the access entry func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) { - if accessEntry.Type != AccessTypePass { + if accessEntry.Type != AccessTypePass && accessEntry.Type != AccessTypeACT { return nil, errors.New("incorrect access entry type") + } + return sessionKeyPassword(password, accessEntry.Salt, accessEntry.KdfParams) +} + +func sessionKeyPassword(password string, salt []byte, kdfParams *KdfParams) ([]byte, error) { return scrypt.Key( []byte(password), - accessEntry.Salt, - accessEntry.KdfParams.N, - accessEntry.KdfParams.R, - accessEntry.KdfParams.P, + salt, + kdfParams.N, + kdfParams.R, + kdfParams.P, 32, ) } @@ -188,9 +200,6 @@ func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt [] return sessionKey, nil } -func (a *API) NodeSessionKey(privateKey *ecdsa.PrivateKey, publicKey *ecdsa.PublicKey, salt []byte) ([]byte, error) { - return NewSessionKeyPK(privateKey, publicKey, salt) -} func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc { return func(m *ManifestEntry) error { if m.Access == nil { @@ -242,7 +251,7 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva if err != nil { return ErrDecrypt } - key, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + key, err := NewSessionKeyPK(pk, publisher, m.Access.Salt) if err != nil { return ErrDecrypt } @@ -261,6 +270,11 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva m.Access = nil return nil case "act": + var ( + sessionKey []byte + err error + ) + publisherBytes, err := hex.DecodeString(m.Access.Publisher) if err != nil { return ErrDecrypt @@ -270,40 +284,35 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva return ErrDecrypt } - sessionKey, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + sessionKey, err = NewSessionKeyPK(pk, publisher, m.Access.Salt) if err != nil { return ErrDecrypt } - hasher := sha3.NewKeccak256() - hasher.Write(append(sessionKey, 0)) - lookupKey := hasher.Sum(nil) - - hasher.Reset() - - hasher.Write(append(sessionKey, 1)) - accessKeyDecryptionKey := hasher.Sum(nil) - - lk := hex.EncodeToString(lookupKey) - list, err := a.GetManifestList(ctx, NOOPDecrypt, storage.Address(common.Hex2Bytes(m.Access.Act)), lk) - - found := "" - for _, v := range list.Entries { - if v.Path == lk { - found = v.Hash - } - } - - if found == "" { - return ErrDecrypt - } - - v, err := hex.DecodeString(found) + found, ciphertext, decryptionKey, err := a.getACTDecryptionKey(ctx, storage.Address(common.Hex2Bytes(m.Access.Act)), sessionKey) if err != nil { return err } - enc := NewRefEncryption(len(v) - 8) - decodedRef, err := enc.Decrypt(v, accessKeyDecryptionKey) + if !found { + // try to fall back to password + if credentials != "" { + sessionKey, err = NewSessionKeyPassword(credentials, m.Access) + if err != nil { + return err + } + found, ciphertext, decryptionKey, err = a.getACTDecryptionKey(ctx, storage.Address(common.Hex2Bytes(m.Access.Act)), sessionKey) + if err != nil { + return err + } + if !found { + return ErrDecrypt + } + } else { + return ErrDecrypt + } + } + enc := NewRefEncryption(len(ciphertext) - 8) + decodedRef, err := enc.Decrypt(ciphertext, decryptionKey) if err != nil { return ErrDecrypt } @@ -326,6 +335,33 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva } } +func (a *API) getACTDecryptionKey(ctx context.Context, actManifestAddress storage.Address, sessionKey []byte) (found bool, ciphertext, decryptionKey []byte, err error) { + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + hasher.Reset() + + hasher.Write(append(sessionKey, 1)) + accessKeyDecryptionKey := hasher.Sum(nil) + hasher.Reset() + + lk := hex.EncodeToString(lookupKey) + list, err := a.GetManifestList(ctx, NOOPDecrypt, actManifestAddress, lk) + if err != nil { + return false, nil, nil, err + } + for _, v := range list.Entries { + if v.Path == lk { + cipherTextBytes, err := hex.DecodeString(v.Hash) + if err != nil { + return false, nil, nil, err + } + return true, cipherTextBytes, accessKeyDecryptionKey, nil + } + } + return false, nil, nil, nil +} + func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) { refBytes, err := hex.DecodeString(ref) if err != nil { @@ -352,7 +388,9 @@ func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byt return m, nil } -func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { +// DoPK is a helper function to the CLI API that handles the entire business logic for +// creating a session key and access entry given the cli context, ec keys and salt +func DoPK(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { if granteePublicKey == "" { return nil, nil, errors.New("need a grantee Public Key") } @@ -383,9 +421,11 @@ func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey st return sessionKey, ae, nil } -func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) { - if len(grantees) == 0 { - return nil, nil, nil, errors.New("did not get any grantee public keys") +// DoACT is a helper function to the CLI API that handles the entire business logic for +// creating a access key, access entry and ACT manifest (including uploading it) given the cli context, ec keys, password grantees and salt +func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string, encryptPasswords []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) { + if len(grantees) == 0 && len(encryptPasswords) == 0 { + return nil, nil, nil, errors.New("did not get any grantee public keys or any encryption passwords") } publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) @@ -430,7 +470,31 @@ func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grant enc := NewRefEncryption(len(accessKey)) encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey) + if err != nil { + return nil, nil, nil, err + } + lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey) + } + for _, pass := range encryptPasswords { + sessionKey, err := sessionKeyPassword(pass, salt, DefaultKdfParams) + if err != nil { + return nil, nil, nil, err + } + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + + hasher.Reset() + hasher.Write(append(sessionKey, 1)) + + accessKeyEncryptionKey := hasher.Sum(nil) + + enc := NewRefEncryption(len(accessKey)) + encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey) + if err != nil { + return nil, nil, nil, err + } lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey) } @@ -454,7 +518,10 @@ func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grant return accessKey, ae, m, nil } -func DoPasswordNew(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { +// DoPassword is a helper function to the CLI API that handles the entire business logic for +// creating a session key and an access entry given the cli context, password and salt. +// By default - DefaultKdfParams are used as the scrypt params +func DoPassword(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { ae, err = NewAccessEntryPassword(salt, DefaultKdfParams) if err != nil { return nil, nil, err diff --git a/swarm/testutil/file.go b/swarm/testutil/file.go new file mode 100644 index 0000000000..ecb0d971ed --- /dev/null +++ b/swarm/testutil/file.go @@ -0,0 +1,44 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testutil + +import ( + "io" + "io/ioutil" + "os" + "strings" + "testing" +) + +// TempFileWithContent is a helper function that creates a temp file that contains the following string content then closes the file handle +// it returns the complete file path +func TempFileWithContent(t *testing.T, content string) string { + tempFile, err := ioutil.TempFile("", "swarm-temp-file") + if err != nil { + t.Fatal(err) + } + + _, err = io.Copy(tempFile, strings.NewReader(content)) + if err != nil { + os.RemoveAll(tempFile.Name()) + t.Fatal(err) + } + if err = tempFile.Close(); err != nil { + t.Fatal(err) + } + return tempFile.Name() +} From 8b9b149d5412dd3c775caf4fc3df39ebad90184f Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 7 Sep 2018 15:23:29 +0200 Subject: [PATCH 103/126] swarm/api/http: bzz-immutable wrong handler bug (#17602) --- swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 2aa1963969..af1269b934 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -129,7 +129,7 @@ func NewServer(api *api.API, corsString string) *Server { }) mux.Handle("/bzz-immutable:/", methodHandler{ "GET": Adapt( - http.HandlerFunc(server.HandleGet), + http.HandlerFunc(server.HandleBzzGet), defaultMiddlewares..., ), }) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 2b6a97ebdf..efefa9fae4 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -672,7 +672,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { nonhashresponses := []string{ `cannot resolve name: no DNS to resolve name: "name"`, - `cannot resolve nonhash: immutable address not a content hash: "nonhash"`, + `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, From ae992a5d73311742389fce3f855575be98fc6972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 7 Sep 2018 18:13:25 +0200 Subject: [PATCH 104/126] core/vm: Hide read only flag from Interpreter interface (#17461) Makes Interface interface a bit more stateless and abstract. Obviously this change is dictated by EVMC design. The EVMC tries to keep the responsibility for EVM features totally inside the VMs, if feasible. This makes VM "stateless" because VM does not need to pass any information between executions, all information is included in parameters of the execute function. --- core/vm/evm.go | 21 +++++++-------------- core/vm/interpreter.go | 25 +++++++++---------------- eth/tracers/tracer_test.go | 2 +- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index a24f6f3865..58618f8115 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -41,7 +41,7 @@ type ( ) // run runs the given contract and takes care of running precompiles with a fallback to the byte code interpreter. -func run(evm *EVM, contract *Contract, input []byte) ([]byte, error) { +func run(evm *EVM, contract *Contract, input []byte, readOnly bool) ([]byte, error) { if contract.CodeAddr != nil { precompiles := PrecompiledContractsHomestead if evm.ChainConfig().IsByzantium(evm.BlockNumber) { @@ -61,7 +61,7 @@ func run(evm *EVM, contract *Contract, input []byte) ([]byte, error) { }(evm.interpreter) evm.interpreter = interpreter } - return interpreter.Run(contract, input) + return interpreter.Run(contract, input, readOnly) } } return nil, ErrNoCompatibleInterpreter @@ -210,7 +210,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas evm.vmConfig.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) }() } - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, false) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally @@ -255,7 +255,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, false) if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != errExecutionReverted { @@ -288,7 +288,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by contract := NewContract(caller, to, nil, gas).AsDelegate() contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, false) if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != errExecutionReverted { @@ -310,13 +310,6 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } - // Make sure the readonly is only set if we aren't in readonly yet - // this makes also sure that the readonly flag isn't removed for - // child calls. - if !evm.interpreter.IsReadOnly() { - evm.interpreter.SetReadOnly(true) - defer func() { evm.interpreter.SetReadOnly(false) }() - } var ( to = AccountRef(addr) @@ -331,7 +324,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in Homestead this also counts for code storage gas errors. - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, true) if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != errExecutionReverted { @@ -382,7 +375,7 @@ func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.I } start := time.Now() - ret, err := run(evm, contract, nil) + ret, err := run(evm, contract, nil, false) // check whether the max code size has been exceeded maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 1e9202424a..0f1b073421 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -48,7 +48,7 @@ type Config struct { type Interpreter interface { // Run loops and evaluates the contract's code with the given input data and returns // the return byte-slice and an error if one occurred. - Run(contract *Contract, input []byte) ([]byte, error) + Run(contract *Contract, input []byte, static bool) ([]byte, error) // CanRun tells if the contract, passed as an argument, can be // run by the current interpreter. This is meant so that the // caller can do something like: @@ -61,10 +61,6 @@ type Interpreter interface { // } // ``` CanRun([]byte) bool - // IsReadOnly reports if the interpreter is in read only mode. - IsReadOnly() bool - // SetReadOnly sets (or unsets) read only mode in the interpreter. - SetReadOnly(bool) } // EVMInterpreter represents an EVM interpreter @@ -125,7 +121,7 @@ func (in *EVMInterpreter) enforceRestrictions(op OpCode, operation operation, st // It's important to note that any errors returned by the interpreter should be // considered a revert-and-consume-all-gas operation except for // errExecutionReverted which means revert-and-keep-gas-left. -func (in *EVMInterpreter) Run(contract *Contract, input []byte) (ret []byte, err error) { +func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { if in.intPool == nil { in.intPool = poolOfIntPools.get() defer func() { @@ -138,6 +134,13 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte) (ret []byte, err in.evm.depth++ defer func() { in.evm.depth-- }() + // Make sure the readOnly is only set if we aren't in readOnly yet. + // This makes also sure that the readOnly flag isn't removed for child calls. + if readOnly && !in.readOnly { + in.readOnly = true + defer func() { in.readOnly = false }() + } + // Reset the previous call's return data. It's unimportant to preserve the old buffer // as every returning call will return new data anyway. in.returnData = nil @@ -263,13 +266,3 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte) (ret []byte, err func (in *EVMInterpreter) CanRun(code []byte) bool { return true } - -// IsReadOnly reports if the interpreter is in read only mode. -func (in *EVMInterpreter) IsReadOnly() bool { - return in.readOnly -} - -// SetReadOnly sets (or unsets) read only mode in the interpreter. -func (in *EVMInterpreter) SetReadOnly(ro bool) { - in.readOnly = ro -} diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 117c376b81..58b6247245 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -49,7 +49,7 @@ func runTrace(tracer *Tracer) (json.RawMessage, error) { contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} - _, err := env.Interpreter().Run(contract, []byte{}) + _, err := env.Interpreter().Run(contract, []byte{}, false) if err != nil { return nil, err } From bcfb7f58b93e6fb5f3da0000672adee80fd6a485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Sep 2018 15:00:54 +0300 Subject: [PATCH 105/126] consensus/clique: only trust snapshot for genesis or les checkpoint --- consensus/clique/clique.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 5472909846..0ff72e55c6 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -388,7 +388,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo } } // If we're at an checkpoint block, make a snapshot if it's known - if number%c.config.Epoch == 0 { + if number == 0 || (number%c.config.Epoch == 0 && chain.GetHeaderByNumber(number-1) == nil) { checkpoint := chain.GetHeaderByNumber(number) if checkpoint != nil { hash := checkpoint.Hash() From 0e32989a08b8b84e7fe4ae397fe4302e93e34782 Mon Sep 17 00:00:00 2001 From: TColl <38299499+TColl@users.noreply.github.com> Date: Mon, 10 Sep 2018 13:22:34 +0100 Subject: [PATCH 106/126] cmd/utils: typos in {Miner, MinerLegacy}GasPriceFlag (#17588) --- cmd/utils/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f431621baf..0fecae9aa0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -343,12 +343,12 @@ var ( } MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", - Usage: "Minimal gas price for mining a transactions", + Usage: "Minimum gas price for mining a transaction", Value: eth.DefaultConfig.MinerGasPrice, } MinerLegacyGasPriceFlag = BigFlag{ Name: "gasprice", - Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)", + Usage: "Minimum gas price for mining a transaction (deprecated, use --miner.gasprice)", Value: eth.DefaultConfig.MinerGasPrice, } MinerEtherbaseFlag = cli.StringFlag{ From 6dd87483d4ded2a5ab76d19e0acaa5c6b06312a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Tue, 11 Sep 2018 11:39:02 +0200 Subject: [PATCH 107/126] Encryption async api (#17603) * swarm/storage/encryption: async segmentwise encryption/decryption * swarm/storage: adapt hasherstore to encryption API change * swarm/api: adapt RefEncryption for AC to new Encryption API * swarm/storage/encryption: address review comments --- swarm/api/encrypt.go | 22 +-- swarm/storage/encryption/encryption.go | 140 ++++++++++++-------- swarm/storage/encryption/encryption_test.go | 47 ++++--- swarm/storage/hasherstore.go | 100 +++++++------- 4 files changed, 177 insertions(+), 132 deletions(-) diff --git a/swarm/api/encrypt.go b/swarm/api/encrypt.go index 9a2e369149..ffe6c16d2d 100644 --- a/swarm/api/encrypt.go +++ b/swarm/api/encrypt.go @@ -25,27 +25,27 @@ import ( ) type RefEncryption struct { - spanEncryption encryption.Encryption - dataEncryption encryption.Encryption - span []byte + refSize int + span []byte } func NewRefEncryption(refSize int) *RefEncryption { span := make([]byte, 8) binary.LittleEndian.PutUint64(span, uint64(refSize)) return &RefEncryption{ - spanEncryption: encryption.New(0, uint32(refSize/32), sha3.NewKeccak256), - dataEncryption: encryption.New(refSize, 0, sha3.NewKeccak256), - span: span, + refSize: refSize, + span: span, } } func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) { - encryptedSpan, err := re.spanEncryption.Encrypt(re.span, key) + spanEncryption := encryption.New(key, 0, uint32(re.refSize/32), sha3.NewKeccak256) + encryptedSpan, err := spanEncryption.Encrypt(re.span) if err != nil { return nil, err } - encryptedData, err := re.dataEncryption.Encrypt(ref, key) + dataEncryption := encryption.New(key, re.refSize, 0, sha3.NewKeccak256) + encryptedData, err := dataEncryption.Encrypt(ref) if err != nil { return nil, err } @@ -57,7 +57,8 @@ func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) { } func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) { - decryptedSpan, err := re.spanEncryption.Decrypt(ref[:8], key) + spanEncryption := encryption.New(key, 0, uint32(re.refSize/32), sha3.NewKeccak256) + decryptedSpan, err := spanEncryption.Decrypt(ref[:8]) if err != nil { return nil, err } @@ -67,7 +68,8 @@ func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) { return nil, errors.New("invalid span in encrypted reference") } - decryptedRef, err := re.dataEncryption.Decrypt(ref[8:], key) + dataEncryption := encryption.New(key, re.refSize, 0, sha3.NewKeccak256) + decryptedRef, err := dataEncryption.Decrypt(ref[8:]) if err != nil { return nil, err } diff --git a/swarm/storage/encryption/encryption.go b/swarm/storage/encryption/encryption.go index e50f2163db..6fbdab062b 100644 --- a/swarm/storage/encryption/encryption.go +++ b/swarm/storage/encryption/encryption.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "fmt" "hash" + "sync" ) const KeyLength = 32 @@ -28,84 +29,119 @@ const KeyLength = 32 type Key []byte type Encryption interface { - Encrypt(data []byte, key Key) ([]byte, error) - Decrypt(data []byte, key Key) ([]byte, error) + Encrypt(data []byte) ([]byte, error) + Decrypt(data []byte) ([]byte, error) } type encryption struct { - padding int - initCtr uint32 - hashFunc func() hash.Hash + key Key // the encryption key (hashSize bytes long) + keyLen int // length of the key = length of blockcipher block + padding int // encryption will pad the data upto this if > 0 + initCtr uint32 // initial counter used for counter mode blockcipher + hashFunc func() hash.Hash // hasher constructor function } -func New(padding int, initCtr uint32, hashFunc func() hash.Hash) *encryption { +// New constructs a new encryptor/decryptor +func New(key Key, padding int, initCtr uint32, hashFunc func() hash.Hash) *encryption { return &encryption{ + key: key, + keyLen: len(key), padding: padding, initCtr: initCtr, hashFunc: hashFunc, } } -func (e *encryption) Encrypt(data []byte, key Key) ([]byte, error) { +// Encrypt encrypts the data and does padding if specified +func (e *encryption) Encrypt(data []byte) ([]byte, error) { length := len(data) + outLength := length isFixedPadding := e.padding > 0 - if isFixedPadding && length > e.padding { - return nil, fmt.Errorf("Data length longer than padding, data length %v padding %v", length, e.padding) + if isFixedPadding { + if length > e.padding { + return nil, fmt.Errorf("Data length longer than padding, data length %v padding %v", length, e.padding) + } + outLength = e.padding } - - paddedData := data - if isFixedPadding && length < e.padding { - paddedData = make([]byte, e.padding) - copy(paddedData[:length], data) - rand.Read(paddedData[length:]) - } - return e.transform(paddedData, key), nil + out := make([]byte, outLength) + e.transform(data, out) + return out, nil } -func (e *encryption) Decrypt(data []byte, key Key) ([]byte, error) { +// Decrypt decrypts the data, if padding was used caller must know original length and truncate +func (e *encryption) Decrypt(data []byte) ([]byte, error) { length := len(data) if e.padding > 0 && length != e.padding { return nil, fmt.Errorf("Data length different than padding, data length %v padding %v", length, e.padding) } - - return e.transform(data, key), nil + out := make([]byte, length) + e.transform(data, out) + return out, nil } -func (e *encryption) transform(data []byte, key Key) []byte { - dataLength := len(data) - transformedData := make([]byte, dataLength) - hasher := e.hashFunc() - ctr := e.initCtr - hashSize := hasher.Size() - for i := 0; i < dataLength; i += hashSize { - hasher.Write(key) - - ctrBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(ctrBytes, ctr) - - hasher.Write(ctrBytes) - - ctrHash := hasher.Sum(nil) - hasher.Reset() - hasher.Write(ctrHash) - - segmentKey := hasher.Sum(nil) - - hasher.Reset() - - segmentSize := min(hashSize, dataLength-i) - for j := 0; j < segmentSize; j++ { - transformedData[i+j] = data[i+j] ^ segmentKey[j] - } - ctr++ +// +func (e *encryption) transform(in, out []byte) { + inLength := len(in) + wg := sync.WaitGroup{} + wg.Add((inLength-1)/e.keyLen + 1) + for i := 0; i < inLength; i += e.keyLen { + l := min(e.keyLen, inLength-i) + // call transformations per segment (asyncronously) + go func(i int, x, y []byte) { + defer wg.Done() + e.Transcrypt(i, x, y) + }(i/e.keyLen, in[i:i+l], out[i:i+l]) } - return transformedData + // pad the rest if out is longer + pad(out[inLength:]) + wg.Wait() } -func GenerateRandomKey() (Key, error) { - key := make([]byte, KeyLength) - _, err := rand.Read(key) - return key, err +// used for segmentwise transformation +// if in is shorter than out, padding is used +func (e *encryption) Transcrypt(i int, in []byte, out []byte) { + // first hash key with counter (initial counter + i) + hasher := e.hashFunc() + hasher.Write(e.key) + + ctrBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(ctrBytes, uint32(i)+e.initCtr) + hasher.Write(ctrBytes) + + ctrHash := hasher.Sum(nil) + hasher.Reset() + + // second round of hashing for selective disclosure + hasher.Write(ctrHash) + segmentKey := hasher.Sum(nil) + hasher.Reset() + + // XOR bytes uptil length of in (out must be at least as long) + inLength := len(in) + for j := 0; j < inLength; j++ { + out[j] = in[j] ^ segmentKey[j] + } + // insert padding if out is longer + pad(out[inLength:]) +} + +func pad(b []byte) { + l := len(b) + for total := 0; total < l; { + read, _ := rand.Read(b[total:]) + total += read + } +} + +// GenerateRandomKey generates a random key of length l +func GenerateRandomKey(l int) Key { + key := make([]byte, l) + var total int + for total < l { + read, _ := rand.Read(key[total:]) + total += read + } + return key } func min(x, y int) int { diff --git a/swarm/storage/encryption/encryption_test.go b/swarm/storage/encryption/encryption_test.go index 5ea546d6bf..c3abefdcec 100644 --- a/swarm/storage/encryption/encryption_test.go +++ b/swarm/storage/encryption/encryption_test.go @@ -22,34 +22,42 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/sha3" ) -var expectedTransformedHex = "470c6c67ba1820d5cb4c23ccef22aa2417a323dc97e5f5dced930d74f2932fd178df80ddf129f4f8a4ccec0225c1c2e6765cbbd92bbad8413c50d93d53f7b2fec975d6f29468eccdaf6458f7a3306a7bc207211ea7f9ee5e6951ce6874aef09eb7ff9bed0aa00920b4dcc105e5f1f8dfbdb0564751311d6ceaca1e4e6d988097582638106404c03cfaef8db0e46674d9e8192c1b62d1cd952389cab3a8dee9329fdb059bafd9bae3df3a6b5a9a10961a0333016c99ee3d65cf18ea8ea7b4c386d59dcbf317460d517d5b55504b5992ffb9c3d7c49ebe1fe3ab7b00ad84b1d76d95f8165141260e6980b2ffe8a0000c36dd77ace4c7a781887f831f740d92d8b4848f1b9e237877b988b5a3e23d7b07b2c8eda0ea9fa748ab10bb6bfd7447b66f44e528d24eff31defdecddaf2bb5e6b8e2aa3d2f1a83de44e0ee3789b75dadba1375b7f3a8f7f1eb2d9b78ad1844d425fb76ea6ba45eabe88f1e062d7552d8c7d44c9c66c2753c41892cb675fd1d564b5f4746e76aa4b24ad121a907b8786715483067f46f1cd4a3dad5125f58c3d348db776e99e8a562ebe603e0e98509c72118ee81259b43f1f770dc9220dddbe4d12d2ffabdad76663d1cf30304eacbb43e9bbe0e4f95aae8609bf8a0786e56e0d20d4875d17cec644ca16824424c6c0700e5082ad5288d1a9fec637b75d24c270086a3606b97cc3240314ac123c04ff4d67ef547504f0eeea5e6252f9b5b75e47d32e583c4dee95ac84b6f03fef412e24c09697d6f7bc8408c2118c524f8e277b82658a5869d6d69d7fde469be27ea33fa22b47f3276f60580f2a05b0fbceb67949d2c8e6d8b097ceef6781d962866b23bc71b54597dd05caafbd533ff4e0013f0fc7e202782ba1e6ccd11242a5bc823af9ad0d5bcb8b78d0ce150fc6f8b97c66f9337adaf3878cbb70733044fd14f318d8e389940d07e0593e35f46dc84e7a9a75dadda153bbe5755af0683cfd262ce3797b1bc648a4e5cef5d9aa47d08cf78c6c9dd872096deb856680d5d932abfd5f2024f908ab84d19b762fdcd31209b1d9adf9cbab904bc58f5b00d5a3e0eb2ade52c7c9a678aadedb1b9257263e6145d34ebcc250d0867e2fdc2ed23b4a8e8aa580424308b543b76595f7e09127e8c301d3f6c228e656df2b5e63a89d634db2f87249025ad3b53d2703d6b93c4ab6b1dee4f9b4d2a0b383ed87458e0e1b2a3c04f9a08b885d856ea7ca9abda9a9c8bd28ecce78675b16829493dbeb66963a3665a8386bc4b97d13cfacacfa80b12e36b598d4f09ab7847481a95b35a937bd4f0107598844fc518c4134e8ca55cec024773dcf0a17c71f602406363dcc03912f23cffad9c611e34605fb03fea3d84aac33886efed6c53d455c786231436c01524cd41e2b6c35b339feaa94d7df0b3572db5595c05a1d4a96e1a17e814cea965be81786784af5697b5c7909fd7701d0f5d991f469e20fbc39536239c7a53278c0b1189253d3d978d54b51d33e0c8ac03ca1f08cfd440130944ae9d353b0b7f263a3a27d15290f35c0ad7cf62e8429ebfa10f5471987c53c5d574c9aebb935bdb8339fee9e1030b3f2845c36272642233afba71f81f10322057b51a33b434a4bc45530581c4a636895fc773e20cf6b8a6cef060920310f4cb79f735fc14e758f646b0cf3bf0ce1f1479788f5b73507203d54022c7afd805d46f45e7ede96b2660869939440d7d575fbf372a7f63421728343dfd88c5fa29ea3f15b68fe360f54afa0681093f2c82888d6babf9558159383c1a76264deec2d7fccb20dbd3ab4b3f137fb3dcf253fb6953df2f663d08e8b39909c9dbff98d49377ce0ce03c580b5e1bca6fcf58aceda42b00b1f478cff2995d47c82f2d5a3dc0e9718a0a46d8330353f2039e68d9565d29e77efea91e058ae03140a4b58b297ae664c9f0e7af78f3633f4aeabd95fd367ac5c67eb63fa2bd7035f4adfe3ab8502952d1675b47118b2006b4a5a1b2a3d03aa670862adc5f1ad1b39f0e4a08e2412128e0ef5fa84366b4b50992cc139e7837f8d65f8eae3c8dc2730f2ef1e1585ce95b0fe354c6c853526558a1732171b6017254ba0f0c22273b55caa2ee680791fb34e22bf897a0998156083c83d03310956b1f947ef474c90b2d7e20beae27f5d33b79ad6d9b1b0188e3cf850108f3a02f9386a314b97fb6c946e572e40024da3e4784c523cf2a70e45f6c0f8f35e9b279aecee183ccb30477ee4f5d57c246ab9918495159307935340a92bd46d6519bcc51af4a785a7eb7fa6eae9def89e2411efd2cb2c33f4d605b77af1ae298967fd846453e8b0a55c57706e79d7badd93268d93be27790bbb051496552b094f37bf96843cdf7604dc7f696976bebbe3561c7bd4b3f2843bd07e34a3c3acf0d6c755014a3e9d922dabdbc892511b6af3214958b3531baceab9082c2b4e19ef802b99db2cfa4076ba681efae8a9546582e4cbf07000fb3903727261b93d1bd08809ddc41b2a61b0bfa6b9210b8731394fd9084bc1406bcdd0992410e90f749a087edafb76f6617209855269bb5c89711d2830ea99f2a0e548991ef8dd3e62ced239dff4e9d8c1e834fb57078dc2a4322acc1ea1dbe64064db4cc2a7cdc32884cb7c31aa3c95634356b4d32f1c92fa039e5ed18c1af6605e2f66e5383fa1fcc610a3cc1eef5fe6296ac378f2440d0bd3c77d458af0bfa6b64cedd0301b116efafbbb8f9e88d4bedf56d11cfe0967fc06932e1f232c7daf2c73b58c52daa82dc22b2290147e358348f991f1473c4e63ac943cabc429f5689da8aee6801fd941778966da87fe137b033a0231a90a2ac759efbcb7c52de8e8f27bcc4e5bcb560524d17b0345727f8092e22b6a0e03ceee355085d4fe81568b5b8b84b69870dd333c9b3bdca45db7e43b876a217819928fec31f3ca4c44c4f03a91d578f9ed883bf1247a10eefe42279484af6f70719e193b07a14fc7d93fd6cee9e883a81cc51b53044fa2f1b2525c10e23fb988b31559e1fae4c082d905486d984d2f1e87c40976a571f92e4fdf09e23cad561d9cd3d1ca94e778d08b82852dbe9bab2c7f147c6078504c30fc3fe749fca95ed23791dcf6b935ab0ed08645c481cd2b7969258405f183593617c0d24ebe79bc6c87df65ddc25f933b12a641cc95cf321c1b2edf5db5813a98d99fd1760b5fac19c2b47c2a750e96da49c97276cf31cbd73e1e93ef6990fdf1b08a3e7963dffcf65aa6496727f6be5d9225cfbbadfd6a3a06a454acbb80e761699be97b9caeb71dc3c20a2c253f349ee190b9d8c1ff95751a281ed88b098e4a78fae8249232d4280daf46580ca6c14411f8c4d0e8fa44f8808a5a2dda36d35f79aa77c1c4c615216ec309aa5f3caff3a449b9740baad11e525a5651416a12dd4688897b5ee5e1e8361a87269e9be0789ea7564f64752788f3ac57bda4aa45582d8dabc4d44759639984514d9b151004bf0d9d3140becde41603ffd9d412ec08a1b2f7262862a761d7ce5b91a239d68cdfe7c615b62217422c5a224e652cffdbbe7b41064c9375bbf2c23b63130afedd93987c1f1e4a1623a77d3ef4960fd232afc7d12159c8f1245984e4bd390685668f10da77bf5f6130a5d405a42fb02d0048d63ca48cd4901f3704975b0493714c78de33c1a969993529bac13e310ee098a7b1e8e93b4fcb269773a1521defef98f403f79a8e24e1594b8dc6055223b79b4d7009498c02790a37dd3a9ac7168b11568877ee852395165f87d881355327e57ba48be948b1325824244a552ca80c2ecb79a04a199f0b6a58b455327d998a67ebae414394928161a311de9d3d7f1fc99c0bc35b0a814e7821d2d6958129775c56fdd9b28b6e43a3132a95eb64805f9941b0c5ca2078c37bc7a4d43ddcbc6e8c66de2ef51c714a81913bebeb8f9c6aa14da7e71854870fba9f093aa77abeb23e8d37f3a50135a3f894d1cdf647d3dceb3f16ad4b4f01dffe99d74cd1f18d37735ac69287df0a8b49b2fc3773bf426c881b64a1241c1143e39b6dca59e28c116947aac39585dd81f6fe3832f97d873884738982976f9f29aa6463662cc81c2abbd5eafbb3d9800ba811410a521065d61ad01cfde2c2d98f5fa0ec63e8b01ec6a2d738a1f6cf8c99725079aa51ed3dc7630deeea0b27ee8edbfa19bdb8504cf80a1ca74ea6777fa8786fd76d4ab019c00da35278bf49be5a42aff264875b69bab9e7f3f9299f2ea6a883fc5fd77e34debf414ce8af37a3e9a7de05f33bc547899d25a13389d9048be8b2b7c9b836b401eac9c53ba3d44a14e683a6b378248fdedcbfbc408e5ec83a6c3f769533ae90e339222d80d433ae820fca43d80bca7fbd9d6e5eb760271c74a95a35a853285f7dcfe10182af922508501d269cc53ed38756e8d202b25782ec208a39981ef6c9e9342eb6cee2142471574ab39ebdbbf140f64e305d6f6aa73d49d5bb765347a27ebd0ba1bdcaa8a383526b41f3361faddafec726e185cc223f8465472d7a53ad2d74717960ba5684bc692f773e680d0247f9de1650c046ea56fa18c1c3fb6636327f863f37dca86867c57cc643ff6b7973f97c91716502c33f0dc66176357315f09064c70df7db026413567ac9ec7605acd80606d962f7463359998b92f8a33ccffc80f6e162c1b5a65404e0cc4e37688ded910e6ae5f774e3238a1248b05c53486d149734dd1e38073603df0aa4a4f01fd0c1a0e82b6a513f7d22d1b09ff107923462fbfffbf5fb1f2c043e305917eee1f7f18f62dc604a1d39e5bf675dca67da52f95d312b31be7720efd391bf4794f0555b299325b5dfea9eb00e695861634ad28717cffff266e30c864ba844b0c3fd88bd0cee4d929530c2bb3663c5170a15a66c412ff2fed2d962dd0ff145f19f8085931cbd6bde4c11c9c2debfbdb6748d1a6dabf1404762343d468944a0495026091bc44c69dad971890b7221e1eac2e985097d344a4e2375938aa93806ad1715be8d05f4068ec67ef411e704b01851b7427c4137e6dadedfbb25378ecf1c6b749214f2655cf43fdbb72cb842aaab3074cd792f30f96e874d92f2ef62667d81ccd663dfd969c18dff790b6b7b261f0a03baa85bfd7db72baffe2ddc4cc3985183301a77daa826b1f76f6b6f2ac075c6a2b86609e4d26eb08f3ed6f341d7946966e654ed2c30a629ff81a57014a84105a9ad36a525033f16e3e60d639bf8f89ae6cfdbcdb41e93859354957dcfe9a847757c3cd946d8994cda126f146b77119bbc87c49ce79ad844715cd9bb0c7a4f800fce14c81d175aed59fce0377e62c6e597ab5acf1b3b7100403f371c19dd0f131c4c572e57e3e13743f9bac24a6a177d71f03c5d185bb7e163ec5866dda629340a964bc442d423ad7bc5187f3da68d1498dfe9f1815b31bae11df3585ece230cc3521f7a0b4b3360ddf898984b528afff75f229915f1f2d5c4491c65e2f38d26c0de7dc483860da7bf52859fdbc21946badec0bbf00ba8143b8b7289cfb7096d3405e3183e56f1cbb8c48f25d058530894d0302cdce7c43ea31768b5b610820c97f6e9a8e31a8e9c4624117d213a03ef7d655513cffd8fb5606fc8790692c99828a47382e3e37b9c1317028f5c9d196ff3c2f09435df7614fb37ea20b2371c52b6c4667799922010edeb28001de2a137889db4d3dbf0a406ba90f3631be485ec5e4110b01502f557f15026716030fb6384499adee3cfc44015638e05b206ce3cbc3cbf21b7dd1dcb3b932629e7cd4f4f6b148f37f976803644e5ce792583daf39608a3cd02ff2f47e9bc6" +var expectedTransformedHex = "352187af3a843decc63ceca6cb01ea39dbcf77caf0a8f705f5c30d557044ceec9392b94a79376f1e5c10cd0c0f2a98e5353bf22b3ea4fdac6677ee553dec192e3db64e179d0474e96088fb4abd2babd67de123fb398bdf84d818f7bda2c1ab60b3ea0e0569ae54aa969658eb4844e6960d2ff44d7c087ee3aaffa1c0ee5df7e50b615f7ad90190f022934ad5300c7d1809bfe71a11cc04cece5274eb97a5f20350630522c1dbb7cebaf4f97f84e03f5cfd88f2b48880b25d12f4d5e75c150f704ef6b46c72e07db2b705ac3644569dccd22fd8f964f6ef787fda63c46759af334e6f665f70eac775a7017acea49f3c7696151cb1b9434fa4ac27fb803921ffb5ec58dafa168098d7d5b97e384be3384cf5bc235c3d887fef89fe76c0065f9b8d6ad837b442340d9e797b46ef5709ea3358bc415df11e4830de986ef0f1c418ffdcc80e9a3cda9bea0ab5676c0d4240465c43ba527e3b4ea50b4f6255b510e5d25774a75449b0bd71e56c537ade4fcf0f4d63c99ae1dbb5a844971e2c19941b8facfcfc8ee3056e7cb3c7114c5357e845b52f7103cb6e00d2308c37b12baa5b769e1cc7b00fc06f2d16e70cc27a82cb9c1a4e40cb0d43907f73df2c9db44f1b51a6b0bc6d09f77ac3be14041fae3f9df2da42df43ae110904f9ecee278030185254d7c6e918a5512024d047f77a992088cb3190a6587aa54d0c7231c1cd2e455e0d4c07f74bece68e29cd8ba0190c0bcfb26d24634af5d91a81ef5d4dd3d614836ce942ddbf7bb1399317f4c03faa675f325f18324bf9433844bfe5c4cc04130c8d5c329562b7cd66e72f7355de8f5375a72202971613c32bd7f3fcdcd51080758cd1d0a46dbe8f0374381dbc359f5864250c63dde8131cbd7c98ae2b0147d6ea4bf65d1443d511b18e6d608bbb46ac036353b4c51df306a10a6f6939c38629a5c18aaf89cac04bd3ad5156e6b92011c88341cb08551bab0a89e6a46538f5af33b86121dba17e3a434c273f385cd2e8cb90bdd32747d8425d929ccbd9b0815c73325988855549a8489dfd047daf777aaa3099e54cf997175a5d9e1edfe363e3b68c70e02f6bf4fcde6a0f3f7d0e7e98bde1a72ae8b6cd27b32990680cc4a04fc467f41c5adcaddabfc71928a3f6872c360c1d765260690dd28b269864c8e380d9c92ef6b89b0094c8f9bb22608b4156381b19b920e9583c9616ce5693b4d2a6c689f02e6a91584a8e501e107403d2689dd0045269dd9946c0e969fb656a3b39d84a798831f5f9290f163eb2f97d3ae25071324e95e2256d9c1e56eb83c26397855323edc202d56ad05894333b7f0ed3c1e4734782eb8bd5477242fd80d7a89b12866f85cfae476322f032465d6b1253993033fccd4723530630ab97a1566460af9c90c9da843c229406e65f3fa578bd6bf04dee9b6153807ddadb8ceefc5c601a8ab26023c67b1ab1e8e0f29ce94c78c308005a781853e7a2e0e51738939a657c987b5e611f32f47b5ff461c52e63e0ea390515a8e1f5393dae54ea526934b5f310b76e3fa050e40718cb4c8a20e58946d6ee1879f08c52764422fe542b3240e75eccb7aa75b1f8a651e37a3bc56b0932cdae0e985948468db1f98eb4b77b82081ea25d8a762db00f7898864984bd80e2f3f35f236bf57291dec28f550769943bcfb6f884b7687589b673642ef7fe5d7d5a87d3eca5017f83ccb9a3310520474479464cb3f433440e7e2f1e28c0aef700a45848573409e7ab66e0cfd4fe5d2147ace81bc65fd8891f6245cd69246bbf5c27830e5ab882dd1d02aba34ff6ca9af88df00fd602892f02fedbdc65dedec203faf3f8ff4a97314e0ddb58b9ab756a61a562597f4088b445fcc3b28a708ca7b1485dcd791b779fbf2b3ef1ec5c6205f595fbe45a02105034147e5a146089c200a49dae33ae051a08ea5f974a21540aaeffa7f9d9e3d35478016fb27b871036eb27217a5b834b461f535752fb5f1c8dded3ae14ce3a2ef6639e2fe41939e3509e46e347a95d50b2080f1ba42c804b290ddc912c952d1cec3f2661369f738feacc0dbf1ea27429c644e45f9e26f30c341acd34c7519b2a1663e334621691e810767e9918c2c547b2e23cce915f97d26aac8d0d2fcd3edb7986ad4e2b8a852edebad534cb6c0e9f0797d3563e5409d7e068e48356c67ce519246cd9c560e881453df97cbba562018811e6cf8c327f399d1d1253ab47a19f4a0ccc7c6d86a9603e0551da310ea595d71305c4aad96819120a92cdbaf1f77ec8df9cc7c838c0d4de1e8692dd81da38268d1d71324bcffdafbe5122e4b81828e021e936d83ae8021eac592aa52cd296b5ce392c7173d622f8e07d18f59bb1b08ba15211af6703463b09b593af3c37735296816d9f2e7a369354a5374ea3955e14ca8ac56d5bfe4aef7a21bd825d6ae85530bee5d2aaaa4914981b3dfdb2e92ec2a27c83d74b59e84ff5c056f7d8945745f2efc3dcf28f288c6cd8383700fb2312f7001f24dd40015e436ae23e052fe9070ea9535b9c989898a9bda3d5382cf10e432fae6ccf0c825b3e6436edd3a9f8846e5606f8563931b5f29ba407c5236e5730225dda211a8504ec1817bc935e1fd9a532b648c502df302ed2063aed008fd5676131ac9e95998e9447b02bd29d77e38fcfd2959f2de929b31970335eb2a74348cc6918bc35b9bf749eab0fe304c946cd9e1ca284e6853c42646e60b6b39e0d3fb3c260abfc5c1b4ca3c3770f344118ca7c7f5c1ad1f123f8f369cd60afc3cdb3e9e81968c5c9fa7c8b014ffe0508dd4f0a2a976d5d1ca8fc9ad7a237d92cfe7b41413d934d6e142824b252699397e48e4bac4e91ebc10602720684bd0863773c548f9a2f9724245e47b129ecf65afd7252aac48c8a8d6fd3d888af592a01fb02dc71ed7538a700d3d16243e4621e0fcf9f8ed2b4e11c9fa9a95338bb1dac74a7d9bc4eb8cbf900b634a2a56469c00f5994e4f0934bdb947640e6d67e47d0b621aacd632bfd3c800bd7d93bd329f494a90e06ed51535831bd6e07ac1b4b11434ef3918fa9511813a002913f33f836454798b8d1787fea9a4c4743ba091ed192ed92f4d33e43a226bf9503e1a83a16dd340b3cbbf38af6db0d99201da8de529b4225f3d2fa2aad6621afc6c79ef3537720591edfc681ae6d00ede53ed724fc71b23b90d2e9b7158aaee98d626a4fe029107df2cb5f90147e07ebe423b1519d848af18af365c71bfd0665db46be493bbe99b79a188de0cf3594aef2299f0324075bdce9eb0b87bc29d62401ba4fd6ae48b1ba33261b5b845279becf38ee03e3dc5c45303321c5fac96fd02a3ad8c9e3b02127b320501333c9e6360440d1ad5e64a6239501502dde1a49c9abe33b66098458eee3d611bb06ffcd234a1b9aef4af5021cd61f0de6789f822ee116b5078aae8c129e8391d8987500d322b58edd1595dc570b57341f2df221b94a96ab7fbcf32a8ca9684196455694024623d7ed49f7d66e8dd453c0bae50e0d8b34377b22d0ece059e2c385dfc70b9089fcd27577c51f4d870b5738ee2b68c361a67809c105c7848b68860a829f29930857a9f9d40b14fd2384ac43bafdf43c0661103794c4bd07d1cfdd4681b6aeaefad53d4c1473359bcc5a83b09189352e5bb9a7498dd0effb89c35aad26954551f8b0621374b449bf515630bd3974dca982279733470fdd059aa9c3df403d8f22b38c4709c82d8f12b888e22990350490e16179caf406293cc9e65f116bafcbe96af132f679877061107a2f690a82a8cb46eea57a90abd23798c5937c6fe6b17be3f9bfa01ce117d2c268181b9095bf49f395fea07ca03838de0588c5e2db633e836d64488c1421e653ea52d810d096048c092d0da6e02fa6613890219f51a76148c8588c2487b171a28f17b7a299204874af0131725d793481333be5f08e86ca837a226850b0c1060891603bfecf9e55cddd22c0dbb28d495342d9cc3de8409f72e52a0115141cffe755c74f061c1a770428ccb0ae59536ee6fc074fbfc6cacb51a549d327527e20f8407477e60355863f1153f9ce95641198663c968874e7fdb29407bd771d94fdda8180cbb0358f5874738db705924b8cbe0cd5e1484aeb64542fe8f38667b7c34baf818c63b1e18440e9fba575254d063fd49f24ef26432f4eb323f3836972dca87473e3e9bb26dc3be236c3aae6bc8a6da567442309da0e8450e242fc9db836e2964f2c76a3b80a2c677979882dda7d7ebf62c93664018bcf4ec431fe6b403d49b3b36618b9c07c2d0d4569cb8d52223903debc72ec113955b206c34f1ae5300990ccfc0180f47d91afdb542b6312d12aeff7e19c645dc0b9fe6e3288e9539f6d5870f99882df187bfa6d24d179dfd1dac22212c8b5339f7171a3efc15b760fed8f68538bc5cbd845c2d1ab41f3a6c692820653eaef7930c02fbe6061d93805d73decdbb945572a7c44ed0241982a6e4d2d730898f82b3d9877cb7bca41cc6dcee67aa0c3d6db76f0b0a708ace0031113e48429de5d886c10e9200f68f32263a2fbf44a5992c2459fda7b8796ba796e3a0804fc25992ed2c9a5fe0580a6b809200ecde6caa0364b58be11564dcb9a616766dd7906db5636ee708b0204f38d309466d8d4a162965dd727e29f5a6c133e9b4ed5bafe803e479f9b2a7640c942c4a40b14ac7dc9828546052761a070f6404008f1ec3605836339c3da95a00b4fd81b2cabf88b51d2087d5b83e8c5b69bf96d8c72cbd278dad3bbb42b404b436f84ad688a22948adf60a81090f1e904291503c16e9f54b05fc76c881a5f95f0e732949e95d3f1bae2d3652a14fe0dda2d68879604657171856ef72637def2a96ac47d7b3fe86eb3198f5e0e626f06be86232305f2ae79ffcd2725e48208f9d8d63523f81915acc957563ab627cd6bc68c2a37d59fb0ed77a90aa9d085d6914a8ebada22a2c2d471b5163aeddd799d90fbb10ed6851ace2c4af504b7d572686700a59d6db46d5e42bb83f8e0c0ffe1dfa6582cc0b34c921ff6e85e83188d24906d5c08bb90069639e713051b3102b53e6f703e8210017878add5df68e6f2b108de279c5490e9eef5590185c4a1c744d4e00d244e1245a8805bd30407b1bc488db44870ccfd75a8af104df78efa2fb7ba31f048a263efdb3b63271fff4922bece9a71187108f65744a24f4947dc556b7440cb4fa45d296bb7f724588d1f245125b21ea063500029bd49650237f53899daf1312809552c81c5827341263cc807a29fe84746170cdfa1ff3838399a5645319bcaff674bb70efccdd88b3d3bb2f2d98111413585dc5d5bd5168f43b3f55e58972a5b2b9b3733febf02f931bd436648cb617c3794841aab961fe41277ab07812e1d3bc4ff6f4350a3e615bfba08c3b9480ef57904d3a16f7e916345202e3f93d11f7a7305170cb8c4eb9ac88ace8bbd1f377bdd5855d3162d6723d4435e84ce529b8f276a8927915ac759a0d04e5ca4a9d3da6291f0333b475df527e99fe38f7a4082662e8125936640c26dd1d17cf284ce6e2b17777a05aa0574f7793a6a062cc6f7263f7ab126b4528a17becfdec49ac0f7d8705aa1704af97fb861faa8a466161b2b5c08a5bacc79fe8500b913d65c8d3c52d1fd52d2ab2c9f52196e712455619c1cd3e0f391b274487944240e2ed8858dd0823c801094310024ae3fe4dd1cf5a2b6487b42cc5937bbafb193ee331d87e378258963d49b9da90899bbb4b88e79f78e866b0213f4719f67da7bcc2fce073c01e87c62ea3cdbcd589cfc41281f2f4c757c742d6d1e" var hashFunc = sha3.NewKeccak256 +var testKey Key + +func init() { + var err error + testKey, err = hexutil.Decode("0x8abf1502f557f15026716030fb6384792583daf39608a3cd02ff2f47e9bc6e49") + if err != nil { + panic(err.Error()) + } +} func TestEncryptDataLongerThanPadding(t *testing.T) { - enc := New(4095, uint32(0), hashFunc) + enc := New(testKey, 4095, uint32(0), hashFunc) data := make([]byte, 4096) - key := make([]byte, 32) expectedError := "Data length longer than padding, data length 4096 padding 4095" - _, err := enc.Encrypt(data, key) + _, err := enc.Encrypt(data) if err == nil || err.Error() != expectedError { t.Fatalf("Expected error \"%v\" got \"%v\"", expectedError, err) } } func TestEncryptDataZeroPadding(t *testing.T) { - enc := New(0, uint32(0), hashFunc) + enc := New(testKey, 0, uint32(0), hashFunc) data := make([]byte, 2048) - key := make([]byte, 32) - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } @@ -59,12 +67,11 @@ func TestEncryptDataZeroPadding(t *testing.T) { } func TestEncryptDataLengthEqualsPadding(t *testing.T) { - enc := New(4096, uint32(0), hashFunc) + enc := New(testKey, 4096, uint32(0), hashFunc) data := make([]byte, 4096) - key := make([]byte, 32) - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } @@ -77,12 +84,11 @@ func TestEncryptDataLengthEqualsPadding(t *testing.T) { } func TestEncryptDataLengthSmallerThanPadding(t *testing.T) { - enc := New(4096, uint32(0), hashFunc) + enc := New(testKey, 4096, uint32(0), hashFunc) data := make([]byte, 4080) - key := make([]byte, 32) - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } @@ -96,14 +102,13 @@ func TestEncryptDataCounterNonZero(t *testing.T) { } func TestDecryptDataLengthNotEqualsPadding(t *testing.T) { - enc := New(4096, uint32(0), hashFunc) + enc := New(testKey, 4096, uint32(0), hashFunc) data := make([]byte, 4097) - key := make([]byte, 32) expectedError := "Data length different than padding, data length 4097 padding 4096" - _, err := enc.Decrypt(data, key) + _, err := enc.Decrypt(data) if err == nil || err.Error() != expectedError { t.Fatalf("Expected error \"%v\" got \"%v\"", expectedError, err) } @@ -117,20 +122,18 @@ func TestEncryptDecryptIsIdentity(t *testing.T) { } func testEncryptDecryptIsIdentity(t *testing.T, padding int, initCtr uint32, dataLength int, keyLength int) { - enc := New(padding, initCtr, hashFunc) + key := GenerateRandomKey(keyLength) + enc := New(key, padding, initCtr, hashFunc) data := make([]byte, dataLength) rand.Read(data) - key := make([]byte, keyLength) - rand.Read(key) - - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } - decrypted, err := enc.Decrypt(encrypted, key) + decrypted, err := enc.Decrypt(encrypted) if err != nil { t.Fatalf("Expected no error got %v", err) } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index bc23077c18..766207eae0 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -26,49 +26,34 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage/encryption" ) -type chunkEncryption struct { - spanEncryption encryption.Encryption - dataEncryption encryption.Encryption -} - type hasherStore struct { - store ChunkStore - hashFunc SwarmHasher - chunkEncryption *chunkEncryption - hashSize int // content hash size - refSize int64 // reference size (content hash + possibly encryption key) - wg *sync.WaitGroup - closed chan struct{} -} - -func newChunkEncryption(chunkSize, refSize int64) *chunkEncryption { - return &chunkEncryption{ - spanEncryption: encryption.New(0, uint32(chunkSize/refSize), sha3.NewKeccak256), - dataEncryption: encryption.New(int(chunkSize), 0, sha3.NewKeccak256), - } + store ChunkStore + toEncrypt bool + hashFunc SwarmHasher + hashSize int // content hash size + refSize int64 // reference size (content hash + possibly encryption key) + wg *sync.WaitGroup + closed chan struct{} } // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. // With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore // and the hasherStore will take core of encryption/decryption of data if necessary func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { - var chunkEncryption *chunkEncryption - hashSize := hashFunc().Size() refSize := int64(hashSize) if toEncrypt { refSize += encryption.KeyLength - chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize) } return &hasherStore{ - store: chunkStore, - hashFunc: hashFunc, - chunkEncryption: chunkEncryption, - hashSize: hashSize, - refSize: refSize, - wg: &sync.WaitGroup{}, - closed: make(chan struct{}), + store: chunkStore, + toEncrypt: toEncrypt, + hashFunc: hashFunc, + hashSize: hashSize, + refSize: refSize, + wg: &sync.WaitGroup{}, + closed: make(chan struct{}), } } @@ -79,7 +64,7 @@ func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, c := chunkData size := chunkData.Size() var encryptionKey encryption.Key - if h.chunkEncryption != nil { + if h.toEncrypt { var err error c, encryptionKey, err = h.encryptChunkData(chunkData) if err != nil { @@ -155,23 +140,14 @@ func (h *hasherStore) encryptChunkData(chunkData ChunkData) (ChunkData, encrypti return nil, nil, fmt.Errorf("Invalid ChunkData, min length 8 got %v", len(chunkData)) } - encryptionKey, err := encryption.GenerateRandomKey() - if err != nil { - return nil, nil, err - } - - encryptedSpan, err := h.chunkEncryption.spanEncryption.Encrypt(chunkData[:8], encryptionKey) - if err != nil { - return nil, nil, err - } - encryptedData, err := h.chunkEncryption.dataEncryption.Encrypt(chunkData[8:], encryptionKey) + key, encryptedSpan, encryptedData, err := h.encrypt(chunkData) if err != nil { return nil, nil, err } c := make(ChunkData, len(encryptedSpan)+len(encryptedData)) copy(c[:8], encryptedSpan) copy(c[8:], encryptedData) - return c, encryptionKey, nil + return c, key, nil } func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryption.Key) (ChunkData, error) { @@ -179,12 +155,7 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp return nil, fmt.Errorf("Invalid ChunkData, min length 8 got %v", len(chunkData)) } - decryptedSpan, err := h.chunkEncryption.spanEncryption.Decrypt(chunkData[:8], encryptionKey) - if err != nil { - return nil, err - } - - decryptedData, err := h.chunkEncryption.dataEncryption.Decrypt(chunkData[8:], encryptionKey) + decryptedSpan, decryptedData, err := h.decrypt(chunkData, encryptionKey) if err != nil { return nil, err } @@ -201,13 +172,46 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp copy(c[:8], decryptedSpan) copy(c[8:], decryptedData[:length]) - return c[:length+8], nil + return c, nil } func (h *hasherStore) RefSize() int64 { return h.refSize } +func (h *hasherStore) encrypt(chunkData ChunkData) (encryption.Key, []byte, []byte, error) { + key := encryption.GenerateRandomKey(encryption.KeyLength) + encryptedSpan, err := h.newSpanEncryption(key).Encrypt(chunkData[:8]) + if err != nil { + return nil, nil, nil, err + } + encryptedData, err := h.newDataEncryption(key).Encrypt(chunkData[8:]) + if err != nil { + return nil, nil, nil, err + } + return key, encryptedSpan, encryptedData, nil +} + +func (h *hasherStore) decrypt(chunkData ChunkData, key encryption.Key) ([]byte, []byte, error) { + encryptedSpan, err := h.newSpanEncryption(key).Encrypt(chunkData[:8]) + if err != nil { + return nil, nil, err + } + encryptedData, err := h.newDataEncryption(key).Encrypt(chunkData[8:]) + if err != nil { + return nil, nil, err + } + return encryptedSpan, encryptedData, nil +} + +func (h *hasherStore) newSpanEncryption(key encryption.Key) encryption.Encryption { + return encryption.New(key, 0, uint32(chunk.DefaultSize/h.refSize), sha3.NewKeccak256) +} + +func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryption { + return encryption.New(key, int(chunk.DefaultSize), 0, sha3.NewKeccak256) +} + func (h *hasherStore) storeChunk(ctx context.Context, chunk *Chunk) { h.wg.Add(1) go func() { From 4bb25042eb957662155079e06bb2efbdd866eb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Sep 2018 17:12:39 +0300 Subject: [PATCH 108/126] consensus/clique, core: chain maker clique + error tests --- consensus/clique/clique.go | 49 +++++--- consensus/clique/snapshot.go | 16 +-- consensus/clique/snapshot_test.go | 194 ++++++++++++++++++++++-------- core/chain_makers.go | 10 +- miner/worker_test.go | 1 + 5 files changed, 195 insertions(+), 75 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 0ff72e55c6..2b69141c12 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -93,27 +93,33 @@ var ( // errMissingSignature is returned if a block's extra-data section doesn't seem // to contain a 65 byte secp256k1 signature. - errMissingSignature = errors.New("extra-data 65 byte suffix signature missing") + errMissingSignature = errors.New("extra-data 65 byte signature suffix missing") // errExtraSigners is returned if non-checkpoint block contain signer data in // their extra-data fields. errExtraSigners = errors.New("non-checkpoint block contains extra signer list") // errInvalidCheckpointSigners is returned if a checkpoint block contains an - // invalid list of signers (i.e. non divisible by 20 bytes, or not the correct - // ones). + // invalid list of signers (i.e. non divisible by 20 bytes). errInvalidCheckpointSigners = errors.New("invalid signer list on checkpoint block") + // errMismatchingCheckpointSigners is returned if a checkpoint block contains a + // list of signers different than the one the local node calculated. + errMismatchingCheckpointSigners = errors.New("mismatching signer list on checkpoint block") + // errInvalidMixDigest is returned if a block's mix digest is non-zero. errInvalidMixDigest = errors.New("non-zero mix digest") // errInvalidUncleHash is returned if a block contains an non-empty uncle list. errInvalidUncleHash = errors.New("non empty uncle hash") - // errInvalidDifficulty is returned if the difficulty of a block is not either - // of 1 or 2, or if the value does not match the turn of the signer. + // errInvalidDifficulty is returned if the difficulty of a block neither 1 or 2. errInvalidDifficulty = errors.New("invalid difficulty") + // errWrongDifficulty is returned if the difficulty of a block doesn't match the + // turn of the signer. + errWrongDifficulty = errors.New("wrong difficulty") + // ErrInvalidTimestamp is returned if the timestamp of a block is lower than // the previous block's timestamp + the minimum block period. ErrInvalidTimestamp = errors.New("invalid timestamp") @@ -122,8 +128,12 @@ var ( // be modified via out-of-range or non-contiguous headers. errInvalidVotingChain = errors.New("invalid voting chain") - // errUnauthorized is returned if a header is signed by a non-authorized entity. - errUnauthorized = errors.New("unauthorized") + // errUnauthorizedSigner is returned if a header is signed by a non-authorized entity. + errUnauthorizedSigner = errors.New("unauthorized signer") + + // errRecentlySigned is returned if a header is signed by an authorized entity + // that already signed a header recently, thus is temporarily not allowed to. + errRecentlySigned = errors.New("recently signed") // errWaitTransactions is returned if an empty block is attempted to be sealed // on an instant chain (0 second period). It's important to refuse these as the @@ -205,6 +215,9 @@ type Clique struct { signer common.Address // Ethereum address of the signing key signFn SignerFn // Signer function to authorize hashes with lock sync.RWMutex // Protects the signer fields + + // The fields below are for testing only + fakeDiff bool // Skip difficulty verifications } // New creates a Clique proof-of-authority consensus engine with the initial @@ -359,7 +372,7 @@ func (c *Clique) verifyCascadingFields(chain consensus.ChainReader, header *type } extraSuffix := len(header.Extra) - extraSeal if !bytes.Equal(header.Extra[extraVanity:extraSuffix], signers) { - return errInvalidCheckpointSigners + return errMismatchingCheckpointSigners } } // All basic checks passed, verify the seal and return @@ -481,23 +494,25 @@ func (c *Clique) verifySeal(chain consensus.ChainReader, header *types.Header, p return err } if _, ok := snap.Signers[signer]; !ok { - return errUnauthorized + return errUnauthorizedSigner } for seen, recent := range snap.Recents { if recent == signer { // Signer is among recents, only fail if the current block doesn't shift it out if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit { - return errUnauthorized + return errRecentlySigned } } } // Ensure that the difficulty corresponds to the turn-ness of the signer - inturn := snap.inturn(header.Number.Uint64(), signer) - if inturn && header.Difficulty.Cmp(diffInTurn) != 0 { - return errInvalidDifficulty - } - if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 { - return errInvalidDifficulty + if !c.fakeDiff { + inturn := snap.inturn(header.Number.Uint64(), signer) + if inturn && header.Difficulty.Cmp(diffInTurn) != 0 { + return errWrongDifficulty + } + if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 { + return errWrongDifficulty + } } return nil } @@ -613,7 +628,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results c return err } if _, authorized := snap.Signers[signer]; !authorized { - return errUnauthorized + return errUnauthorizedSigner } // If we're amongst the recent signers, wait for the next block for seen, recent := range snap.Recents { diff --git a/consensus/clique/snapshot.go b/consensus/clique/snapshot.go index 2333d69247..54d4555f3b 100644 --- a/consensus/clique/snapshot.go +++ b/consensus/clique/snapshot.go @@ -57,12 +57,12 @@ type Snapshot struct { Tally map[common.Address]Tally `json:"tally"` // Current vote tally to avoid recalculating } -// signers implements the sort interface to allow sorting a list of addresses -type signers []common.Address +// signersAscending implements the sort interface to allow sorting a list of addresses +type signersAscending []common.Address -func (s signers) Len() int { return len(s) } -func (s signers) Less(i, j int) bool { return bytes.Compare(s[i][:], s[j][:]) < 0 } -func (s signers) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s signersAscending) Len() int { return len(s) } +func (s signersAscending) Less(i, j int) bool { return bytes.Compare(s[i][:], s[j][:]) < 0 } +func (s signersAscending) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // newSnapshot creates a new snapshot with the specified startup parameters. This // method does not initialize the set of recent signers, so only ever use if for @@ -214,11 +214,11 @@ func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) { return nil, err } if _, ok := snap.Signers[signer]; !ok { - return nil, errUnauthorized + return nil, errUnauthorizedSigner } for _, recent := range snap.Recents { if recent == signer { - return nil, errUnauthorized + return nil, errRecentlySigned } } snap.Recents[number] = signer @@ -298,7 +298,7 @@ func (s *Snapshot) signers() []common.Address { for sig := range s.Signers { sigs = append(sigs, sig) } - sort.Sort(signers(sigs)) + sort.Sort(signersAscending(sigs)) return sigs } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 17719884f0..71fe7ce8b2 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -19,24 +19,18 @@ package clique import ( "bytes" "crypto/ecdsa" - "math/big" + "sort" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) -type testerVote struct { - signer string - voted string - auth bool -} - // testerAccountPool is a pool to maintain currently active tester accounts, // mapped from textual names used in the tests below to actual Ethereum private // keys capable of signing transactions. @@ -50,17 +44,26 @@ func newTesterAccountPool() *testerAccountPool { } } -func (ap *testerAccountPool) sign(header *types.Header, signer string) { - // Ensure we have a persistent key for the signer - if ap.accounts[signer] == nil { - ap.accounts[signer], _ = crypto.GenerateKey() +// checkpoint creates a Clique checkpoint signer section from the provided list +// of authorized signers and embeds it into the provided header. +func (ap *testerAccountPool) checkpoint(header *types.Header, signers []string) { + auths := make([]common.Address, len(signers)) + for i, signer := range signers { + auths[i] = ap.address(signer) + } + sort.Sort(signersAscending(auths)) + for i, auth := range auths { + copy(header.Extra[extraVanity+i*common.AddressLength:], auth.Bytes()) } - // Sign the header and embed the signature in extra data - sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer]) - copy(header.Extra[len(header.Extra)-65:], sig) } +// address retrieves the Ethereum address of a tester account by label, creating +// a new account if no previous one exists yet. func (ap *testerAccountPool) address(account string) common.Address { + // Return the zero account for non-addresses + if account == "" { + return common.Address{} + } // Ensure we have a persistent key for the account if ap.accounts[account] == nil { ap.accounts[account], _ = crypto.GenerateKey() @@ -69,32 +72,38 @@ func (ap *testerAccountPool) address(account string) common.Address { return crypto.PubkeyToAddress(ap.accounts[account].PublicKey) } -// testerChainReader implements consensus.ChainReader to access the genesis -// block. All other methods and requests will panic. -type testerChainReader struct { - db ethdb.Database -} - -func (r *testerChainReader) Config() *params.ChainConfig { return params.AllCliqueProtocolChanges } -func (r *testerChainReader) CurrentHeader() *types.Header { panic("not supported") } -func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") } -func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block { panic("not supported") } -func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header { panic("not supported") } -func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header { - if number == 0 { - return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0) +// sign calculates a Clique digital signature for the given block and embeds it +// back into the header. +func (ap *testerAccountPool) sign(header *types.Header, signer string) { + // Ensure we have a persistent key for the signer + if ap.accounts[signer] == nil { + ap.accounts[signer], _ = crypto.GenerateKey() } - return nil + // Sign the header and embed the signature in extra data + sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer]) + copy(header.Extra[len(header.Extra)-extraSeal:], sig) } -// Tests that voting is evaluated correctly for various simple and complex scenarios. -func TestVoting(t *testing.T) { +// testerVote represents a single block signed by a parcitular account, where +// the account may or may not have cast a Clique vote. +type testerVote struct { + signer string + voted string + auth bool + checkpoint []string + newbatch bool +} + +// Tests that Clique signer voting is evaluated correctly for various simple and +// complex scenarios, as well as that a few special corner cases fail correctly. +func TestClique(t *testing.T) { // Define the various voting scenarios to test tests := []struct { epoch uint64 signers []string votes []testerVote results []string + failure error }{ { // Single signer, no votes cast @@ -322,10 +331,49 @@ func TestVoting(t *testing.T) { votes: []testerVote{ {signer: "A", voted: "C", auth: true}, {signer: "B"}, - {signer: "A"}, // Checkpoint block, (don't vote here, it's validated outside of snapshots) + {signer: "A", checkpoint: []string{"A", "B"}}, {signer: "B", voted: "C", auth: true}, }, results: []string{"A", "B"}, + }, { + // An unauthorized signer should not be able to sign blocks + signers: []string{"A"}, + votes: []testerVote{ + {signer: "B"}, + }, + failure: errUnauthorizedSigner, + }, { + // An authorized signer that signed recenty should not be able to sign again + signers: []string{"A", "B"}, + votes: []testerVote{ + {signer: "A"}, + {signer: "A"}, + }, + failure: errRecentlySigned, + }, { + // Recent signatures should not reset on checkpoint blocks imported in a batch + epoch: 3, + signers: []string{"A", "B", "C"}, + votes: []testerVote{ + {signer: "A"}, + {signer: "B"}, + {signer: "A", checkpoint: []string{"A", "B", "C"}}, + {signer: "A"}, + }, + failure: errRecentlySigned, + }, { + // Recent signatures should not reset on checkpoint blocks imported in a new + // batch (https://github.com/ethereum/go-ethereum/issues/17593). Whilst this + // seems overly specific and weird, it was a Rinkeby consensus split. + epoch: 3, + signers: []string{"A", "B", "C"}, + votes: []testerVote{ + {signer: "A"}, + {signer: "B"}, + {signer: "A", checkpoint: []string{"A", "B", "C"}}, + {signer: "A", newbatch: true}, + }, + failure: errRecentlySigned, }, } // Run through the scenarios and test them @@ -356,28 +404,78 @@ func TestVoting(t *testing.T) { genesis.Commit(db) // Assemble a chain of headers from the cast votes - headers := make([]*types.Header, len(tt.votes)) - for j, vote := range tt.votes { - headers[j] = &types.Header{ - Number: big.NewInt(int64(j) + 1), - Time: big.NewInt(int64(j) * 15), - Coinbase: accounts.address(vote.voted), - Extra: make([]byte, extraVanity+extraSeal), + config := *params.TestChainConfig + config.Clique = ¶ms.CliqueConfig{ + Period: 1, + Epoch: tt.epoch, + } + engine := New(config.Clique, db) + engine.fakeDiff = true + + blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { + // Cast the vote contained in this block + gen.SetCoinbase(accounts.address(tt.votes[j].voted)) + if tt.votes[j].auth { + var nonce types.BlockNonce + copy(nonce[:], nonceAuthVote) + gen.SetNonce(nonce) } + }) + // Iterate through the blocks and seal them individually + for j, block := range blocks { + // Geth the header and prepare it for signing + header := block.Header() if j > 0 { - headers[j].ParentHash = headers[j-1].Hash() + header.ParentHash = blocks[j-1].Hash() } - if vote.auth { - copy(headers[j].Nonce[:], nonceAuthVote) + header.Extra = make([]byte, extraVanity+extraSeal) + if auths := tt.votes[j].checkpoint; auths != nil { + header.Extra = make([]byte, extraVanity+len(auths)*common.AddressLength+extraSeal) + accounts.checkpoint(header, auths) } - accounts.sign(headers[j], vote.signer) + header.Difficulty = diffInTurn // Ignored, we just need a valid number + + // Generate the signature, embed it into the header and the block + accounts.sign(header, tt.votes[j].signer) + blocks[j] = block.WithSeal(header) + } + // Split the blocks up into individual import batches (cornercase testing) + batches := [][]*types.Block{nil} + for j, block := range blocks { + if tt.votes[j].newbatch { + batches = append(batches, nil) + } + batches[len(batches)-1] = append(batches[len(batches)-1], block) } // Pass all the headers through clique and ensure tallying succeeds - head := headers[len(headers)-1] - - snap, err := New(¶ms.CliqueConfig{Epoch: tt.epoch}, db).snapshot(&testerChainReader{db: db}, head.Number.Uint64(), head.Hash(), headers) + chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}) if err != nil { - t.Errorf("test %d: failed to create voting snapshot: %v", i, err) + t.Errorf("test %d: failed to create test chain: %v", i, err) + continue + } + failed := false + for j := 0; j < len(batches)-1; j++ { + if k, err := chain.InsertChain(batches[j]); err != nil { + t.Errorf("test %d: failed to import batch %d, block %d: %v", i, j, k, err) + failed = true + break + } + } + if failed { + continue + } + if _, err = chain.InsertChain(batches[len(batches)-1]); err != tt.failure { + t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure) + } + if tt.failure != nil { + continue + } + // No failure was produced or requested, generate the final voting snapshot + head := blocks[len(blocks)-1] + + snap, err := engine.snapshot(chain, head.NumberU64(), head.Hash(), nil) + if err != nil { + t.Errorf("test %d: failed to retrieve voting snapshot: %v", i, err) continue } // Verify the final list of signers against the expected ones diff --git a/core/chain_makers.go b/core/chain_makers.go index de0fc6be9e..3516734772 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -67,6 +67,11 @@ func (b *BlockGen) SetExtra(data []byte) { b.header.Extra = data } +// SetNonce sets the nonce field of the generated block. +func (b *BlockGen) SetNonce(nonce types.BlockNonce) { + b.header.Nonce = nonce +} + // AddTx adds a transaction to the generated block. If no coinbase has // been set, the block's coinbase is set to the zero address. // @@ -190,13 +195,14 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(b.header.Number) == 0 { misc.ApplyDAOHardFork(statedb) } - // Execute any user modifications to the block and finalize it + // Execute any user modifications to the block if gen != nil { gen(i, b) } - if b.engine != nil { + // Finalize and seal the block block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts) + // Write state changes to db root, err := statedb.Commit(config.IsEIP158(b.header.Number)) if err != nil { diff --git a/miner/worker_test.go b/miner/worker_test.go index 6de1177cdc..ad10d48efa 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -161,6 +161,7 @@ func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, eng t.Errorf("account balance mismatch: have %d, want %d", balance, 1000) } b.txPool.AddLocals(newTxs) + // Ensure the new tx events has been processed time.Sleep(100 * time.Millisecond) block, state = w.pending() From 2d98099c25f85a531f0acf28d89ca710f4569c6c Mon Sep 17 00:00:00 2001 From: chenyufeng Date: Tue, 11 Sep 2018 23:05:28 +0800 Subject: [PATCH 109/126] rlp: fix comment typo (#17640) --- rlp/typecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rlp/typecache.go b/rlp/typecache.go index 3df799e1ec..8c2dd518e2 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -76,7 +76,7 @@ func cachedTypeInfo1(typ reflect.Type, tags tags) (*typeinfo, error) { // another goroutine got the write lock first return info, nil } - // put a dummmy value into the cache before generating. + // put a dummy value into the cache before generating. // if the generator tries to lookup itself, it will get // the dummy value and won't call itself recursively. typeCache[key] = new(typeinfo) From 933ebaa47ee14c64dc9b014f727b0f8b59d2ca90 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 12 Sep 2018 08:25:04 +0200 Subject: [PATCH 110/126] cmd/swarm: password threw on upload manifest --- cmd/swarm/access.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index 12cfbfc1a4..df1edf0464 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -62,7 +62,6 @@ func accessNewPass(ctx *cli.Context) { utils.Fatalf("had an error printing the manifests: %v", err) } } else { - utils.Fatalf("uploading manifests") err = uploadManifests(ctx, m, nil) if err != nil { utils.Fatalf("had an error uploading the manifests: %v", err) From b040b750751cac7fd46893f4be3d2b30e741fb73 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Wed, 12 Sep 2018 15:11:35 +0800 Subject: [PATCH 111/126] cmd/clef: fix incorrect file permissions for secrets.dat Signed-off-by: YaoZengzeng --- cmd/clef/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 85704754de..f363a86f2c 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -225,7 +225,7 @@ func initializeSecrets(c *cli.Context) error { if _, err := os.Stat(location); err == nil { return fmt.Errorf("file %v already exists, will not overwrite", location) } - err = ioutil.WriteFile(location, masterSeed, 0700) + err = ioutil.WriteFile(location, masterSeed, 0400) if err != nil { return err } @@ -540,14 +540,14 @@ func readMasterKey(ctx *cli.Context) ([]byte, error) { // checkFile is a convenience function to check if a file // * exists -// * is mode 0600 +// * is mode 0400 func checkFile(filename string) error { info, err := os.Stat(filename) if err != nil { return fmt.Errorf("failed stat on %s: %v", filename, err) } // Check the unix permission bits - if info.Mode().Perm()&077 != 0 { + if info.Mode().Perm()&0377 != 0 { return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String()) } return nil From bfce00385f1c8dab222b7ddab6c336177a5ae731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Wed, 12 Sep 2018 11:24:56 +0200 Subject: [PATCH 112/126] Kademlia refactor (#17641) * swarm/network: simplify kademlia/hive; rid interfaces * swarm, swarm/network/stream, swarm/netork/simulations,, swarm/pss: adapt to new Kad API * swarm/network: minor changes re review; add missing lock to NeighbourhoodDepthC --- swarm/network/discovery.go | 76 ++++---- swarm/network/discovery_test.go | 2 +- swarm/network/hive.go | 71 ++----- swarm/network/hive_test.go | 8 +- swarm/network/kademlia.go | 154 +++++++-------- swarm/network/kademlia_test.go | 183 ++++++++---------- swarm/network/networkid_test.go | 2 +- swarm/network/protocol.go | 47 +---- .../simulations/discovery/discovery_test.go | 4 +- swarm/network/stream/delivery.go | 12 +- swarm/network/stream/snapshot_sync_test.go | 9 +- swarm/network/stream/stream.go | 18 +- swarm/network_test.go | 3 +- swarm/pss/pss.go | 43 ++-- swarm/pss/pss_test.go | 47 ++--- swarm/swarm.go | 2 +- 16 files changed, 260 insertions(+), 421 deletions(-) diff --git a/swarm/network/discovery.go b/swarm/network/discovery.go index 55bf7c0332..3019594809 100644 --- a/swarm/network/discovery.go +++ b/swarm/network/discovery.go @@ -26,30 +26,30 @@ import ( // discovery bzz extension for requesting and relaying node address records -// discPeer wraps BzzPeer and embeds an Overlay connectivity driver -type discPeer struct { +// Peer wraps BzzPeer and embeds Kademlia overlay connectivity driver +type Peer struct { *BzzPeer - overlay Overlay - sentPeers bool // whether we already sent peer closer to this address - mtx sync.RWMutex + kad *Kademlia + sentPeers bool // whether we already sent peer closer to this address + mtx sync.RWMutex // peers map[string]bool // tracks node records sent to the peer depth uint8 // the proximity order advertised by remote as depth of saturation } -// NewDiscovery constructs a discovery peer -func newDiscovery(p *BzzPeer, o Overlay) *discPeer { - d := &discPeer{ - overlay: o, +// NewPeer constructs a discovery peer +func NewPeer(p *BzzPeer, kad *Kademlia) *Peer { + d := &Peer{ + kad: kad, BzzPeer: p, peers: make(map[string]bool), } // record remote as seen so we never send a peer its own record - d.seen(d) + d.seen(p.BzzAddr) return d } // HandleMsg is the message handler that delegates incoming messages -func (d *discPeer) HandleMsg(ctx context.Context, msg interface{}) error { +func (d *Peer) HandleMsg(ctx context.Context, msg interface{}) error { switch msg := msg.(type) { case *peersMsg: @@ -64,24 +64,18 @@ func (d *discPeer) HandleMsg(ctx context.Context, msg interface{}) error { } // NotifyDepth sends a message to all connections if depth of saturation is changed -func NotifyDepth(depth uint8, h Overlay) { - f := func(val OverlayConn, po int, _ bool) bool { - dp, ok := val.(*discPeer) - if ok { - dp.NotifyDepth(depth) - } +func NotifyDepth(depth uint8, kad *Kademlia) { + f := func(val *Peer, po int, _ bool) bool { + val.NotifyDepth(depth) return true } - h.EachConn(nil, 255, f) + kad.EachConn(nil, 255, f) } // NotifyPeer informs all peers about a newly added node -func NotifyPeer(p OverlayAddr, k Overlay) { - f := func(val OverlayConn, po int, _ bool) bool { - dp, ok := val.(*discPeer) - if ok { - dp.NotifyPeer(p, uint8(po)) - } +func NotifyPeer(p *BzzAddr, k *Kademlia) { + f := func(val *Peer, po int, _ bool) bool { + val.NotifyPeer(p, uint8(po)) return true } k.EachConn(p.Address(), 255, f) @@ -91,22 +85,20 @@ func NotifyPeer(p OverlayAddr, k Overlay) { // the peer's PO is within the recipients advertised depth // OR the peer is closer to the recipient than self // unless already notified during the connection session -func (d *discPeer) NotifyPeer(a OverlayAddr, po uint8) { +func (d *Peer) NotifyPeer(a *BzzAddr, po uint8) { // immediately return if (po < d.getDepth() && pot.ProxCmp(d.localAddr, d, a) != 1) || d.seen(a) { return } - // log.Trace(fmt.Sprintf("%08x peer %08x notified of peer %08x", d.localAddr.Over()[:4], d.Address()[:4], a.Address()[:4])) resp := &peersMsg{ - Peers: []*BzzAddr{ToAddr(a)}, + Peers: []*BzzAddr{a}, } go d.Send(context.TODO(), resp) } // NotifyDepth sends a subPeers Msg to the receiver notifying them about // a change in the depth of saturation -func (d *discPeer) NotifyDepth(po uint8) { - // log.Trace(fmt.Sprintf("%08x peer %08x notified of new depth %v", d.localAddr.Over()[:4], d.Address()[:4], po)) +func (d *Peer) NotifyDepth(po uint8) { go d.Send(context.TODO(), &subPeersMsg{Depth: po}) } @@ -141,7 +133,7 @@ func (msg peersMsg) String() string { // handlePeersMsg called by the protocol when receiving peerset (for target address) // list of nodes ([]PeerAddr in peersMsg) is added to the overlay db using the // Register interface method -func (d *discPeer) handlePeersMsg(msg *peersMsg) error { +func (d *Peer) handlePeersMsg(msg *peersMsg) error { // register all addresses if len(msg.Peers) == 0 { return nil @@ -149,12 +141,12 @@ func (d *discPeer) handlePeersMsg(msg *peersMsg) error { for _, a := range msg.Peers { d.seen(a) - NotifyPeer(a, d.overlay) + NotifyPeer(a, d.kad) } - return d.overlay.Register(toOverlayAddrs(msg.Peers...)) + return d.kad.Register(msg.Peers...) } -// subPeers msg is communicating the depth/sharpness/focus of the overlay table of a peer +// subPeers msg is communicating the depth of the overlay table of a peer type subPeersMsg struct { Depth uint8 } @@ -164,21 +156,20 @@ func (msg subPeersMsg) String() string { return fmt.Sprintf("%T: request peers > PO%02d. ", msg, msg.Depth) } -func (d *discPeer) handleSubPeersMsg(msg *subPeersMsg) error { +func (d *Peer) handleSubPeersMsg(msg *subPeersMsg) error { if !d.sentPeers { d.setDepth(msg.Depth) var peers []*BzzAddr - d.overlay.EachConn(d.Over(), 255, func(p OverlayConn, po int, isproxbin bool) bool { + d.kad.EachConn(d.Over(), 255, func(p *Peer, po int, isproxbin bool) bool { if pob, _ := pof(d, d.localAddr, 0); pob > po { return false } - if !d.seen(p) { - peers = append(peers, ToAddr(p.Off())) + if !d.seen(p.BzzAddr) { + peers = append(peers, p.BzzAddr) } return true }) if len(peers) > 0 { - // log.Debug(fmt.Sprintf("%08x: %v peers sent to %v", d.overlay.BaseAddr(), len(peers), d)) go d.Send(context.TODO(), &peersMsg{Peers: peers}) } } @@ -186,9 +177,9 @@ func (d *discPeer) handleSubPeersMsg(msg *subPeersMsg) error { return nil } -// seen takes an Overlay peer and checks if it was sent to a peer already +// seen takes an peer address and checks if it was sent to a peer already // if not, marks the peer as sent -func (d *discPeer) seen(p OverlayPeer) bool { +func (d *Peer) seen(p *BzzAddr) bool { d.mtx.Lock() defer d.mtx.Unlock() k := string(p.Address()) @@ -199,12 +190,13 @@ func (d *discPeer) seen(p OverlayPeer) bool { return false } -func (d *discPeer) getDepth() uint8 { +func (d *Peer) getDepth() uint8 { d.mtx.RLock() defer d.mtx.RUnlock() return d.depth } -func (d *discPeer) setDepth(depth uint8) { + +func (d *Peer) setDepth(depth uint8) { d.mtx.Lock() defer d.mtx.Unlock() d.depth = depth diff --git a/swarm/network/discovery_test.go b/swarm/network/discovery_test.go index 0427d81cad..494bc81969 100644 --- a/swarm/network/discovery_test.go +++ b/swarm/network/discovery_test.go @@ -33,7 +33,7 @@ func TestDiscovery(t *testing.T) { id := s.IDs[0] raddr := NewAddrFromNodeID(id) - pp.Register([]OverlayAddr{OverlayAddr(raddr)}) + pp.Register(raddr) // start the hive and wait for the connection pp.Start(s.Server) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 3660210883..425c1d5a1e 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -32,31 +32,10 @@ import ( Hive is the logistic manager of the swarm When the hive is started, a forever loop is launched that -asks the Overlay Topology driver (e.g., generic kademlia nodetable) +asks the kademlia nodetable to suggest peers to bootstrap connectivity */ -// Overlay is the interface for kademlia (or other topology drivers) -type Overlay interface { - // suggest peers to connect to - SuggestPeer() (OverlayAddr, int, bool) - // register and deregister peer connections - On(OverlayConn) (depth uint8, changed bool) - Off(OverlayConn) - // register peer addresses - Register([]OverlayAddr) error - // iterate over connected peers - EachConn([]byte, int, func(OverlayConn, int, bool) bool) - // iterate over known peers (address records) - EachAddr([]byte, int, func(OverlayAddr, int, bool) bool) - // pretty print the connectivity - String() string - // base Overlay address of the node itself - BaseAddr() []byte - // connectivity health check used for testing - Healthy(*PeerPot) *Health -} - // HiveParams holds the config options to hive type HiveParams struct { Discovery bool // if want discovery of not @@ -78,7 +57,7 @@ func NewHiveParams() *HiveParams { // Hive manages network connections of the swarm node type Hive struct { *HiveParams // settings - Overlay // the overlay connectiviy driver + *Kademlia // the overlay connectiviy driver Store state.Store // storage interface to save peers across sessions addPeer func(*discover.Node) // server callback to connect to a peer // bookkeeping @@ -88,12 +67,12 @@ type Hive struct { // NewHive constructs a new hive // HiveParams: config parameters -// Overlay: connectivity driver using a network topology +// Kademlia: connectivity driver using a network topology // StateStore: to save peers across sessions -func NewHive(params *HiveParams, overlay Overlay, store state.Store) *Hive { +func NewHive(params *HiveParams, kad *Kademlia, store state.Store) *Hive { return &Hive{ HiveParams: params, - Overlay: overlay, + Kademlia: kad, Store: store, } } @@ -133,7 +112,7 @@ func (h *Hive) Stop() error { } } log.Info(fmt.Sprintf("%08x hive stopped, dropping peers", h.BaseAddr()[:4])) - h.EachConn(nil, 255, func(p OverlayConn, _ int, _ bool) bool { + h.EachConn(nil, 255, func(p *Peer, _ int, _ bool) bool { log.Info(fmt.Sprintf("%08x dropping peer %08x", h.BaseAddr()[:4], p.Address()[:4])) p.Drop(nil) return true @@ -151,14 +130,14 @@ func (h *Hive) connect() { addr, depth, changed := h.SuggestPeer() if h.Discovery && changed { - NotifyDepth(uint8(depth), h) + NotifyDepth(uint8(depth), h.Kademlia) } if addr == nil { continue } log.Trace(fmt.Sprintf("%08x hive connect() suggested %08x", h.BaseAddr()[:4], addr.Address()[:4])) - under, err := discover.ParseNode(string(addr.(Addr).Under())) + under, err := discover.ParseNode(string(addr.Under())) if err != nil { log.Warn(fmt.Sprintf("%08x unable to connect to bee %08x: invalid node URL: %v", h.BaseAddr()[:4], addr.Address()[:4], err)) continue @@ -170,19 +149,19 @@ func (h *Hive) connect() { // Run protocol run function func (h *Hive) Run(p *BzzPeer) error { - dp := newDiscovery(p, h) + dp := NewPeer(p, h.Kademlia) depth, changed := h.On(dp) // if we want discovery, advertise change of depth if h.Discovery { if changed { // if depth changed, send to all peers - NotifyDepth(depth, h) + NotifyDepth(depth, h.Kademlia) } else { // otherwise just send depth to new peer dp.NotifyDepth(depth) } } - NotifyPeer(p.Off(), h) + NotifyPeer(p.BzzAddr, h.Kademlia) defer h.Off(dp) return dp.Run(dp.HandleMsg) } @@ -206,17 +185,6 @@ func (h *Hive) PeerInfo(id discover.NodeID) interface{} { } } -// ToAddr returns the serialisable version of u -func ToAddr(pa OverlayPeer) *BzzAddr { - if addr, ok := pa.(*BzzAddr); ok { - return addr - } - if p, ok := pa.(*discPeer); ok { - return p.BzzAddr - } - return pa.(*BzzPeer).BzzAddr -} - // loadPeers, savePeer implement persistence callback/ func (h *Hive) loadPeers() error { var as []*BzzAddr @@ -230,28 +198,19 @@ func (h *Hive) loadPeers() error { } log.Info(fmt.Sprintf("hive %08x: peers loaded", h.BaseAddr()[:4])) - return h.Register(toOverlayAddrs(as...)) -} - -// toOverlayAddrs transforms an array of BzzAddr to OverlayAddr -func toOverlayAddrs(as ...*BzzAddr) (oas []OverlayAddr) { - for _, a := range as { - oas = append(oas, OverlayAddr(a)) - } - return + return h.Register(as...) } // savePeers, savePeer implement persistence callback/ func (h *Hive) savePeers() error { var peers []*BzzAddr - h.Overlay.EachAddr(nil, 256, func(pa OverlayAddr, i int, _ bool) bool { + h.Kademlia.EachAddr(nil, 256, func(pa *BzzAddr, i int, _ bool) bool { if pa == nil { log.Warn(fmt.Sprintf("empty addr: %v", i)) return true } - apa := ToAddr(pa) - log.Trace("saving peer", "peer", apa) - peers = append(peers, apa) + log.Trace("saving peer", "peer", pa) + peers = append(peers, pa) return true }) if err := h.Store.Put("peers", peers); err != nil { diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index c2abfb2aad..7ea000c1ae 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -41,7 +41,7 @@ func TestRegisterAndConnect(t *testing.T) { id := s.IDs[0] raddr := NewAddrFromNodeID(id) - pp.Register([]OverlayAddr{OverlayAddr(raddr)}) + pp.Register(raddr) // start the hive and wait for the connection err := pp.Start(s.Server) @@ -77,7 +77,7 @@ func TestHiveStatePersistance(t *testing.T) { peers := make(map[string]bool) for _, id := range s.IDs { raddr := NewAddrFromNodeID(id) - pp.Register([]OverlayAddr{OverlayAddr(raddr)}) + pp.Register(raddr) peers[raddr.String()] = true } @@ -97,8 +97,8 @@ func TestHiveStatePersistance(t *testing.T) { pp.Start(s1.Server) i := 0 - pp.Overlay.EachAddr(nil, 256, func(addr OverlayAddr, po int, nn bool) bool { - delete(peers, addr.(*BzzAddr).String()) + pp.Kademlia.EachAddr(nil, 256, func(addr *BzzAddr, po int, nn bool) bool { + delete(peers, addr.String()) i++ return true }) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 0177d449c4..55a0c6f13d 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -62,7 +62,7 @@ type KadParams struct { RetryExponent int // exponent to multiply retry intervals with MaxRetries int // maximum number of redial attempts // function to sanction or prevent suggesting a peer - Reachable func(OverlayAddr) bool + Reachable func(*BzzAddr) bool } // NewKadParams returns a params struct with default values @@ -106,45 +106,22 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { } } -// OverlayPeer interface captures the common aspect of view of a peer from the Overlay -// topology driver -type OverlayPeer interface { - Address() []byte -} - -// OverlayConn represents a connected peer -type OverlayConn interface { - OverlayPeer - Drop(error) // call to indicate a peer should be expunged - Off() OverlayAddr // call to return a persitent OverlayAddr -} - -// OverlayAddr represents a kademlia peer record -type OverlayAddr interface { - OverlayPeer - Update(OverlayAddr) OverlayAddr // returns the updated version of the original -} - -// entry represents a Kademlia table entry (an extension of OverlayPeer) +// entry represents a Kademlia table entry (an extension of BzzAddr) type entry struct { - OverlayPeer + *BzzAddr + conn *Peer seenAt time.Time retries int } -// newEntry creates a kademlia peer from an OverlayPeer interface -func newEntry(p OverlayPeer) *entry { +// newEntry creates a kademlia peer from a *Peer +func newEntry(p *BzzAddr) *entry { return &entry{ - OverlayPeer: p, - seenAt: time.Now(), + BzzAddr: p, + seenAt: time.Now(), } } -// Bin is the binary (bitvector) serialisation of the entry address -func (e *entry) Bin() string { - return pot.ToBin(e.addr().Address()) -} - // Label is a short tag for the entry for debug func Label(e *entry) string { return fmt.Sprintf("%s (%d)", e.Hex()[:4], e.retries) @@ -152,29 +129,12 @@ func Label(e *entry) string { // Hex is the hexadecimal serialisation of the entry address func (e *entry) Hex() string { - return fmt.Sprintf("%x", e.addr().Address()) + return fmt.Sprintf("%x", e.Address()) } -// String is the short tag for the entry -func (e *entry) String() string { - return fmt.Sprintf("%s (%d)", e.Hex()[:8], e.retries) -} - -// addr returns the kad peer record (OverlayAddr) corresponding to the entry -func (e *entry) addr() OverlayAddr { - a, _ := e.OverlayPeer.(OverlayAddr) - return a -} - -// conn returns the connected peer (OverlayPeer) corresponding to the entry -func (e *entry) conn() OverlayConn { - c, _ := e.OverlayPeer.(OverlayConn) - return c -} - -// Register enters each OverlayAddr as kademlia peer record into the +// Register enters each address as kademlia peer record into the // database of known peer addresses -func (k *Kademlia) Register(peers []OverlayAddr) error { +func (k *Kademlia) Register(peers ...*BzzAddr) error { k.lock.Lock() defer k.lock.Unlock() var known, size int @@ -203,7 +163,6 @@ func (k *Kademlia) Register(peers []OverlayAddr) error { if k.addrCountC != nil && size-known > 0 { k.addrCountC <- k.addrs.Size() } - // log.Trace(fmt.Sprintf("%x registered %v peers, %v known, total: %v", k.BaseAddr()[:4], size, known, k.addrs.Size())) k.sendNeighbourhoodDepthChange() return nil @@ -212,7 +171,7 @@ func (k *Kademlia) Register(peers []OverlayAddr) error { // SuggestPeer returns a known peer for the lowest proximity bin for the // lowest bincount below depth // naturally if there is an empty row it returns a peer for that -func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { +func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { k.lock.Lock() defer k.lock.Unlock() minsize := k.MinBinSize @@ -224,15 +183,18 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { if po < depth { return false } - a = k.callable(val) + e := val.(*entry) + c := k.callable(e) + if c { + a = e.BzzAddr + } ppo = po - return a == nil + return !c }) if a != nil { log.Trace(fmt.Sprintf("%08x candidate nearest neighbour found: %v (%v)", k.BaseAddr()[:4], a, ppo)) return a, 0, false } - // log.Trace(fmt.Sprintf("%08x no candidate nearest neighbours to connect to (Depth: %v, minProxSize: %v) %#v", k.BaseAddr()[:4], depth, k.MinProxBinSize, a)) var bpo []int prev := -1 @@ -250,7 +212,6 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { }) // all buckets are full, ie., minsize == k.MinBinSize if len(bpo) == 0 { - // log.Debug(fmt.Sprintf("%08x: all bins saturated", k.BaseAddr()[:4])) return nil, 0, false } // as long as we got candidate peers to connect to @@ -264,8 +225,12 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { return false } return f(func(val pot.Val, _ int) bool { - a = k.callable(val) - return a == nil + e := val.(*entry) + c := k.callable(e) + if c { + a = e.BzzAddr + } + return !c }) }) // found a candidate @@ -282,25 +247,26 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { } // On inserts the peer as a kademlia peer into the live peers -func (k *Kademlia) On(p OverlayConn) (uint8, bool) { +func (k *Kademlia) On(p *Peer) (uint8, bool) { k.lock.Lock() defer k.lock.Unlock() - e := newEntry(p) var ins bool k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(v pot.Val) pot.Val { // if not found live if v == nil { ins = true // insert new online peer into conns - return e + return p } // found among live peers, do nothing return v }) if ins { + a := newEntry(p.BzzAddr) + a.conn = p // insert new online peer into addrs k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { - return e + return a }) // send new address count value only if the peer is inserted if k.addrCountC != nil { @@ -324,6 +290,8 @@ func (k *Kademlia) On(p OverlayConn) (uint8, bool) { // Not receiving from the returned channel will block On function // when the neighbourhood depth is changed. func (k *Kademlia) NeighbourhoodDepthC() <-chan int { + k.lock.Lock() + defer k.lock.Unlock() if k.nDepthC == nil { k.nDepthC = make(chan int) } @@ -357,7 +325,7 @@ func (k *Kademlia) AddrCountC() <-chan int { } // Off removes a peer from among live peers -func (k *Kademlia) Off(p OverlayConn) { +func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() var del bool @@ -367,7 +335,7 @@ func (k *Kademlia) Off(p OverlayConn) { panic(fmt.Sprintf("connected peer not found %v", p)) } del = true - return newEntry(p.Off()) + return newEntry(p.BzzAddr) }) if del { @@ -383,7 +351,7 @@ func (k *Kademlia) Off(p OverlayConn) { } } -func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(conn OverlayConn, po int) bool) { +func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(conn *Peer, po int) bool) { k.lock.RLock() defer k.lock.RUnlock() @@ -403,7 +371,7 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con for bin := startPo; bin <= endPo; bin++ { f(func(val pot.Val, _ int) bool { - return eachBinFunc(val.(*entry).conn(), bin) + return eachBinFunc(val.(*Peer), bin) }) } return true @@ -413,13 +381,13 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con // EachConn is an iterator with args (base, po, f) applies f to each live peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used -func (k *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { +func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int, bool) bool) { k.lock.RLock() defer k.lock.RUnlock() k.eachConn(base, o, f) } -func (k *Kademlia) eachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { +func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int, bool) bool) { if len(base) == 0 { base = k.base } @@ -428,20 +396,20 @@ func (k *Kademlia) eachConn(base []byte, o int, f func(OverlayConn, int, bool) b if po > o { return true } - return f(val.(*entry).conn(), po, po >= depth) + return f(val.(*Peer), po, po >= depth) }) } // EachAddr called with (base, po, f) is an iterator applying f to each known peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used -func (k *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int, bool) bool) { +func (k *Kademlia) EachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool) { k.lock.RLock() defer k.lock.RUnlock() k.eachAddr(base, o, f) } -func (k *Kademlia) eachAddr(base []byte, o int, f func(OverlayAddr, int, bool) bool) { +func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool) { if len(base) == 0 { base = k.base } @@ -450,7 +418,7 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(OverlayAddr, int, bool) b if po > o { return true } - return f(val.(*entry).addr(), po, po >= depth) + return f(val.(*entry).BzzAddr, po, po >= depth) }) } @@ -472,12 +440,11 @@ func (k *Kademlia) neighbourhoodDepth() (depth int) { return depth } -// callable when called with val, -func (k *Kademlia) callable(val pot.Val) OverlayAddr { - e := val.(*entry) +// callable decides if an address entry represents a callable peer +func (k *Kademlia) callable(e *entry) bool { // not callable if peer is live or exceeded maxRetries - if e.conn() != nil || e.retries > k.MaxRetries { - return nil + if e.conn != nil || e.retries > k.MaxRetries { + return false } // calculate the allowed number of retries based on time lapsed since last seen timeAgo := int64(time.Since(e.seenAt)) @@ -491,17 +458,17 @@ func (k *Kademlia) callable(val pot.Val) OverlayAddr { // peer can be retried again if retries < e.retries { log.Trace(fmt.Sprintf("%08x: %v long time since last try (at %v) needed before retry %v, wait only warrants %v", k.BaseAddr()[:4], e, timeAgo, e.retries, retries)) - return nil + return false } // function to sanction or prevent suggesting a peer - if k.Reachable != nil && !k.Reachable(e.addr()) { + if k.Reachable != nil && !k.Reachable(e.BzzAddr) { log.Trace(fmt.Sprintf("%08x: peer %v is temporarily not callable", k.BaseAddr()[:4], e)) - return nil + return false } e.retries++ log.Trace(fmt.Sprintf("%08x: peer %v is callable", k.BaseAddr()[:4], e)) - return e.addr() + return true } // BaseAddr return the kademlia base address @@ -516,7 +483,8 @@ func (k *Kademlia) String() string { return k.string() } -// String returns kademlia table + kaddb table displayed with ascii +// string returns kademlia table + kaddb table displayed with ascii +// caller must hold the lock func (k *Kademlia) string() string { wsrow := " " var rows []string @@ -538,7 +506,7 @@ func (k *Kademlia) string() string { row := []string{fmt.Sprintf("%2d", size)} rest -= size f(func(val pot.Val, vpo int) bool { - e := val.(*entry) + e := val.(*Peer) row = append(row, fmt.Sprintf("%x", e.Address()[:2])) rowlen++ return rowlen < 4 @@ -594,8 +562,9 @@ type PeerPot struct { EmptyBins []int } -// NewPeerPotMap creates a map of pot record of OverlayAddr with keys +// NewPeerPotMap creates a map of pot record of *BzzAddr with keys // as hexadecimal representations of the address. +// used for testing only func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot { // create a table of all nodes for health check np := pot.NewPot(nil, 0) @@ -640,6 +609,7 @@ func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot { // saturation returns the lowest proximity order that the bin for that order // has less than n peers +// It is used in Healthy function for testing only func (k *Kademlia) saturation(n int) int { prev := -1 k.addrs.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { @@ -654,7 +624,7 @@ func (k *Kademlia) saturation(n int) int { } // full returns true if all required bins have connected peers. -// It is used in Healthy function. +// It is used in Healthy function for testing only func (k *Kademlia) full(emptyBins []int) (full bool) { prev := 0 e := len(emptyBins) @@ -688,10 +658,13 @@ func (k *Kademlia) full(emptyBins []int) (full bool) { return e == 0 } +// knowNearestNeighbours tests if all known nearest neighbours given as arguments +// are found in the addressbook +// It is used in Healthy function for testing only func (k *Kademlia) knowNearestNeighbours(peers [][]byte) bool { pm := make(map[string]bool) - k.eachAddr(nil, 255, func(p OverlayAddr, po int, nn bool) bool { + k.eachAddr(nil, 255, func(p *BzzAddr, po int, nn bool) bool { if !nn { return false } @@ -709,10 +682,13 @@ func (k *Kademlia) knowNearestNeighbours(peers [][]byte) bool { return true } +// gotNearestNeighbours tests if all known nearest neighbours given as arguments +// are connected peers +// It is used in Healthy function for testing only func (k *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - k.eachConn(nil, 255, func(p OverlayConn, po int, nn bool) bool { + k.eachConn(nil, 255, func(p *Peer, po int, nn bool) bool { if !nn { return false } @@ -735,6 +711,7 @@ func (k *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool, n int, missin } // Health state of the Kademlia +// used for testing only type Health struct { KnowNN bool // whether node knows all its nearest neighbours GotNN bool // whether node is connected to all its nearest neighbours @@ -746,6 +723,7 @@ type Health struct { // Healthy reports the health state of the kademlia connectivity // returns a Health struct +// used for testing only func (k *Kademlia) Healthy(pp *PeerPot) *Health { k.lock.RLock() defer k.lock.RUnlock() diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index b60e1e9a39..903c8dbdac 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -38,71 +38,42 @@ func testKadPeerAddr(s string) *BzzAddr { return &BzzAddr{OAddr: a, UAddr: a} } -type testDropPeer struct { - Peer - dropc chan error -} - -type dropError struct { - error - addr string -} - -func (d *testDropPeer) Drop(err error) { - err2 := &dropError{err, binStr(d)} - d.dropc <- err2 -} - -type testKademlia struct { - *Kademlia - Discovery bool - dropc chan error -} - -func newTestKademlia(b string) *testKademlia { +func newTestKademlia(b string) *Kademlia { params := NewKadParams() params.MinBinSize = 1 params.MinProxBinSize = 2 base := pot.NewAddressFromString(b) - return &testKademlia{ - NewKademlia(base, params), - false, - make(chan error), - } + return NewKademlia(base, params) } -func (k *testKademlia) newTestKadPeer(s string) Peer { - return &testDropPeer{&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc} +func newTestKadPeer(k *Kademlia, s string) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k) } -func (k *testKademlia) On(ons ...string) *testKademlia { +func On(k *Kademlia, ons ...string) { for _, s := range ons { - k.Kademlia.On(k.newTestKadPeer(s).(OverlayConn)) + k.On(newTestKadPeer(k, s)) } - return k } -func (k *testKademlia) Off(offs ...string) *testKademlia { +func Off(k *Kademlia, offs ...string) { for _, s := range offs { - k.Kademlia.Off(k.newTestKadPeer(s).(OverlayConn)) + k.Off(newTestKadPeer(k, s)) } - - return k } -func (k *testKademlia) Register(regs ...string) *testKademlia { - var as []OverlayAddr +func Register(k *Kademlia, regs ...string) { + var as []*BzzAddr for _, s := range regs { as = append(as, testKadPeerAddr(s)) } - err := k.Kademlia.Register(as) + err := k.Register(as...) if err != nil { panic(err.Error()) } - return k } -func testSuggestPeer(t *testing.T, k *testKademlia, expAddr string, expPo int, expWant bool) error { +func testSuggestPeer(k *Kademlia, expAddr string, expPo int, expWant bool) error { addr, o, want := k.SuggestPeer() if binStr(addr) != expAddr { return fmt.Errorf("incorrect peer address suggested. expected %v, got %v", expAddr, binStr(addr)) @@ -116,7 +87,7 @@ func testSuggestPeer(t *testing.T, k *testKademlia, expAddr string, expPo int, e return nil } -func binStr(a OverlayPeer) string { +func binStr(a *BzzAddr) string { if a == nil { return "" } @@ -125,15 +96,17 @@ func binStr(a OverlayPeer) string { func TestSuggestPeerBug(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("00000000").On( + k := newTestKademlia("00000000") + On(k, "10000000", "11000000", "01000000", "00010000", "00011000", - ).Off( + ) + Off(k, "01000000", ) - err := testSuggestPeer(t, k, "01000000", 0, false) + err := testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -141,140 +114,140 @@ func TestSuggestPeerBug(t *testing.T) { func TestSuggestPeerFindPeers(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("00000000").On("00100000") - err := testSuggestPeer(t, k, "", 0, false) + k := newTestKademlia("00000000") + On(k, "00100000") + err := testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } // 2 row gap, saturated proxbin, no callables -> want PO 0 - k.On("00010000") - err = testSuggestPeer(t, k, "", 0, false) + On(k, "00010000") + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } // 1 row gap (1 less), saturated proxbin, no callables -> want PO 1 - k.On("10000000") - err = testSuggestPeer(t, k, "", 1, false) + On(k, "10000000") + err = testSuggestPeer(k, "", 1, false) if err != nil { t.Fatal(err.Error()) } // no gap (1 less), saturated proxbin, no callables -> do not want more - k.On("01000000", "00100001") - err = testSuggestPeer(t, k, "", 0, false) + On(k, "01000000", "00100001") + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } // oversaturated proxbin, > do not want more - k.On("00100001") - err = testSuggestPeer(t, k, "", 0, false) + On(k, "00100001") + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } // reintroduce gap, disconnected peer callable - // log.Info(k.String()) - k.Off("01000000") - err = testSuggestPeer(t, k, "01000000", 0, false) + Off(k, "01000000") + err = testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } // second time disconnected peer not callable // with reasonably set Interval - err = testSuggestPeer(t, k, "", 1, true) + err = testSuggestPeer(k, "", 1, true) if err != nil { t.Fatal(err.Error()) } // on and off again, peer callable again - k.On("01000000") - k.Off("01000000") - err = testSuggestPeer(t, k, "01000000", 0, false) + On(k, "01000000") + Off(k, "01000000") + err = testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("01000000") + On(k, "01000000") // new closer peer appears, it is immediately wanted - k.Register("00010001") - err = testSuggestPeer(t, k, "00010001", 0, false) + Register(k, "00010001") + err = testSuggestPeer(k, "00010001", 0, false) if err != nil { t.Fatal(err.Error()) } // PO1 disconnects - k.On("00010001") + On(k, "00010001") log.Info(k.String()) - k.Off("01000000") + Off(k, "01000000") log.Info(k.String()) // second time, gap filling - err = testSuggestPeer(t, k, "01000000", 0, false) + err = testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("01000000") - err = testSuggestPeer(t, k, "", 0, false) + On(k, "01000000") + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } k.MinBinSize = 2 - err = testSuggestPeer(t, k, "", 0, true) + err = testSuggestPeer(k, "", 0, true) if err != nil { t.Fatal(err.Error()) } - k.Register("01000001") - err = testSuggestPeer(t, k, "01000001", 0, false) + Register(k, "01000001") + err = testSuggestPeer(k, "01000001", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("10000001") + On(k, "10000001") log.Trace(fmt.Sprintf("Kad:\n%v", k.String())) - err = testSuggestPeer(t, k, "", 1, true) + err = testSuggestPeer(k, "", 1, true) if err != nil { t.Fatal(err.Error()) } - k.On("01000001") - err = testSuggestPeer(t, k, "", 0, false) + On(k, "01000001") + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } k.MinBinSize = 3 - k.Register("10000010") - err = testSuggestPeer(t, k, "10000010", 0, false) + Register(k, "10000010") + err = testSuggestPeer(k, "10000010", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("10000010") - err = testSuggestPeer(t, k, "", 1, false) + On(k, "10000010") + err = testSuggestPeer(k, "", 1, false) if err != nil { t.Fatal(err.Error()) } - k.On("01000010") - err = testSuggestPeer(t, k, "", 2, false) + On(k, "01000010") + err = testSuggestPeer(k, "", 2, false) if err != nil { t.Fatal(err.Error()) } - k.On("00100010") - err = testSuggestPeer(t, k, "", 3, false) + On(k, "00100010") + err = testSuggestPeer(k, "", 3, false) if err != nil { t.Fatal(err.Error()) } - k.On("00010010") - err = testSuggestPeer(t, k, "", 0, false) + On(k, "00010010") + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -282,10 +255,8 @@ func TestSuggestPeerFindPeers(t *testing.T) { } func TestSuggestPeerRetries(t *testing.T) { - t.Skip("Test is disabled, because it is flaky. It fails with kademlia_test.go:346: incorrect peer address suggested. expected , got 01000000") - // 2 row gap, unsaturated proxbin, no callables -> want PO 0 k := newTestKademlia("00000000") - k.RetryInterval = int64(100 * time.Millisecond) // cycle + k.RetryInterval = int64(300 * time.Millisecond) // cycle k.MaxRetries = 50 k.RetryExponent = 2 sleep := func(n int) { @@ -296,53 +267,53 @@ func TestSuggestPeerRetries(t *testing.T) { time.Sleep(time.Duration(ts)) } - k.Register("01000000") - k.On("00000001", "00000010") - err := testSuggestPeer(t, k, "01000000", 0, false) + Register(k, "01000000") + On(k, "00000001", "00000010") + err := testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - err = testSuggestPeer(t, k, "", 0, false) + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } sleep(1) - err = testSuggestPeer(t, k, "01000000", 0, false) + err = testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - err = testSuggestPeer(t, k, "", 0, false) + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } sleep(1) - err = testSuggestPeer(t, k, "01000000", 0, false) + err = testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - err = testSuggestPeer(t, k, "", 0, false) + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } sleep(2) - err = testSuggestPeer(t, k, "01000000", 0, false) + err = testSuggestPeer(k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - err = testSuggestPeer(t, k, "", 0, false) + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } sleep(2) - err = testSuggestPeer(t, k, "", 0, false) + err = testSuggestPeer(k, "", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -350,7 +321,9 @@ func TestSuggestPeerRetries(t *testing.T) { } func TestKademliaHiveString(t *testing.T) { - k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") + k := newTestKademlia("00000000") + On(k, "01000000", "00100000") + Register(k, "10000000", "10000001") k.MaxProxDisplay = 8 h := k.String() expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" @@ -378,7 +351,7 @@ func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { continue } p := &BzzAddr{OAddr: a, UAddr: a} - if err := k.Register([]OverlayAddr{p}); err != nil { + if err := k.Register(p); err != nil { t.Fatal(err) } } @@ -392,12 +365,12 @@ func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { if a == nil { break } - k.On(&BzzPeer{BzzAddr: a.(*BzzAddr)}) + k.On(NewPeer(&BzzPeer{BzzAddr: a}, k)) } h := k.Healthy(pp) if !(h.GotNN && h.KnowNN && h.Full) { - t.Error("not healthy") + t.Fatalf("not healthy: %#v\n%v", h, k.String()) } } diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index 05134b083b..91a1f6d7bf 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -92,7 +92,7 @@ func TestNetworkID(t *testing.T) { if kademlias[node].addrs.Size() != len(netIDGroup)-1 { t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].addrs.Size(), len(netIDGroup)-1) } - kademlias[node].EachAddr(nil, 0, func(addr OverlayAddr, _ int, _ bool) bool { + kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int, _ bool) bool { found := false for _, nd := range netIDGroup { p := ToOverlayAddr(nd.Bytes()) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 7f7ca5eed5..ef0956d5f8 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -62,32 +62,6 @@ var DiscoverySpec = &protocols.Spec{ }, } -// Addr interface that peerPool needs -type Addr interface { - OverlayPeer - Over() []byte - Under() []byte - String() string - Update(OverlayAddr) OverlayAddr -} - -// Peer interface represents an live peer connection -type Peer interface { - Addr // the address of a peer - Conn // the live connection (protocols.Peer) - LastActive() time.Time // last time active -} - -// Conn interface represents an live peer connection -type Conn interface { - ID() discover.NodeID // the key that uniquely identifies the Node for the peerPool - Handshake(context.Context, interface{}, func(interface{}) error) (interface{}, error) // can send messages - Send(context.Context, interface{}) error // can send messages - Drop(error) // disconnect this peer - Run(func(context.Context, interface{}) error) error // the run function to run a protocol - Off() OverlayAddr -} - // BzzConfig captures the config params used by the hive type BzzConfig struct { OverlayAddr []byte // base address of the overlay network @@ -114,7 +88,7 @@ type Bzz struct { // * bzz config // * overlay driver // * peer store -func NewBzz(config *BzzConfig, kad Overlay, store state.Store, streamerSpec *protocols.Spec, streamerRun func(*BzzPeer) error) *Bzz { +func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec *protocols.Spec, streamerRun func(*BzzPeer) error) *Bzz { return &Bzz{ Hive: NewHive(config.HiveParams, kad, store), NetworkID: config.NetworkID, @@ -131,7 +105,7 @@ func (b *Bzz) UpdateLocalAddr(byteaddr []byte) *BzzAddr { b.localAddr = b.localAddr.Update(&BzzAddr{ UAddr: byteaddr, OAddr: b.localAddr.OAddr, - }).(*BzzAddr) + }) return b.localAddr } @@ -274,7 +248,7 @@ type BzzPeer struct { LightNode bool } -func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer { +func NewBzzPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer { return &BzzPeer{ Peer: p, localAddr: addr, @@ -282,11 +256,6 @@ func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer { } } -// Off returns the overlay peer record for offline persistence -func (p *BzzPeer) Off() OverlayAddr { - return p.BzzAddr -} - // LastActive returns the time the peer was last active func (p *BzzPeer) LastActive() time.Time { return p.lastActive @@ -388,8 +357,8 @@ func (a *BzzAddr) ID() discover.NodeID { } // Update updates the underlay address of a peer record -func (a *BzzAddr) Update(na OverlayAddr) OverlayAddr { - return &BzzAddr{a.OAddr, na.(Addr).Under()} +func (a *BzzAddr) Update(na *BzzAddr) *BzzAddr { + return &BzzAddr{a.OAddr, na.UAddr} } // String pretty prints the address @@ -410,9 +379,9 @@ func RandomAddr() *BzzAddr { } // NewNodeIDFromAddr transforms the underlay address to an adapters.NodeID -func NewNodeIDFromAddr(addr Addr) discover.NodeID { - log.Info(fmt.Sprintf("uaddr=%s", string(addr.Under()))) - node := discover.MustParseNode(string(addr.Under())) +func NewNodeIDFromAddr(addr *BzzAddr) discover.NodeID { + log.Info(fmt.Sprintf("uaddr=%s", string(addr.UAddr))) + node := discover.MustParseNode(string(addr.UAddr)) return node.ID } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index acf3479e52..913d6d8373 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -556,8 +556,8 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) { kp.MinProxBinSize = testMinProxBinSize if ctx.Config.Reachable != nil { - kp.Reachable = func(o network.OverlayAddr) bool { - return ctx.Config.Reachable(o.(*network.BzzAddr).ID()) + kp.Reachable = func(o *network.BzzAddr) bool { + return ctx.Config.Reachable(o.ID()) } } kad := network.NewKademlia(addr.Over(), kp) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 36040339d3..6273525355 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -47,15 +47,15 @@ var ( type Delivery struct { db *storage.DBAPI - overlay network.Overlay + kad *network.Kademlia receiveC chan *ChunkDeliveryMsg getPeer func(discover.NodeID) *Peer } -func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery { +func NewDelivery(kad *network.Kademlia, db *storage.DBAPI) *Delivery { d := &Delivery{ db: db, - overlay: overlay, + kad: kad, receiveC: make(chan *ChunkDeliveryMsg, deliveryCap), } @@ -172,7 +172,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * t := time.NewTimer(10 * time.Minute) defer t.Stop() - log.Debug("waiting delivery", "peer", sp.ID(), "hash", req.Addr, "node", common.Bytes2Hex(d.overlay.BaseAddr()), "created", created) + log.Debug("waiting delivery", "peer", sp.ID(), "hash", req.Addr, "node", common.Bytes2Hex(d.kad.BaseAddr()), "created", created) start := time.Now() select { case <-chunk.ReqC: @@ -269,8 +269,8 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, hash []byte, skipCheck var err error requestFromPeersCount.Inc(1) - d.overlay.EachConn(hash, 255, func(p network.OverlayConn, po int, nn bool) bool { - spId := p.(network.Peer).ID() + d.kad.EachConn(hash, 255, func(p *network.Peer, po int, nn bool) bool { + spId := p.ID() for _, p := range peersToSkip { if p == spId { log.Trace("Delivery.RequestFromPeers: skip peer", "peer", spId) diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 4e1ab09fcf..313019d6a1 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -457,15 +457,10 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { //returns the number of subscriptions requested func startSyncing(r *Registry, conf *synctestConfig) (int, error) { var err error - - kad, ok := r.delivery.overlay.(*network.Kademlia) - if !ok { - return 0, fmt.Errorf("Not a Kademlia!") - } - + kad := r.delivery.kad subCnt := 0 //iterate over each bin and solicit needed subscription to bins - kad.EachBin(r.addr.Over(), pof, 0, func(conn network.OverlayConn, po int) bool { + kad.EachBin(r.addr.Over(), pof, 0, func(conn *network.Peer, po int) bool { //identify begin and start index of the bin(s) we want to subscribe to histRange := &Range{} diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index cd0580a0c0..deffdfc3f9 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -130,7 +130,7 @@ func NewRegistry(addr *network.BzzAddr, delivery *Delivery, db *storage.DBAPI, i // wait for kademlia table to be healthy time.Sleep(options.SyncUpdateDelay) - kad := streamer.delivery.overlay.(*network.Kademlia) + kad := streamer.delivery.kad depthC := latestIntC(kad.NeighbourhoodDepthC()) addressBookSizeC := latestIntC(kad.AddrCountC()) @@ -398,9 +398,7 @@ func (r *Registry) Run(p *network.BzzPeer) error { // and they are no longer required after iteration, request to Quit // them will be send to appropriate peers. func (r *Registry) updateSyncing() { - // if overlay in not Kademlia, panic - kad := r.delivery.overlay.(*network.Kademlia) - + kad := r.delivery.kad // map of all SYNC streams for all peers // used at the and of the function to remove servers // that are not needed anymore @@ -421,8 +419,7 @@ func (r *Registry) updateSyncing() { r.peersMu.RUnlock() // request subscriptions for all nodes and bins - kad.EachBin(r.addr.Over(), pot.DefaultPof(256), 0, func(conn network.OverlayConn, bin int) bool { - p := conn.(network.Peer) + kad.EachBin(r.addr.Over(), pot.DefaultPof(256), 0, func(p *network.Peer, bin int) bool { log.Debug(fmt.Sprintf("Requesting subscription by: registry %s from peer %s for bin: %d", r.addr.ID(), p.ID(), bin)) // bin is always less then 256 and it is safe to convert it to type uint8 @@ -461,10 +458,11 @@ func (r *Registry) updateSyncing() { func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { peer := protocols.NewPeer(p, rw, Spec) - bzzPeer := network.NewBzzTestPeer(peer, r.addr) - r.delivery.overlay.On(bzzPeer) - defer r.delivery.overlay.Off(bzzPeer) - return r.Run(bzzPeer) + bp := network.NewBzzPeer(peer, r.addr) + np := network.NewPeer(bp, r.delivery.kad) + r.delivery.kad.On(np) + defer r.delivery.kad.Off(np) + return r.Run(bp) } // HandleMsg is the message handler that delegates incoming messages diff --git a/swarm/network_test.go b/swarm/network_test.go index 176c635d82..9bc6143d84 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -34,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/storage" colorable "github.com/mattn/go-colorable" @@ -293,7 +292,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa if err != nil { return nil, cleanup, err } - bucket.Store(simulation.BucketKeyKademlia, swarm.bzz.Hive.Overlay.(*network.Kademlia)) + bucket.Store(simulation.BucketKeyKademlia, swarm.bzz.Hive.Kademlia) log.Info("new swarm", "bzzKey", config.BzzKey, "baseAddr", fmt.Sprintf("%x", swarm.bzz.BaseAddr())) return swarm, cleanup, nil }, diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 8459211ddb..b55c97fddf 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -110,10 +110,10 @@ func (params *PssParams) WithPrivateKey(privatekey *ecdsa.PrivateKey) *PssParams // // Implements node.Service type Pss struct { - network.Overlay // we can get the overlayaddress from this - privateKey *ecdsa.PrivateKey // pss can have it's own independent key - w *whisper.Whisper // key and encryption backend - auxAPIs []rpc.API // builtins (handshake, test) can add APIs + *network.Kademlia // we can get the Kademlia address from this + privateKey *ecdsa.PrivateKey // pss can have it's own independent key + w *whisper.Whisper // key and encryption backend + auxAPIs []rpc.API // builtins (handshake, test) can add APIs // sending and forwarding fwdPool map[string]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer @@ -151,9 +151,9 @@ func (p *Pss) String() string { // Creates a new Pss instance. // -// In addition to params, it takes a swarm network overlay +// In addition to params, it takes a swarm network Kademlia // and a FileStore storage for message cache storage. -func NewPss(k network.Overlay, params *PssParams) (*Pss, error) { +func NewPss(k *network.Kademlia, params *PssParams) (*Pss, error) { if params.privateKey == nil { return nil, errors.New("missing private key for pss") } @@ -162,7 +162,7 @@ func NewPss(k network.Overlay, params *PssParams) (*Pss, error) { Version: pssVersion, } ps := &Pss{ - Overlay: k, + Kademlia: k, privateKey: params.privateKey, w: whisper.New(&whisper.DefaultConfig), quitC: make(chan struct{}), @@ -290,9 +290,9 @@ func (p *Pss) addAPI(api rpc.API) { p.auxAPIs = append(p.auxAPIs, api) } -// Returns the swarm overlay address of the pss node +// Returns the swarm Kademlia address of the pss node func (p *Pss) BaseAddr() []byte { - return p.Overlay.BaseAddr() + return p.Kademlia.BaseAddr() } // Returns the pss node's public key @@ -356,11 +356,11 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { } if int64(pssmsg.Expire) < time.Now().Unix() { metrics.GetOrRegisterCounter("pss.expire", nil).Inc(1) - log.Warn("pss filtered expired message", "from", common.ToHex(p.Overlay.BaseAddr()), "to", common.ToHex(pssmsg.To)) + log.Warn("pss filtered expired message", "from", common.ToHex(p.Kademlia.BaseAddr()), "to", common.ToHex(pssmsg.To)) return nil } if p.checkFwdCache(pssmsg) { - log.Trace("pss relay block-cache match (process)", "from", common.ToHex(p.Overlay.BaseAddr()), "to", (common.ToHex(pssmsg.To))) + log.Trace("pss relay block-cache match (process)", "from", common.ToHex(p.Kademlia.BaseAddr()), "to", (common.ToHex(pssmsg.To))) return nil } p.addFwdCache(pssmsg) @@ -442,12 +442,12 @@ func (p *Pss) executeHandlers(topic Topic, payload []byte, from *PssAddress, asy // will return false if using partial address func (p *Pss) isSelfRecipient(msg *PssMsg) bool { - return bytes.Equal(msg.To, p.Overlay.BaseAddr()) + return bytes.Equal(msg.To, p.Kademlia.BaseAddr()) } -// test match of leftmost bytes in given message to node's overlay address +// test match of leftmost bytes in given message to node's Kademlia address func (p *Pss) isSelfPossibleRecipient(msg *PssMsg) bool { - local := p.Overlay.BaseAddr() + local := p.Kademlia.BaseAddr() return bytes.Equal(msg.To[:], local[:len(msg.To)]) } @@ -816,14 +816,7 @@ func (p *Pss) forward(msg *PssMsg) error { // send with kademlia // find the closest peer to the recipient and attempt to send sent := 0 - p.Overlay.EachConn(to, 256, func(op network.OverlayConn, po int, isproxbin bool) bool { - // we need p2p.protocols.Peer.Send - // cast and resolve - sp, ok := op.(senderPeer) - if !ok { - log.Crit("Pss cannot use kademlia peer type") - return false - } + p.Kademlia.EachConn(to, 256, func(sp *network.Peer, po int, isproxbin bool) bool { info := sp.Info() // check if the peer is running pss @@ -840,7 +833,7 @@ func (p *Pss) forward(msg *PssMsg) error { } // get the protocol peer from the forwarding peer cache - sendMsg := fmt.Sprintf("MSG TO %x FROM %x VIA %x", to, p.BaseAddr(), op.Address()) + sendMsg := fmt.Sprintf("MSG TO %x FROM %x VIA %x", to, p.BaseAddr(), sp.Address()) p.fwdPoolMu.RLock() pp := p.fwdPool[sp.Info().ID] p.fwdPoolMu.RUnlock() @@ -859,11 +852,11 @@ func (p *Pss) forward(msg *PssMsg) error { // - if the peer is end recipient but the full address has not been disclosed // - if the peer address matches the partial address fully // - if the peer is in proxbin - if len(msg.To) < addressLength && bytes.Equal(msg.To, op.Address()[:len(msg.To)]) { + if len(msg.To) < addressLength && bytes.Equal(msg.To, sp.Address()[:len(msg.To)]) { log.Trace(fmt.Sprintf("Pss keep forwarding: Partial address + full partial match")) return true } else if isproxbin { - log.Trace(fmt.Sprintf("%x is in proxbin, keep forwarding", common.ToHex(op.Address()))) + log.Trace(fmt.Sprintf("%x is in proxbin, keep forwarding", common.ToHex(sp.Address()))) return true } // at this point we stop forwarding, and the state is as follows: diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 41b03db286..6ba04cb5d1 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -556,23 +556,6 @@ OUTER: } } -type pssTestPeer struct { - *protocols.Peer - addr []byte -} - -func (t *pssTestPeer) Address() []byte { - return t.addr -} - -func (t *pssTestPeer) Update(addr network.OverlayAddr) network.OverlayAddr { - return addr -} - -func (t *pssTestPeer) Off() network.OverlayAddr { - return &pssTestPeer{} -} - // forwarding should skip peers that do not have matching pss capabilities func TestMismatch(t *testing.T) { @@ -582,7 +565,7 @@ func TestMismatch(t *testing.T) { t.Fatal(err) } - // initialize overlay + // initialize kad baseaddr := network.RandomAddr() kad := network.NewKademlia((baseaddr).Over(), network.NewKadParams()) rw := &p2p.MsgPipeRW{} @@ -594,10 +577,10 @@ func TestMismatch(t *testing.T) { Version: 0, } nid, _ := discover.HexID("0x01") - wrongpsspeer := &pssTestPeer{ - Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(wrongpssaddr.Over()), []p2p.Cap{wrongpsscap}), rw, nil), - addr: wrongpssaddr.Over(), - } + wrongpsspeer := network.NewPeer(&network.BzzPeer{ + Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(wrongpssaddr.Over()), []p2p.Cap{wrongpsscap}), rw, nil), + BzzAddr: &network.BzzAddr{OAddr: wrongpssaddr.Over(), UAddr: nil}, + }, kad) // one peer doesn't even have pss (boo!) nopssaddr := network.RandomAddr() @@ -606,16 +589,16 @@ func TestMismatch(t *testing.T) { Version: 1, } nid, _ = discover.HexID("0x02") - nopsspeer := &pssTestPeer{ - Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(nopssaddr.Over()), []p2p.Cap{nopsscap}), rw, nil), - addr: nopssaddr.Over(), - } + nopsspeer := network.NewPeer(&network.BzzPeer{ + Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(nopssaddr.Over()), []p2p.Cap{nopsscap}), rw, nil), + BzzAddr: &network.BzzAddr{OAddr: nopssaddr.Over(), UAddr: nil}, + }, kad) // add peers to kademlia and activate them // it's safe so don't check errors - kad.Register([]network.OverlayAddr{wrongpsspeer}) + kad.Register(wrongpsspeer.BzzAddr) kad.On(wrongpsspeer) - kad.Register([]network.OverlayAddr{nopsspeer}) + kad.Register(nopsspeer.BzzAddr) kad.On(nopsspeer) // create pss @@ -1636,17 +1619,17 @@ func newServices(allowRaw bool) adapters.Services { } } -func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *PssParams) *Pss { +func newTestPss(privkey *ecdsa.PrivateKey, kad *network.Kademlia, ppextra *PssParams) *Pss { var nid discover.NodeID copy(nid[:], crypto.FromECDSAPub(&privkey.PublicKey)) addr := network.NewAddrFromNodeID(nid) // set up routing if kademlia is not passed to us - if overlay == nil { + if kad == nil { kp := network.NewKadParams() kp.MinProxBinSize = 3 - overlay = network.NewKademlia(addr.Over(), kp) + kad = network.NewKademlia(addr.Over(), kp) } // create pss @@ -1654,7 +1637,7 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss if ppextra != nil { pp.SymKeyCacheCapacity = ppextra.SymKeyCacheCapacity } - ps, err := NewPss(overlay, pp) + ps, err := NewPss(kad, pp) if err != nil { return nil } diff --git a/swarm/swarm.go b/swarm/swarm.go index 736cd37de8..baf71b9623 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -356,7 +356,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { log.Error("bzz failed", "err", err) return err } - log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", self.bzz.Hive.Overlay.BaseAddr())) + log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", self.bzz.Hive.BaseAddr())) if self.ps != nil { self.ps.Start(srv) From 0732617b655df7f35aff985612af8ba379cdb3fb Mon Sep 17 00:00:00 2001 From: EOS Classic Date: Wed, 12 Sep 2018 19:33:57 +0900 Subject: [PATCH 113/126] consensus: implement Constantinople EIP 1234 --- consensus/ethash/consensus.go | 74 +++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index af0e733ae7..874f098195 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -38,10 +38,11 @@ import ( // Ethash proof-of-work protocol constants. var ( - FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium - maxUncles = 2 // Maximum number of uncles allowed in a single block - allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks + FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople + maxUncles = 2 // Maximum number of uncles allowed in a single block + allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks ) // Various error messages to mark blocks invalid. These should be private to @@ -299,6 +300,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainReader, time uint64, p func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { + case config.IsConstantinople(next): + return calcDifficultyConstantinople(time, parent) case config.IsByzantium(next): return calcDifficultyByzantium(time, parent) case config.IsHomestead(next): @@ -317,8 +320,68 @@ var ( big10 = big.NewInt(10) bigMinus99 = big.NewInt(-99) big2999999 = big.NewInt(2999999) + big4999999 = big.NewInt(4999999) ) +// calcDifficultyConstantinople is the difficulty adjustment algorithm. It returns +// the difficulty that a new block should have when created at time given the +// parent block's time and difficulty. The calculation uses the Constantinople rules. +func calcDifficultyConstantinople(time uint64, parent *types.Header) *big.Int { + // https://github.com/ethereum/EIPs/issues/100. + // algorithm: + // diff = (parent_diff + + // (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) + // ) + 2^(periodCount - 2) + + bigTime := new(big.Int).SetUint64(time) + bigParentTime := new(big.Int).Set(parent.Time) + + // holds intermediate values to make the algo easier to read & audit + x := new(big.Int) + y := new(big.Int) + + // (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9 + x.Sub(bigTime, bigParentTime) + x.Div(x, big9) + if parent.UncleHash == types.EmptyUncleHash { + x.Sub(big1, x) + } else { + x.Sub(big2, x) + } + // max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99) + if x.Cmp(bigMinus99) < 0 { + x.Set(bigMinus99) + } + // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) + y.Div(parent.Difficulty, params.DifficultyBoundDivisor) + x.Mul(y, x) + x.Add(parent.Difficulty, x) + + // minimum difficulty can ever be (before exponential factor) + if x.Cmp(params.MinimumDifficulty) < 0 { + x.Set(params.MinimumDifficulty) + } + // calculate a fake block number for the ice-age delay: + // https://github.com/ethereum/EIPs/pull/1234 + // fake_block_number = max(0, block.number - 5_000_000) + fakeBlockNumber := new(big.Int) + if parent.Number.Cmp(big4999999) >= 0 { + fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, big4999999) // Note, parent is 1 less than the actual block number + } + // for the exponential factor + periodCount := fakeBlockNumber + periodCount.Div(periodCount, expDiffPeriod) + + // the exponential factor, commonly referred to as "the bomb" + // diff = diff + 2^(periodCount - 2) + if periodCount.Cmp(big1) > 0 { + y.Sub(periodCount, big2) + y.Exp(big2, y, nil) + x.Add(x, y) + } + return x +} + // calcDifficultyByzantium is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time given the // parent block's time and difficulty. The calculation uses the Byzantium rules. @@ -592,6 +655,9 @@ func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header if config.IsByzantium(header.Number) { blockReward = ByzantiumBlockReward } + if config.IsConstantinople(header.Number) { + blockReward = ConstantinopleBlockReward + } // Accumulate the rewards for the miner and any included uncles reward := new(big.Int).Set(blockReward) r := new(big.Int) From ff3a5d24d2e40fd66f7813173e9cfc31144f3c53 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 12 Sep 2018 14:39:45 +0200 Subject: [PATCH 114/126] swarm/storage: remove redundant increments for dataIdx and entryCnt (#17484) * swarm/storage: remove redundant increments for dataIdx and entryCnt * swarm/storage: add Delete to LDBStore * swarm/storage: wait for garbage collection --- swarm/storage/ldbstore.go | 25 ++++++++++++++++----- swarm/storage/ldbstore_test.go | 41 +++++++++++++--------------------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index bd3501ea26..675b5de01a 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -147,13 +147,10 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { } data, _ := s.db.Get(keyEntryCnt) s.entryCnt = BytesToU64(data) - s.entryCnt++ data, _ = s.db.Get(keyAccessCnt) s.accessCnt = BytesToU64(data) - s.accessCnt++ data, _ = s.db.Get(keyDataIdx) s.dataIdx = BytesToU64(data) - s.dataIdx++ return s, nil } @@ -251,6 +248,8 @@ func decodeOldData(data []byte, chunk *Chunk) { } func (s *LDBStore) collectGarbage(ratio float32) { + log.Trace("collectGarbage", "ratio", ratio) + metrics.GetOrRegisterCounter("ldbstore.collectgarbage", nil).Inc(1) it := s.db.NewIterator() @@ -492,6 +491,18 @@ func (s *LDBStore) ReIndex() { log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) } +func (s *LDBStore) Delete(addr Address) { + s.lock.Lock() + defer s.lock.Unlock() + + ikey := getIndexKey(addr) + + var indx dpaDBIndex + s.tryAccessIdx(ikey, &indx) + + s.delete(indx.Idx, ikey, s.po(addr)) +} + func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { metrics.GetOrRegisterCounter("ldbstore.delete", nil).Inc(1) @@ -517,8 +528,8 @@ func (s *LDBStore) CurrentBucketStorageIndex(po uint8) uint64 { } func (s *LDBStore) Size() uint64 { - s.lock.Lock() - defer s.lock.Unlock() + s.lock.RLock() + defer s.lock.RUnlock() return s.entryCnt } @@ -602,21 +613,23 @@ mainLoop: } close(c) for e > s.capacity { + log.Trace("for >", "e", e, "s.capacity", s.capacity) // Collect garbage in a separate goroutine // to be able to interrupt this loop by s.quit. done := make(chan struct{}) go func() { s.collectGarbage(gcArrayFreeRatio) + log.Trace("collectGarbage closing done") close(done) }() - e = s.entryCnt select { case <-s.quit: s.lock.Unlock() break mainLoop case <-done: } + e = s.entryCnt } s.lock.Unlock() } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 5ee88baa51..9694a724e4 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -27,8 +27,8 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" ldberrors "github.com/syndtr/goleveldb/leveldb/errors" @@ -209,7 +209,7 @@ func testIterator(t *testing.T, mock bool) { for poc = 0; poc <= 255; poc++ { err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Address, n uint64) bool { log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc))) - chunkkeys_results[n-1] = k + chunkkeys_results[n] = k i++ return true }) @@ -324,12 +324,12 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { log.Info("got back chunk", "chunk", ret) } - if ldb.entryCnt != uint64(n+1) { - t.Fatalf("expected entryCnt to be equal to %v, but got %v", n+1, ldb.entryCnt) + if ldb.entryCnt != uint64(n) { + t.Fatalf("expected entryCnt to be equal to %v, but got %v", n, ldb.entryCnt) } - if ldb.accessCnt != uint64(2*n+1) { - t.Fatalf("expected accessCnt to be equal to %v, but got %v", n+1, ldb.accessCnt) + if ldb.accessCnt != uint64(2*n) { + t.Fatalf("expected accessCnt to be equal to %v, but got %v", 2*n, ldb.accessCnt) } } @@ -362,7 +362,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) { log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) // wait for garbage collection to kick in on the responsible actor - time.Sleep(5 * time.Second) + time.Sleep(1 * time.Second) var missing int for i := 0; i < n; i++ { @@ -416,14 +416,7 @@ func TestLDBStoreAddRemove(t *testing.T) { for i := 0; i < n; i++ { // delete all even index chunks if i%2 == 0 { - - key := chunks[i].Addr - ikey := getIndexKey(key) - - var indx dpaDBIndex - ldb.tryAccessIdx(ikey, &indx) - - ldb.delete(indx.Idx, ikey, ldb.po(key)) + ldb.Delete(chunks[i].Addr) } } @@ -452,12 +445,12 @@ func TestLDBStoreAddRemove(t *testing.T) { // TestLDBStoreRemoveThenCollectGarbage tests that we can delete chunks and that we can trigger garbage collection func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { - capacity := 10 + capacity := 11 ldb, cleanup := newLDBStore(t) ldb.setCapacity(uint64(capacity)) - n := 7 + n := 11 chunks := []*Chunk{} for i := 0; i < capacity; i++ { @@ -477,13 +470,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { // delete all chunks for i := 0; i < n; i++ { - key := chunks[i].Addr - ikey := getIndexKey(key) - - var indx dpaDBIndex - ldb.tryAccessIdx(ikey, &indx) - - ldb.delete(indx.Idx, ikey, ldb.po(key)) + ldb.Delete(chunks[i].Addr) } log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) @@ -491,9 +478,10 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { cleanup() ldb, cleanup = newLDBStore(t) + capacity = 10 ldb.setCapacity(uint64(capacity)) - n = 10 + n = 11 for i := 0; i < n; i++ { ldb.Put(context.TODO(), chunks[i]) @@ -504,6 +492,9 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { <-chunks[i].dbStoredC } + // wait for garbage collection + time.Sleep(1 * time.Second) + // expect for first chunk to be missing, because it has the smallest access value idx := 0 ret, err := ldb.Get(context.TODO(), chunks[idx].Addr) From 3ff2f756368f2d8ec0d1d9d25f6ba9cdabd7383e Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 13 Sep 2018 11:42:19 +0200 Subject: [PATCH 115/126] swarm: Chunk refactor (#17659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Janos Guljas Co-authored-by: Balint Gabor Co-authored-by: Anton Evangelatov Co-authored-by: Viktor Trón --- cmd/swarm/hash.go | 2 +- cmd/swarm/swarm-smoke/main.go | 6 +- cmd/swarm/swarm-smoke/upload_and_sync.go | 18 +- swarm/api/api.go | 7 - swarm/api/config.go | 4 +- swarm/api/http/server_test.go | 6 +- swarm/api/manifest.go | 66 +- swarm/fuse/swarmfs_test.go | 9 +- swarm/network/fetcher.go | 305 ++++++++ swarm/network/fetcher_test.go | 459 ++++++++++++ swarm/network/priorityqueue/priorityqueue.go | 38 +- .../priorityqueue/priorityqueue_test.go | 6 +- swarm/network/simulation/simulation.go | 16 +- swarm/network/stream/common_test.go | 17 +- swarm/network/stream/delivery.go | 237 +++---- swarm/network/stream/delivery_test.go | 93 +-- swarm/network/stream/intervals_test.go | 78 ++- swarm/network/stream/messages.go | 87 +-- swarm/network/stream/peer.go | 51 +- .../network/stream/snapshot_retrieval_test.go | 54 +- swarm/network/stream/snapshot_sync_test.go | 170 +++-- swarm/network/stream/stream.go | 28 +- swarm/network/stream/streamer_test.go | 8 +- swarm/network/stream/syncer.go | 94 ++- swarm/network/stream/syncer_test.go | 32 +- swarm/network_test.go | 9 +- swarm/storage/chunker.go | 102 ++- swarm/storage/chunker_test.go | 124 ++-- swarm/storage/chunkstore.go | 69 -- swarm/storage/common.go | 44 -- swarm/storage/common_test.go | 202 ++++-- swarm/storage/dbapi.go | 54 -- swarm/storage/filestore_test.go | 34 +- swarm/storage/hasherstore.go | 120 ++-- swarm/storage/hasherstore_test.go | 20 +- swarm/storage/ldbstore.go | 318 +++++---- swarm/storage/ldbstore_test.go | 214 +++--- swarm/storage/localstore.go | 114 ++- swarm/storage/localstore_test.go | 68 +- swarm/storage/memstore.go | 85 +-- swarm/storage/memstore_test.go | 133 +--- swarm/storage/mru/handler.go | 47 +- swarm/storage/mru/lookup.go | 6 +- swarm/storage/mru/metadata.go | 6 +- swarm/storage/mru/request.go | 2 +- swarm/storage/mru/resource_test.go | 46 +- swarm/storage/mru/signedupdate.go | 9 +- swarm/storage/mru/testutil.go | 21 +- swarm/storage/mru/updateheader.go | 4 +- swarm/storage/netstore.go | 398 +++++++---- swarm/storage/netstore_test.go | 651 ++++++++++++++++-- swarm/storage/pyramid.go | 77 ++- swarm/storage/types.go | 193 +++--- swarm/swarm.go | 50 +- swarm/swarm_test.go | 7 +- 55 files changed, 3193 insertions(+), 1925 deletions(-) create mode 100644 swarm/network/fetcher.go create mode 100644 swarm/network/fetcher_test.go delete mode 100644 swarm/storage/chunkstore.go delete mode 100644 swarm/storage/common.go delete mode 100644 swarm/storage/dbapi.go diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index bca4955b15..d679806e38 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -39,7 +39,7 @@ func hash(ctx *cli.Context) { defer f.Close() stat, _ := f.Stat() - fileStore := storage.NewFileStore(storage.NewMapChunkStore(), storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false) if err != nil { utils.Fatalf("%v\n", err) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 87bc39816d..70aee19229 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -48,7 +48,7 @@ func main() { cli.StringFlag{ Name: "cluster-endpoint", Value: "testing", - Usage: "cluster to point to (open, or testing)", + Usage: "cluster to point to (local, open or testing)", Destination: &cluster, }, cli.IntFlag{ @@ -76,8 +76,8 @@ func main() { }, cli.IntFlag{ Name: "filesize", - Value: 1, - Usage: "file size for generated random file in MB", + Value: 1024, + Usage: "file size for generated random file in KB", Destination: &filesize, }, } diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index d5300b63d9..5e0ff4b0f3 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -39,6 +39,11 @@ import ( func generateEndpoints(scheme string, cluster string, from int, to int) { if cluster == "prod" { cluster = "" + } else if cluster == "local" { + for port := from; port <= to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port)) + } + return } else { cluster = cluster + "." } @@ -53,13 +58,13 @@ func generateEndpoints(scheme string, cluster string, from int, to int) { } func cliUploadAndSync(c *cli.Context) error { - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size", filesize) }(time.Now()) + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, from, to) log.Info("uploading to " + endpoints[0] + " and syncing") - f, cleanup := generateRandomFile(filesize * 1000000) + f, cleanup := generateRandomFile(filesize * 1000) defer cleanup() hash, err := upload(f, endpoints[0]) @@ -76,12 +81,7 @@ func cliUploadAndSync(c *cli.Context) error { log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) - if filesize < 10 { - time.Sleep(35 * time.Second) - } else { - time.Sleep(15 * time.Second) - time.Sleep(2 * time.Duration(filesize) * time.Second) - } + time.Sleep(3 * time.Second) wg := sync.WaitGroup{} for _, endpoint := range endpoints { @@ -109,7 +109,7 @@ func cliUploadAndSync(c *cli.Context) error { // fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file func fetch(hash string, endpoint string, original []byte, ruid string) error { log.Trace("sleeping", "ruid", ruid) - time.Sleep(5 * time.Second) + time.Sleep(3 * time.Second) log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) res, err := http.Get(endpoint + "/bzz:/" + hash + "/") diff --git a/swarm/api/api.go b/swarm/api/api.go index d733ad989c..d7b6d84190 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -250,13 +250,6 @@ func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Han return } -// Upload to be used only in TEST -func (a *API) Upload(ctx context.Context, uploadDir, index string, toEncrypt bool) (hash string, err error) { - fs := NewFileSystem(a) - hash, err = fs.Upload(uploadDir, index, toEncrypt) - return hash, err -} - // Retrieve FileStore reader API func (a *API) Retrieve(ctx context.Context, addr storage.Address) (reader storage.LazySectionReader, isEncrypted bool) { return a.fileStore.Retrieve(ctx, addr) diff --git a/swarm/api/config.go b/swarm/api/config.go index 3044dc2e52..baa13105a6 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -62,6 +62,7 @@ type Config struct { NetworkID uint64 SwapEnabled bool SyncEnabled bool + SyncingSkipCheck bool DeliverySkipCheck bool LightNodeEnabled bool SyncUpdateDelay time.Duration @@ -89,7 +90,8 @@ func NewConfig() (c *Config) { NetworkID: network.DefaultNetworkID, SwapEnabled: false, SyncEnabled: true, - DeliverySkipCheck: false, + SyncingSkipCheck: false, + DeliverySkipCheck: true, SyncUpdateDelay: 15 * time.Second, SwapAPI: "", } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index efefa9fae4..2acdbd1ad4 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -477,12 +477,12 @@ func testBzzGetPath(encrypted bool, t *testing.T) { var wait func(context.Context) error ctx := context.TODO() addr[i], wait, err = srv.FileStore.Store(ctx, reader[i], int64(len(mf)), encrypted) - for j := i + 1; j < len(testmanifest); j++ { - testmanifest[j] = strings.Replace(testmanifest[j], fmt.Sprintf("", i), addr[i].Hex(), -1) - } if err != nil { t.Fatal(err) } + for j := i + 1; j < len(testmanifest); j++ { + testmanifest[j] = strings.Replace(testmanifest[j], fmt.Sprintf("", i), addr[i].Hex(), -1) + } err = wait(ctx) if err != nil { t.Fatal(err) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index a1329a800f..d44ad2277c 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -69,9 +69,12 @@ func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, if err != nil { return nil, err } - key, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt) - wait(ctx) - return key, err + addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt) + if err != nil { + return nil, err + } + err = wait(ctx) + return addr, err } // Manifest hack for supporting Mutable Resource Updates from the bzz: scheme @@ -87,8 +90,12 @@ func (a *API) NewResourceManifest(ctx context.Context, resourceAddr string) (sto if err != nil { return nil, err } - key, _, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false) - return key, err + addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false) + if err != nil { + return nil, err + } + err = wait(ctx) + return addr, err } // ManifestWriter is used to add and remove entries from an underlying manifest @@ -106,21 +113,26 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC return &ManifestWriter{a, trie, quitC}, nil } -// AddEntry stores the given data and adds the resulting key to the manifest -func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (key storage.Address, err error) { +// AddEntry stores the given data and adds the resulting address to the manifest +func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) { entry := newManifestTrieEntry(e, nil) if data != nil { - key, _, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) + var wait func(context.Context) error + addr, wait, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) if err != nil { return nil, err } - entry.Hash = key.Hex() + err = wait(ctx) + if err != nil { + return nil, err + } + entry.Hash = addr.Hex() } if entry.Hash == "" { - return key, errors.New("missing entry hash") + return addr, errors.New("missing entry hash") } m.trie.addEntry(entry, m.quitC) - return key, nil + return addr, nil } // RemoveEntry removes the given path from the manifest @@ -129,7 +141,7 @@ func (m *ManifestWriter) RemoveEntry(path string) error { return nil } -// Store stores the manifest, returning the resulting storage key +// Store stores the manifest, returning the resulting storage address func (m *ManifestWriter) Store() (storage.Address, error) { return m.trie.ref, m.trie.recalcAndStore() } @@ -211,51 +223,51 @@ type manifestTrieEntry struct { subtrie *manifestTrie } -func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand - log.Trace("manifest lookup", "key", hash) +func loadManifest(ctx context.Context, fileStore *storage.FileStore, addr storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand + log.Trace("manifest lookup", "addr", addr) // retrieve manifest via FileStore - manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash) - log.Trace("reader retrieved", "key", hash) - return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC, decrypt) + manifestReader, isEncrypted := fileStore.Retrieve(ctx, addr) + log.Trace("reader retrieved", "addr", addr) + return readManifest(manifestReader, addr, fileStore, isEncrypted, quitC, decrypt) } -func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(mr storage.LazySectionReader, addr storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests size, err := mr.Size(mr.Context(), quitC) if err != nil { // size == 0 // can't determine size means we don't have the root chunk - log.Trace("manifest not found", "key", hash) + log.Trace("manifest not found", "addr", addr) err = fmt.Errorf("Manifest not Found") return } if size > manifestSizeLimit { - log.Warn("manifest exceeds size limit", "key", hash, "size", size, "limit", manifestSizeLimit) + log.Warn("manifest exceeds size limit", "addr", addr, "size", size, "limit", manifestSizeLimit) err = fmt.Errorf("Manifest size of %v bytes exceeds the %v byte limit", size, manifestSizeLimit) return } manifestData := make([]byte, size) read, err := mr.Read(manifestData) if int64(read) < size { - log.Trace("manifest not found", "key", hash) + log.Trace("manifest not found", "addr", addr) if err == nil { err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size) } return } - log.Debug("manifest retrieved", "key", hash) + log.Debug("manifest retrieved", "addr", addr) var man struct { Entries []*manifestTrieEntry `json:"entries"` } err = json.Unmarshal(manifestData, &man) if err != nil { - err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err) - log.Trace("malformed manifest", "key", hash) + err = fmt.Errorf("Manifest %v is malformed: %v", addr.Log(), err) + log.Trace("malformed manifest", "addr", addr) return } - log.Trace("manifest entries", "key", hash, "len", len(man.Entries)) + log.Trace("manifest entries", "addr", addr, "len", len(man.Entries)) trie = &manifestTrie{ fileStore: fileStore, @@ -406,12 +418,12 @@ func (mt *manifestTrie) recalcAndStore() error { sr := bytes.NewReader(manifest) ctx := context.TODO() - key, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted) + addr, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted) if err2 != nil { return err2 } err2 = wait(ctx) - mt.ref = key + mt.ref = addr return err2 } diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 6efeb78d9b..87d9185505 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -20,7 +20,6 @@ package fuse import ( "bytes" - "context" "crypto/rand" "flag" "fmt" @@ -111,7 +110,7 @@ func createTestFilesAndUploadToSwarm(t *testing.T, api *api.API, files map[strin } //upload directory to swarm and return hash - bzzhash, err := api.Upload(context.TODO(), uploadDir, "", toEncrypt) + bzzhash, err := Upload(uploadDir, "", api, toEncrypt) if err != nil { t.Fatalf("Error uploading directory %v: %vm encryption: %v", uploadDir, err, toEncrypt) } @@ -1695,3 +1694,9 @@ func TestFUSE(t *testing.T) { t.Run("appendFileContentsToEndNonEncrypted", ta.appendFileContentsToEndNonEncrypted) } } + +func Upload(uploadDir, index string, a *api.API, toEncrypt bool) (hash string, err error) { + fs := api.NewFileSystem(a) + hash, err = fs.Upload(uploadDir, index, toEncrypt) + return hash, err +} diff --git a/swarm/network/fetcher.go b/swarm/network/fetcher.go new file mode 100644 index 0000000000..35e2f01328 --- /dev/null +++ b/swarm/network/fetcher.go @@ -0,0 +1,305 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "context" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var searchTimeout = 1 * time.Second + +// Time to consider peer to be skipped. +// Also used in stream delivery. +var RequestTimeout = 10 * time.Second + +type RequestFunc func(context.Context, *Request) (*discover.NodeID, chan struct{}, error) + +// Fetcher is created when a chunk is not found locally. It starts a request handler loop once and +// keeps it alive until all active requests are completed. This can happen: +// 1. either because the chunk is delivered +// 2. or becuse the requestor cancelled/timed out +// Fetcher self destroys itself after it is completed. +// TODO: cancel all forward requests after termination +type Fetcher struct { + protoRequestFunc RequestFunc // request function fetcher calls to issue retrieve request for a chunk + addr storage.Address // the address of the chunk to be fetched + offerC chan *discover.NodeID // channel of sources (peer node id strings) + requestC chan struct{} + skipCheck bool +} + +type Request struct { + Addr storage.Address // chunk address + Source *discover.NodeID // nodeID of peer to request from (can be nil) + SkipCheck bool // whether to offer the chunk first or deliver directly + peersToSkip *sync.Map // peers not to request chunk from (only makes sense if source is nil) +} + +// NewRequest returns a new instance of Request based on chunk address skip check and +// a map of peers to skip. +func NewRequest(addr storage.Address, skipCheck bool, peersToSkip *sync.Map) *Request { + return &Request{ + Addr: addr, + SkipCheck: skipCheck, + peersToSkip: peersToSkip, + } +} + +// SkipPeer returns if the peer with nodeID should not be requested to deliver a chunk. +// Peers to skip are kept per Request and for a time period of RequestTimeout. +// This function is used in stream package in Delivery.RequestFromPeers to optimize +// requests for chunks. +func (r *Request) SkipPeer(nodeID string) bool { + val, ok := r.peersToSkip.Load(nodeID) + if !ok { + return false + } + t, ok := val.(time.Time) + if ok && time.Now().After(t.Add(RequestTimeout)) { + // deadine expired + r.peersToSkip.Delete(nodeID) + return false + } + return true +} + +// FetcherFactory is initialised with a request function and can create fetchers +type FetcherFactory struct { + request RequestFunc + skipCheck bool +} + +// NewFetcherFactory takes a request function and skip check parameter and creates a FetcherFactory +func NewFetcherFactory(request RequestFunc, skipCheck bool) *FetcherFactory { + return &FetcherFactory{ + request: request, + skipCheck: skipCheck, + } +} + +// New contructs a new Fetcher, for the given chunk. All peers in peersToSkip are not requested to +// deliver the given chunk. peersToSkip should always contain the peers which are actively requesting +// this chunk, to make sure we don't request back the chunks from them. +// The created Fetcher is started and returned. +func (f *FetcherFactory) New(ctx context.Context, source storage.Address, peersToSkip *sync.Map) storage.NetFetcher { + fetcher := NewFetcher(source, f.request, f.skipCheck) + go fetcher.run(ctx, peersToSkip) + return fetcher +} + +// NewFetcher creates a new Fetcher for the given chunk address using the given request function. +func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher { + return &Fetcher{ + addr: addr, + protoRequestFunc: rf, + offerC: make(chan *discover.NodeID), + requestC: make(chan struct{}), + skipCheck: skipCheck, + } +} + +// Offer is called when an upstream peer offers the chunk via syncing as part of `OfferedHashesMsg` and the node does not have the chunk locally. +func (f *Fetcher) Offer(ctx context.Context, source *discover.NodeID) { + // First we need to have this select to make sure that we return if context is done + select { + case <-ctx.Done(): + return + default: + } + + // This select alone would not guarantee that we return of context is done, it could potentially + // push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements) + select { + case f.offerC <- source: + case <-ctx.Done(): + } +} + +// Request is called when an upstream peer request the chunk as part of `RetrieveRequestMsg`, or from a local request through FileStore, and the node does not have the chunk locally. +func (f *Fetcher) Request(ctx context.Context) { + // First we need to have this select to make sure that we return if context is done + select { + case <-ctx.Done(): + return + default: + } + + // This select alone would not guarantee that we return of context is done, it could potentially + // push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements) + select { + case f.requestC <- struct{}{}: + case <-ctx.Done(): + } +} + +// start prepares the Fetcher +// it keeps the Fetcher alive within the lifecycle of the passed context +func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { + var ( + doRequest bool // determines if retrieval is initiated in the current iteration + wait *time.Timer // timer for search timeout + waitC <-chan time.Time // timer channel + sources []*discover.NodeID // known sources, ie. peers that offered the chunk + requested bool // true if the chunk was actually requested + ) + gone := make(chan *discover.NodeID) // channel to signal that a peer we requested from disconnected + + // loop that keeps the fetching process alive + // after every request a timer is set. If this goes off we request again from another peer + // note that the previous request is still alive and has the chance to deliver, so + // rerequesting extends the search. ie., + // if a peer we requested from is gone we issue a new request, so the number of active + // requests never decreases + for { + select { + + // incoming offer + case source := <-f.offerC: + log.Trace("new source", "peer addr", source, "request addr", f.addr) + // 1) the chunk is offered by a syncing peer + // add to known sources + sources = append(sources, source) + // launch a request to the source iff the chunk was requested (not just expected because its offered by a syncing peer) + doRequest = requested + + // incoming request + case <-f.requestC: + log.Trace("new request", "request addr", f.addr) + // 2) chunk is requested, set requested flag + // launch a request iff none been launched yet + doRequest = !requested + requested = true + + // peer we requested from is gone. fall back to another + // and remove the peer from the peers map + case id := <-gone: + log.Trace("peer gone", "peer id", id.String(), "request addr", f.addr) + peers.Delete(id.String()) + doRequest = requested + + // search timeout: too much time passed since the last request, + // extend the search to a new peer if we can find one + case <-waitC: + log.Trace("search timed out: rerequesting", "request addr", f.addr) + doRequest = requested + + // all Fetcher context closed, can quit + case <-ctx.Done(): + log.Trace("terminate fetcher", "request addr", f.addr) + // TODO: send cancelations to all peers left over in peers map (i.e., those we requested from) + return + } + + // need to issue a new request + if doRequest { + var err error + sources, err = f.doRequest(ctx, gone, peers, sources) + if err != nil { + log.Warn("unable to request", "request addr", f.addr, "err", err) + } + } + + // if wait channel is not set, set it to a timer + if requested { + if wait == nil { + wait = time.NewTimer(searchTimeout) + defer wait.Stop() + waitC = wait.C + } else { + // stop the timer and drain the channel if it was not drained earlier + if !wait.Stop() { + select { + case <-wait.C: + default: + } + } + // reset the timer to go off after searchTimeout + wait.Reset(searchTimeout) + } + } + doRequest = false + } +} + +// doRequest attempts at finding a peer to request the chunk from +// * first it tries to request explicitly from peers that are known to have offered the chunk +// * if there are no such peers (available) it tries to request it from a peer closest to the chunk address +// excluding those in the peersToSkip map +// * if no such peer is found an error is returned +// +// if a request is successful, +// * the peer's address is added to the set of peers to skip +// * the peer's address is removed from prospective sources, and +// * a go routine is started that reports on the gone channel if the peer is disconnected (or terminated their streamer) +func (f *Fetcher) doRequest(ctx context.Context, gone chan *discover.NodeID, peersToSkip *sync.Map, sources []*discover.NodeID) ([]*discover.NodeID, error) { + var i int + var sourceID *discover.NodeID + var quit chan struct{} + + req := &Request{ + Addr: f.addr, + SkipCheck: f.skipCheck, + peersToSkip: peersToSkip, + } + + foundSource := false + // iterate over known sources + for i = 0; i < len(sources); i++ { + req.Source = sources[i] + var err error + sourceID, quit, err = f.protoRequestFunc(ctx, req) + if err == nil { + // remove the peer from known sources + // Note: we can modify the source although we are looping on it, because we break from the loop immediately + sources = append(sources[:i], sources[i+1:]...) + foundSource = true + break + } + } + + // if there are no known sources, or none available, we try request from a closest node + if !foundSource { + req.Source = nil + var err error + sourceID, quit, err = f.protoRequestFunc(ctx, req) + if err != nil { + // if no peers found to request from + return sources, err + } + } + // add peer to the set of peers to skip from now + peersToSkip.Store(sourceID.String(), time.Now()) + + // if the quit channel is closed, it indicates that the source peer we requested from + // disconnected or terminated its streamer + // here start a go routine that watches this channel and reports the source peer on the gone channel + // this go routine quits if the fetcher global context is done to prevent process leak + go func() { + select { + case <-quit: + gone <- sourceID + case <-ctx.Done(): + } + }() + return sources, nil +} diff --git a/swarm/network/fetcher_test.go b/swarm/network/fetcher_test.go new file mode 100644 index 0000000000..21b81d652c --- /dev/null +++ b/swarm/network/fetcher_test.go @@ -0,0 +1,459 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/p2p/discover" +) + +var requestedPeerID = discover.MustHexID("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439") +var sourcePeerID = discover.MustHexID("2dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439") + +// mockRequester pushes every request to the requestC channel when its doRequest function is called +type mockRequester struct { + // requests []Request + requestC chan *Request // when a request is coming it is pushed to requestC + waitTimes []time.Duration // with waitTimes[i] you can define how much to wait on the ith request (optional) + ctr int //counts the number of requests + quitC chan struct{} +} + +func newMockRequester(waitTimes ...time.Duration) *mockRequester { + return &mockRequester{ + requestC: make(chan *Request), + waitTimes: waitTimes, + quitC: make(chan struct{}), + } +} + +func (m *mockRequester) doRequest(ctx context.Context, request *Request) (*discover.NodeID, chan struct{}, error) { + waitTime := time.Duration(0) + if m.ctr < len(m.waitTimes) { + waitTime = m.waitTimes[m.ctr] + m.ctr++ + } + time.Sleep(waitTime) + m.requestC <- request + + // if there is a Source in the request use that, if not use the global requestedPeerId + source := request.Source + if source == nil { + source = &requestedPeerID + } + return source, m.quitC, nil +} + +// TestFetcherSingleRequest creates a Fetcher using mockRequester, and run it with a sample set of peers to skip. +// mockRequester pushes a Request on a channel every time the request function is called. Using +// this channel we test if calling Fetcher.Request calls the request function, and whether it uses +// the correct peers to skip which we provided for the fetcher.run function. +func TestFetcherSingleRequest(t *testing.T) { + requester := newMockRequester() + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + peers := []string{"a", "b", "c", "d"} + peersToSkip := &sync.Map{} + for _, p := range peers { + peersToSkip.Store(p, time.Now()) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go fetcher.run(ctx, peersToSkip) + + rctx := context.Background() + fetcher.Request(rctx) + + select { + case request := <-requester.requestC: + // request should contain all peers from peersToSkip provided to the fetcher + for _, p := range peers { + if _, ok := request.peersToSkip.Load(p); !ok { + t.Fatalf("request.peersToSkip misses peer") + } + } + + // source peer should be also added to peersToSkip eventually + time.Sleep(100 * time.Millisecond) + if _, ok := request.peersToSkip.Load(requestedPeerID.String()); !ok { + t.Fatalf("request.peersToSkip does not contain peer returned by the request function") + } + + // fetch should trigger a request, if it doesn't happen in time, test should fail + case <-time.After(200 * time.Millisecond): + t.Fatalf("fetch timeout") + } +} + +// TestCancelStopsFetcher tests that a cancelled fetcher does not initiate further requests even if its fetch function is called +func TestFetcherCancelStopsFetcher(t *testing.T) { + requester := newMockRequester() + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + + ctx, cancel := context.WithCancel(context.Background()) + + // we start the fetcher, and then we immediately cancel the context + go fetcher.run(ctx, peersToSkip) + cancel() + + rctx, rcancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer rcancel() + // we call Request with an active context + fetcher.Request(rctx) + + // fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening + select { + case <-requester.requestC: + t.Fatalf("cancelled fetcher initiated request") + case <-time.After(200 * time.Millisecond): + } +} + +// TestFetchCancelStopsRequest tests that calling a Request function with a cancelled context does not initiate a request +func TestFetcherCancelStopsRequest(t *testing.T) { + requester := newMockRequester(100 * time.Millisecond) + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // we start the fetcher with an active context + go fetcher.run(ctx, peersToSkip) + + rctx, rcancel := context.WithCancel(context.Background()) + rcancel() + + // we call Request with a cancelled context + fetcher.Request(rctx) + + // fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening + select { + case <-requester.requestC: + t.Fatalf("cancelled fetch function initiated request") + case <-time.After(200 * time.Millisecond): + } + + // if there is another Request with active context, there should be a request, because the fetcher itself is not cancelled + rctx = context.Background() + fetcher.Request(rctx) + + select { + case <-requester.requestC: + case <-time.After(200 * time.Millisecond): + t.Fatalf("expected request") + } +} + +// TestOfferUsesSource tests Fetcher Offer behavior. +// In this case there should be 1 (and only one) request initiated from the source peer, and the +// source nodeid should appear in the peersToSkip map. +func TestFetcherOfferUsesSource(t *testing.T) { + requester := newMockRequester(100 * time.Millisecond) + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // start the fetcher + go fetcher.run(ctx, peersToSkip) + + rctx := context.Background() + // call the Offer function with the source peer + fetcher.Offer(rctx, &sourcePeerID) + + // fetcher should not initiate request + select { + case <-requester.requestC: + t.Fatalf("fetcher initiated request") + case <-time.After(200 * time.Millisecond): + } + + // call Request after the Offer + rctx = context.Background() + fetcher.Request(rctx) + + // there should be exactly 1 request coming from fetcher + var request *Request + select { + case request = <-requester.requestC: + if *request.Source != sourcePeerID { + t.Fatalf("Expected source id %v got %v", sourcePeerID, request.Source) + } + case <-time.After(200 * time.Millisecond): + t.Fatalf("fetcher did not initiate request") + } + + select { + case <-requester.requestC: + t.Fatalf("Fetcher number of requests expected 1 got 2") + case <-time.After(200 * time.Millisecond): + } + + // source peer should be added to peersToSkip eventually + time.Sleep(100 * time.Millisecond) + if _, ok := request.peersToSkip.Load(sourcePeerID.String()); !ok { + t.Fatalf("SourcePeerId not added to peersToSkip") + } +} + +func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) { + requester := newMockRequester(100 * time.Millisecond) + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // start the fetcher + go fetcher.run(ctx, peersToSkip) + + // call Request first + rctx := context.Background() + fetcher.Request(rctx) + + // there should be a request coming from fetcher + var request *Request + select { + case request = <-requester.requestC: + if request.Source != nil { + t.Fatalf("Incorrect source peer id, expected nil got %v", request.Source) + } + case <-time.After(200 * time.Millisecond): + t.Fatalf("fetcher did not initiate request") + } + + // after the Request call Offer + fetcher.Offer(context.Background(), &sourcePeerID) + + // there should be a request coming from fetcher + select { + case request = <-requester.requestC: + if *request.Source != sourcePeerID { + t.Fatalf("Incorrect source peer id, expected %v got %v", sourcePeerID, request.Source) + } + case <-time.After(200 * time.Millisecond): + t.Fatalf("fetcher did not initiate request") + } + + // source peer should be added to peersToSkip eventually + time.Sleep(100 * time.Millisecond) + if _, ok := request.peersToSkip.Load(sourcePeerID.String()); !ok { + t.Fatalf("SourcePeerId not added to peersToSkip") + } +} + +// TestFetcherRetryOnTimeout tests that fetch retries after searchTimeOut has passed +func TestFetcherRetryOnTimeout(t *testing.T) { + requester := newMockRequester() + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + + // set searchTimeOut to low value so the test is quicker + defer func(t time.Duration) { + searchTimeout = t + }(searchTimeout) + searchTimeout = 250 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // start the fetcher + go fetcher.run(ctx, peersToSkip) + + // call the fetch function with an active context + rctx := context.Background() + fetcher.Request(rctx) + + // after 100ms the first request should be initiated + time.Sleep(100 * time.Millisecond) + + select { + case <-requester.requestC: + default: + t.Fatalf("fetch did not initiate request") + } + + // after another 100ms no new request should be initiated, because search timeout is 250ms + time.Sleep(100 * time.Millisecond) + + select { + case <-requester.requestC: + t.Fatalf("unexpected request from fetcher") + default: + } + + // after another 300ms search timeout is over, there should be a new request + time.Sleep(300 * time.Millisecond) + + select { + case <-requester.requestC: + default: + t.Fatalf("fetch did not retry request") + } +} + +// TestFetcherFactory creates a FetcherFactory and checks if the factory really creates and starts +// a Fetcher when it return a fetch function. We test the fetching functionality just by checking if +// a request is initiated when the fetch function is called +func TestFetcherFactory(t *testing.T) { + requester := newMockRequester(100 * time.Millisecond) + addr := make([]byte, 32) + fetcherFactory := NewFetcherFactory(requester.doRequest, false) + + peersToSkip := &sync.Map{} + + fetcher := fetcherFactory.New(context.Background(), addr, peersToSkip) + + fetcher.Request(context.Background()) + + // check if the created fetchFunction really starts a fetcher and initiates a request + select { + case <-requester.requestC: + case <-time.After(200 * time.Millisecond): + t.Fatalf("fetch timeout") + } + +} + +func TestFetcherRequestQuitRetriesRequest(t *testing.T) { + requester := newMockRequester() + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + // make sure searchTimeout is long so it is sure the request is not retried because of timeout + defer func(t time.Duration) { + searchTimeout = t + }(searchTimeout) + searchTimeout = 10 * time.Second + + peersToSkip := &sync.Map{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go fetcher.run(ctx, peersToSkip) + + rctx := context.Background() + fetcher.Request(rctx) + + select { + case <-requester.requestC: + case <-time.After(200 * time.Millisecond): + t.Fatalf("request is not initiated") + } + + close(requester.quitC) + + select { + case <-requester.requestC: + case <-time.After(200 * time.Millisecond): + t.Fatalf("request is not initiated after failed request") + } +} + +// TestRequestSkipPeer checks if PeerSkip function will skip provided peer +// and not skip unknown one. +func TestRequestSkipPeer(t *testing.T) { + addr := make([]byte, 32) + peers := []discover.NodeID{ + discover.MustHexID("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), + discover.MustHexID("2dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), + } + + peersToSkip := new(sync.Map) + peersToSkip.Store(peers[0].String(), time.Now()) + r := NewRequest(addr, false, peersToSkip) + + if !r.SkipPeer(peers[0].String()) { + t.Errorf("peer not skipped") + } + + if r.SkipPeer(peers[1].String()) { + t.Errorf("peer skipped") + } +} + +// TestRequestSkipPeerExpired checks if a peer to skip is not skipped +// after RequestTimeout has passed. +func TestRequestSkipPeerExpired(t *testing.T) { + addr := make([]byte, 32) + peer := discover.MustHexID("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439") + + // set RequestTimeout to a low value and reset it after the test + defer func(t time.Duration) { RequestTimeout = t }(RequestTimeout) + RequestTimeout = 250 * time.Millisecond + + peersToSkip := new(sync.Map) + peersToSkip.Store(peer.String(), time.Now()) + r := NewRequest(addr, false, peersToSkip) + + if !r.SkipPeer(peer.String()) { + t.Errorf("peer not skipped") + } + + time.Sleep(500 * time.Millisecond) + + if r.SkipPeer(peer.String()) { + t.Errorf("peer skipped") + } +} + +// TestRequestSkipPeerPermanent checks if a peer to skip is not skipped +// after RequestTimeout is not skipped if it is set for a permanent skipping +// by value to peersToSkip map is not time.Duration. +func TestRequestSkipPeerPermanent(t *testing.T) { + addr := make([]byte, 32) + peer := discover.MustHexID("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439") + + // set RequestTimeout to a low value and reset it after the test + defer func(t time.Duration) { RequestTimeout = t }(RequestTimeout) + RequestTimeout = 250 * time.Millisecond + + peersToSkip := new(sync.Map) + peersToSkip.Store(peer.String(), true) + r := NewRequest(addr, false, peersToSkip) + + if !r.SkipPeer(peer.String()) { + t.Errorf("peer not skipped") + } + + time.Sleep(500 * time.Millisecond) + + if !r.SkipPeer(peer.String()) { + t.Errorf("peer not skipped") + } +} diff --git a/swarm/network/priorityqueue/priorityqueue.go b/swarm/network/priorityqueue/priorityqueue.go index fab638c9e2..5385026054 100644 --- a/swarm/network/priorityqueue/priorityqueue.go +++ b/swarm/network/priorityqueue/priorityqueue.go @@ -28,10 +28,13 @@ package priorityqueue import ( "context" "errors" + + "github.com/ethereum/go-ethereum/log" ) var ( - errContention = errors.New("queue contention") + ErrContention = errors.New("contention") + errBadPriority = errors.New("bad priority") wakey = struct{}{} @@ -39,7 +42,7 @@ var ( // PriorityQueue is the basic structure type PriorityQueue struct { - queues []chan interface{} + Queues []chan interface{} wakeup chan struct{} } @@ -50,27 +53,29 @@ func New(n int, l int) *PriorityQueue { queues[i] = make(chan interface{}, l) } return &PriorityQueue{ - queues: queues, + Queues: queues, wakeup: make(chan struct{}, 1), } } // Run is a forever loop popping items from the queues func (pq *PriorityQueue) Run(ctx context.Context, f func(interface{})) { - top := len(pq.queues) - 1 + top := len(pq.Queues) - 1 p := top READ: for { - q := pq.queues[p] + q := pq.Queues[p] select { case <-ctx.Done(): return case x := <-q: + log.Trace("priority.queue f(x)", "p", p, "len(Queues[p])", len(pq.Queues[p])) f(x) p = top default: if p > 0 { p-- + log.Trace("priority.queue p > 0", "p", p) continue READ } p = top @@ -78,6 +83,7 @@ READ: case <-ctx.Done(): return case <-pq.wakeup: + log.Trace("priority.queue wakeup", "p", p) } } } @@ -85,23 +91,15 @@ READ: // Push pushes an item to the appropriate queue specified in the priority argument // if context is given it waits until either the item is pushed or the Context aborts -// otherwise returns errContention if the queue is full -func (pq *PriorityQueue) Push(ctx context.Context, x interface{}, p int) error { - if p < 0 || p >= len(pq.queues) { +func (pq *PriorityQueue) Push(x interface{}, p int) error { + if p < 0 || p >= len(pq.Queues) { return errBadPriority } - if ctx == nil { - select { - case pq.queues[p] <- x: - default: - return errContention - } - } else { - select { - case pq.queues[p] <- x: - case <-ctx.Done(): - return ctx.Err() - } + log.Trace("priority.queue push", "p", p, "len(Queues[p])", len(pq.Queues[p])) + select { + case pq.Queues[p] <- x: + default: + return ErrContention } select { case pq.wakeup <- wakey: diff --git a/swarm/network/priorityqueue/priorityqueue_test.go b/swarm/network/priorityqueue/priorityqueue_test.go index cd54250f8e..ed8b575c20 100644 --- a/swarm/network/priorityqueue/priorityqueue_test.go +++ b/swarm/network/priorityqueue/priorityqueue_test.go @@ -30,7 +30,7 @@ func TestPriorityQueue(t *testing.T) { results = append(results, v.(string)) wg.Done() }) - pq.Push(context.Background(), "2.0", 2) + pq.Push("2.0", 2) wg.Wait() if results[0] != "2.0" { t.Errorf("expected first result %q, got %q", "2.0", results[0]) @@ -66,7 +66,7 @@ Loop: { priorities: []int{0, 0, 0}, values: []string{"0.0", "0.0", "0.1"}, - errors: []error{nil, nil, errContention}, + errors: []error{nil, nil, ErrContention}, }, } { var results []string @@ -74,7 +74,7 @@ Loop: pq := New(3, 2) wg.Add(len(tc.values)) for j, value := range tc.values { - err := pq.Push(nil, value, tc.priorities[j]) + err := pq.Push(value, tc.priorities[j]) if tc.errors != nil && err != tc.errors[j] { t.Errorf("expected push error %v, got %v", tc.errors[j], err) continue Loop diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index 74f9d98ee9..096f7322c0 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -94,7 +94,7 @@ func New(services map[string]ServiceFunc) (s *Simulation) { } s.Net = simulations.NewNetwork( - adapters.NewSimAdapter(adapterServices), + adapters.NewTCPAdapter(adapterServices), &simulations.NetworkConfig{ID: "0"}, ) @@ -164,17 +164,6 @@ var maxParallelCleanups = 10 func (s *Simulation) Close() { close(s.done) - // Close all connections before calling the Network Shutdown. - // It is possible that p2p.Server.Stop will block if there are - // existing connections. - for _, c := range s.Net.Conns { - if c.Up { - s.Net.Disconnect(c.One, c.Other) - } - } - s.shutdownWG.Wait() - s.Net.Shutdown() - sem := make(chan struct{}, maxParallelCleanups) s.mu.RLock() cleanupFuncs := make([]func(), len(s.cleanupFuncs)) @@ -206,6 +195,9 @@ func (s *Simulation) Close() { } close(s.runC) } + + s.shutdownWG.Wait() + s.Net.Shutdown() } // Done returns a channel that is closed when the simulation diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 491dc9fd58..e0d776e349 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -107,9 +107,14 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora return nil, nil, nil, removeDataDir, err } - db := storage.NewDBAPI(localStore) - delivery := NewDelivery(to, db) - streamer := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), nil) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, nil, removeDataDir, err + } + + delivery := NewDelivery(to, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + streamer := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), nil) teardown := func() { streamer.Close() removeDataDir() @@ -150,14 +155,14 @@ func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { } } -func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (*storage.Chunk, error) { +func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (storage.Chunk, error) { return nil, errors.New("get not well defined on round robin store") } -func (rrs *roundRobinStore) Put(ctx context.Context, chunk *storage.Chunk) { +func (rrs *roundRobinStore) Put(ctx context.Context, chunk storage.Chunk) error { i := atomic.AddUint32(&rrs.index, 1) idx := int(i) % len(rrs.stores) - rrs.stores[idx].Put(ctx, chunk) + return rrs.stores[idx].Put(ctx, chunk) } func (rrs *roundRobinStore) Close() { diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 6273525355..d0f27eebcf 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -19,12 +19,11 @@ package stream import ( "context" "errors" - "time" - "github.com/ethereum/go-ethereum/common" + "fmt" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/discover" - cp "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/spancontext" @@ -46,39 +45,34 @@ var ( ) type Delivery struct { - db *storage.DBAPI - kad *network.Kademlia - receiveC chan *ChunkDeliveryMsg - getPeer func(discover.NodeID) *Peer + chunkStore storage.SyncChunkStore + kad *network.Kademlia + getPeer func(discover.NodeID) *Peer } -func NewDelivery(kad *network.Kademlia, db *storage.DBAPI) *Delivery { - d := &Delivery{ - db: db, - kad: kad, - receiveC: make(chan *ChunkDeliveryMsg, deliveryCap), +func NewDelivery(kad *network.Kademlia, chunkStore storage.SyncChunkStore) *Delivery { + return &Delivery{ + chunkStore: chunkStore, + kad: kad, } - - go d.processReceivedChunks() - return d } // SwarmChunkServer implements Server type SwarmChunkServer struct { deliveryC chan []byte batchC chan []byte - db *storage.DBAPI + chunkStore storage.ChunkStore currentLen uint64 quit chan struct{} } // NewSwarmChunkServer is SwarmChunkServer constructor -func NewSwarmChunkServer(db *storage.DBAPI) *SwarmChunkServer { +func NewSwarmChunkServer(chunkStore storage.ChunkStore) *SwarmChunkServer { s := &SwarmChunkServer{ - deliveryC: make(chan []byte, deliveryCap), - batchC: make(chan []byte), - db: db, - quit: make(chan struct{}), + deliveryC: make(chan []byte, deliveryCap), + batchC: make(chan []byte), + chunkStore: chunkStore, + quit: make(chan struct{}), } go s.processDeliveries() return s @@ -123,13 +117,11 @@ func (s *SwarmChunkServer) Close() { // GetData retrives chunk data from db store func (s *SwarmChunkServer) GetData(ctx context.Context, key []byte) ([]byte, error) { - chunk, err := s.db.Get(ctx, storage.Address(key)) - if err == storage.ErrFetching { - <-chunk.ReqC - } else if err != nil { + chunk, err := s.chunkStore.Get(ctx, storage.Address(key)) + if err != nil { return nil, err } - return chunk.SData, nil + return chunk.Data(), nil } // RetrieveRequestMsg is the protocol msg for chunk retrieve requests @@ -153,57 +145,39 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return err } streamer := s.Server.(*SwarmChunkServer) - chunk, created := d.db.GetOrCreateRequest(ctx, req.Addr) - if chunk.ReqC != nil { - if created { - if err := d.RequestFromPeers(ctx, chunk.Addr[:], true, sp.ID()); err != nil { - log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Addr, "err", err) - chunk.SetErrored(storage.ErrChunkForward) - return nil - } + + var cancel func() + // TODO: do something with this hardcoded timeout, maybe use TTL in the future + ctx, cancel = context.WithTimeout(context.WithValue(ctx, "peer", sp.ID().String()), network.RequestTimeout) + + go func() { + select { + case <-ctx.Done(): + case <-streamer.quit: } - go func() { - var osp opentracing.Span - ctx, osp = spancontext.StartSpan( - ctx, - "waiting.delivery") - defer osp.Finish() + cancel() + }() - t := time.NewTimer(10 * time.Minute) - defer t.Stop() - - log.Debug("waiting delivery", "peer", sp.ID(), "hash", req.Addr, "node", common.Bytes2Hex(d.kad.BaseAddr()), "created", created) - start := time.Now() - select { - case <-chunk.ReqC: - log.Debug("retrieve request ReqC closed", "peer", sp.ID(), "hash", req.Addr, "time", time.Since(start)) - case <-t.C: - log.Debug("retrieve request timeout", "peer", sp.ID(), "hash", req.Addr) - chunk.SetErrored(storage.ErrChunkTimeout) - return - } - chunk.SetErrored(nil) - - if req.SkipCheck { - err := sp.Deliver(ctx, chunk, s.priority) - if err != nil { - log.Warn("ERROR in handleRetrieveRequestMsg, DROPPING peer!", "err", err) - sp.Drop(err) - } - } - streamer.deliveryC <- chunk.Addr[:] - }() - return nil - } - // TODO: call the retrieve function of the outgoing syncer - if req.SkipCheck { - log.Trace("deliver", "peer", sp.ID(), "hash", chunk.Addr) - if length := len(chunk.SData); length < 9 { - log.Error("Chunk.SData to deliver is too short", "len(chunk.SData)", length, "address", chunk.Addr) + go func() { + chunk, err := d.chunkStore.Get(ctx, req.Addr) + if err != nil { + log.Warn("ChunkStore.Get can not retrieve chunk", "err", err) + return } - return sp.Deliver(ctx, chunk, s.priority) - } - streamer.deliveryC <- chunk.Addr[:] + if req.SkipCheck { + err = sp.Deliver(ctx, chunk, s.priority) + if err != nil { + log.Warn("ERROR in handleRetrieveRequestMsg", "err", err) + } + return + } + select { + case streamer.deliveryC <- chunk.Address()[:]: + case <-streamer.quit: + } + + }() + return nil } @@ -213,6 +187,7 @@ type ChunkDeliveryMsg struct { peer *Peer // set in handleChunkDeliveryMsg } +// TODO: Fix context SNAFU func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *ChunkDeliveryMsg) error { var osp opentracing.Span ctx, osp = spancontext.StartSpan( @@ -220,81 +195,63 @@ func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *Ch "chunk.delivery") defer osp.Finish() - req.peer = sp - d.receiveC <- req + processReceivedChunksCount.Inc(1) + + go func() { + req.peer = sp + err := d.chunkStore.Put(ctx, storage.NewChunk(req.Addr, req.SData)) + if err != nil { + if err == storage.ErrChunkInvalid { + // we removed this log because it spams the logs + // TODO: Enable this log line + // log.Warn("invalid chunk delivered", "peer", sp.ID(), "chunk", req.Addr, ) + req.peer.Drop(err) + } + } + }() return nil } -func (d *Delivery) processReceivedChunks() { -R: - for req := range d.receiveC { - processReceivedChunksCount.Inc(1) - - if len(req.SData) > cp.DefaultSize+8 { - log.Warn("received chunk is bigger than expected", "len", len(req.SData)) - continue R - } - - // this should be has locally - chunk, err := d.db.Get(context.TODO(), req.Addr) - if err == nil { - continue R - } - if err != storage.ErrFetching { - log.Error("processReceivedChunks db error", "addr", req.Addr, "err", err, "chunk", chunk) - continue R - } - select { - case <-chunk.ReqC: - log.Error("someone else delivered?", "hash", chunk.Addr.Hex()) - continue R - default: - } - - chunk.SData = req.SData - d.db.Put(context.TODO(), chunk) - - go func(req *ChunkDeliveryMsg) { - err := chunk.WaitToStore() - if err == storage.ErrChunkInvalid { - req.peer.Drop(err) - } - }(req) - } -} - // RequestFromPeers sends a chunk retrieve request to -func (d *Delivery) RequestFromPeers(ctx context.Context, hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error { - var success bool - var err error +func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) (*discover.NodeID, chan struct{}, error) { requestFromPeersCount.Inc(1) + var sp *Peer + spID := req.Source - d.kad.EachConn(hash, 255, func(p *network.Peer, po int, nn bool) bool { - spId := p.ID() - for _, p := range peersToSkip { - if p == spId { - log.Trace("Delivery.RequestFromPeers: skip peer", "peer", spId) + if spID != nil { + sp = d.getPeer(*spID) + if sp == nil { + return nil, nil, fmt.Errorf("source peer %v not found", spID.String()) + } + } else { + d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int, nn bool) bool { + id := p.ID() + // TODO: skip light nodes that do not accept retrieve requests + if req.SkipPeer(id.String()) { + log.Trace("Delivery.RequestFromPeers: skip peer", "peer id", id) return true } - } - sp := d.getPeer(spId) + sp = d.getPeer(id) + if sp == nil { + log.Warn("Delivery.RequestFromPeers: peer not found", "id", id) + return true + } + spID = &id + return false + }) if sp == nil { - log.Warn("Delivery.RequestFromPeers: peer not found", "id", spId) - return true + return nil, nil, errors.New("no peer found") } - err = sp.SendPriority(ctx, &RetrieveRequestMsg{ - Addr: hash, - SkipCheck: skipCheck, - }, Top) - if err != nil { - return true - } - requestFromPeersEachCount.Inc(1) - success = true - return false - }) - if success { - return nil } - return errors.New("no peer found") + + err := sp.SendPriority(ctx, &RetrieveRequestMsg{ + Addr: req.Addr, + SkipCheck: req.SkipCheck, + }, Top) + if err != nil { + return nil, nil, err + } + requestFromPeersEachCount.Inc(1) + + return spID, sp.quit, nil } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 972cc859a3..ece54d4ee9 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -47,7 +47,13 @@ func TestStreamerRetrieveRequest(t *testing.T) { peerID := tester.IDs[0] - streamer.delivery.RequestFromPeers(context.TODO(), hash0[:], true) + ctx := context.Background() + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + streamer.delivery.RequestFromPeers(ctx, req) err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", @@ -93,7 +99,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { { Code: 5, Msg: &RetrieveRequestMsg{ - Addr: chunk.Addr[:], + Addr: chunk.Address()[:], }, Peer: peerID, }, @@ -139,10 +145,11 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { }) hash := storage.Address(hash0[:]) - chunk := storage.NewChunk(hash, nil) - chunk.SData = hash - localStore.Put(context.TODO(), chunk) - chunk.WaitToStore() + chunk := storage.NewChunk(hash, hash) + err = localStore.Put(context.TODO(), chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", @@ -178,10 +185,11 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } hash = storage.Address(hash1[:]) - chunk = storage.NewChunk(hash, nil) - chunk.SData = hash1[:] - localStore.Put(context.TODO(), chunk) - chunk.WaitToStore() + chunk = storage.NewChunk(hash, hash1[:]) + err = localStore.Put(context.TODO(), chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", @@ -235,16 +243,6 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { chunkKey := hash0[:] chunkData := hash1[:] - chunk, created := localStore.GetOrCreateRequest(context.TODO(), chunkKey) - - if !created { - t.Fatal("chunk already exists") - } - select { - case <-chunk.ReqC: - t.Fatal("chunk is already received") - default: - } err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", @@ -261,7 +259,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { }, }, p2ptest.Exchange{ - Label: "ChunkDeliveryRequest message", + Label: "ChunkDelivery message", Triggers: []p2ptest.Trigger{ { Code: 6, @@ -277,21 +275,26 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { if err != nil { t.Fatalf("Expected no error, got %v", err) } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() - timeout := time.NewTimer(1 * time.Second) - - select { - case <-timeout.C: - t.Fatal("timeout receiving chunk") - case <-chunk.ReqC: + // wait for the chunk to get stored + storedChunk, err := localStore.Get(ctx, chunkKey) + for err != nil { + select { + case <-ctx.Done(): + t.Fatalf("Chunk is not in localstore after timeout, err: %v", err) + default: + } + storedChunk, err = localStore.Get(ctx, chunkKey) + time.Sleep(50 * time.Millisecond) } - storedChunk, err := localStore.Get(context.TODO(), chunkKey) if err != nil { t.Fatalf("Expected no error, got %v", err) } - if !bytes.Equal(storedChunk.SData, chunkData) { + if !bytes.Equal(storedChunk.Data(), chunkData) { t.Fatal("Retrieved chunk has different data than original") } @@ -324,19 +327,20 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck store.Close() } localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, }) bucket.Store(bucketKeyRegistry, r) - retrieveFunc := func(ctx context.Context, chunk *storage.Chunk) error { - return delivery.RequestFromPeers(ctx, chunk.Addr[:], skipCheck) - } - netStore := storage.NewNetStore(localStore, retrieveFunc) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) @@ -498,7 +502,6 @@ func BenchmarkDeliveryFromNodesWithCheck(b *testing.B) { func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skipCheck bool) { sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { - id := ctx.Config.ID addr := network.NewAddrFromNodeID(id) store, datadir, err := createTestLocalStorageForID(id, addr) @@ -511,20 +514,20 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip store.Close() } localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, DoSync: true, SyncUpdateDelay: 0, }) - retrieveFunc := func(ctx context.Context, chunk *storage.Chunk) error { - return delivery.RequestFromPeers(ctx, chunk.Addr[:], skipCheck) - } - netStore := storage.NewNetStore(localStore, retrieveFunc) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index f4294134b9..452aaca76d 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -38,13 +38,18 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -func TestIntervals(t *testing.T) { +func TestIntervalsLive(t *testing.T) { testIntervals(t, true, nil, false) - testIntervals(t, false, NewRange(9, 26), false) - testIntervals(t, true, NewRange(9, 26), false) - testIntervals(t, true, nil, true) +} + +func TestIntervalsHistory(t *testing.T) { + testIntervals(t, false, NewRange(9, 26), false) testIntervals(t, false, NewRange(9, 26), true) +} + +func TestIntervalsLiveAndHistory(t *testing.T) { + testIntervals(t, true, NewRange(9, 26), false) testIntervals(t, true, NewRange(9, 26), true) } @@ -70,17 +75,21 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { os.RemoveAll(datadir) } localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, }) bucket.Store(bucketKeyRegistry, r) r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) { - return newTestExternalClient(db), nil + return newTestExternalClient(netStore), nil }) r.RegisterServerFunc(externalStreamName, func(p *Peer, t string, live bool) (Server, error) { return newTestExternalServer(t, externalStreamSessionAt, externalStreamMaxKeys, nil), nil @@ -101,9 +110,13 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { t.Fatal(err) } - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + t.Fatal(err) + } + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() storer := nodeIDs[0] @@ -136,11 +149,6 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { liveErrC := make(chan error) historyErrC := make(chan error) - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { - log.Error("WaitKademlia error: %v", "err", err) - return err - } - log.Debug("Watching for disconnections") disconnections := sim.PeerEvents( context.Background(), @@ -148,6 +156,11 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), ) + err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) + if err != nil { + return err + } + go func() { for d := range disconnections { if d.Error != nil { @@ -172,7 +185,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { var liveHashesChan chan []byte liveHashesChan, err = getHashes(ctx, registry, storer, NewStream(externalStreamName, "", true)) if err != nil { - log.Error("Subscription error: %v", "err", err) + log.Error("get hashes", "err", err) return } i := externalStreamSessionAt @@ -216,6 +229,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { var historyHashesChan chan []byte historyHashesChan, err = getHashes(ctx, registry, storer, NewStream(externalStreamName, "", false)) if err != nil { + log.Error("get hashes", "err", err) return } @@ -252,10 +266,6 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { } }() - err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) - if err != nil { - return err - } if err := <-liveErrC; err != nil { return err } @@ -302,34 +312,32 @@ func enableNotifications(r *Registry, peerID discover.NodeID, s Stream) error { type testExternalClient struct { hashes chan []byte - db *storage.DBAPI + store storage.SyncChunkStore enableNotificationsC chan struct{} } -func newTestExternalClient(db *storage.DBAPI) *testExternalClient { +func newTestExternalClient(store storage.SyncChunkStore) *testExternalClient { return &testExternalClient{ hashes: make(chan []byte), - db: db, + store: store, enableNotificationsC: make(chan struct{}), } } -func (c *testExternalClient) NeedData(ctx context.Context, hash []byte) func() { - chunk, _ := c.db.GetOrCreateRequest(ctx, hash) - if chunk.ReqC == nil { +func (c *testExternalClient) NeedData(ctx context.Context, hash []byte) func(context.Context) error { + wait := c.store.FetchFunc(ctx, storage.Address(hash)) + if wait == nil { return nil } - c.hashes <- hash - //NOTE: This was failing on go1.9.x with a deadlock. - //Sometimes this function would just block - //It is commented now, but it may be well worth after the chunk refactor - //to re-enable this and see if the problem has been addressed - /* - return func() { - return chunk.WaitToStore() + select { + case c.hashes <- hash: + case <-ctx.Done(): + log.Warn("testExternalClient NeedData context", "err", ctx.Err()) + return func(_ context.Context) error { + return ctx.Err() } - */ - return nil + } + return wait } func (c *testExternalClient) BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) { diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index a19f635892..2e1a81e822 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -18,9 +18,7 @@ package stream import ( "context" - "errors" "fmt" - "sync" "time" "github.com/ethereum/go-ethereum/metrics" @@ -31,6 +29,8 @@ import ( opentracing "github.com/opentracing/opentracing-go" ) +var syncBatchTimeout = 30 * time.Second + // Stream defines a unique stream identifier. type Stream struct { // Name is used for Client and Server functions identification. @@ -117,8 +117,7 @@ func (p *Peer) handleSubscribeMsg(ctx context.Context, req *SubscribeMsg) (err e go func() { if err := p.SendOfferedHashes(os, from, to); err != nil { - log.Warn("SendOfferedHashes dropping peer", "err", err) - p.Drop(err) + log.Warn("SendOfferedHashes error", "peer", p.ID().TerminalString(), "err", err) } }() @@ -135,8 +134,7 @@ func (p *Peer) handleSubscribeMsg(ctx context.Context, req *SubscribeMsg) (err e } go func() { if err := p.SendOfferedHashes(os, req.History.From, req.History.To); err != nil { - log.Warn("SendOfferedHashes dropping peer", "err", err) - p.Drop(err) + log.Warn("SendOfferedHashes error", "peer", p.ID().TerminalString(), "err", err) } }() } @@ -202,38 +200,52 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg if err != nil { return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err) } - wg := sync.WaitGroup{} + + ctr := 0 + errC := make(chan error) + ctx, cancel := context.WithTimeout(ctx, syncBatchTimeout) + + ctx = context.WithValue(ctx, "source", p.ID().String()) for i := 0; i < len(hashes); i += HashSize { hash := hashes[i : i+HashSize] if wait := c.NeedData(ctx, hash); wait != nil { + ctr++ want.Set(i/HashSize, true) - wg.Add(1) // create request and wait until the chunk data arrives and is stored - go func(w func()) { - w() - wg.Done() + go func(w func(context.Context) error) { + select { + case errC <- w(ctx): + case <-ctx.Done(): + } }(wait) } } - // done := make(chan bool) - // go func() { - // wg.Wait() - // close(done) - // }() - // go func() { - // select { - // case <-done: - // s.next <- s.batchDone(p, req, hashes) - // case <-time.After(1 * time.Second): - // p.Drop(errors.New("timeout waiting for batch to be delivered")) - // } - // }() + go func() { - wg.Wait() + defer cancel() + for i := 0; i < ctr; i++ { + select { + case err := <-errC: + if err != nil { + log.Debug("client.handleOfferedHashesMsg() error waiting for chunk, dropping peer", "peer", p.ID(), "err", err) + p.Drop(err) + return + } + case <-ctx.Done(): + log.Debug("client.handleOfferedHashesMsg() context done", "ctx.Err()", ctx.Err()) + return + case <-c.quit: + log.Debug("client.handleOfferedHashesMsg() quit") + return + } + } select { case c.next <- c.batchDone(p, req, hashes): case <-c.quit: + log.Debug("client.handleOfferedHashesMsg() quit") + case <-ctx.Done(): + log.Debug("client.handleOfferedHashesMsg() context done", "ctx.Err()", ctx.Err()) } }() // only send wantedKeysMsg if all missing chunks of the previous batch arrived @@ -242,7 +254,7 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg c.sessionAt = req.From } from, to := c.nextBatch(req.To + 1) - log.Trace("received offered batch", "peer", p.ID(), "stream", req.Stream, "from", req.From, "to", req.To) + log.Trace("set next batch", "peer", p.ID(), "stream", req.Stream, "from", req.From, "to", req.To, "addr", p.streamer.addr.ID()) if from == to { return nil } @@ -254,25 +266,25 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg To: to, } go func() { + log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "from", msg.From, "to", msg.To) select { - case <-time.After(120 * time.Second): - log.Warn("handleOfferedHashesMsg timeout, so dropping peer") - p.Drop(errors.New("handle offered hashes timeout")) - return case err := <-c.next: if err != nil { - log.Warn("c.next dropping peer", "err", err) + log.Warn("c.next error dropping peer", "err", err) p.Drop(err) return } case <-c.quit: + log.Debug("client.handleOfferedHashesMsg() quit") + return + case <-ctx.Done(): + log.Debug("client.handleOfferedHashesMsg() context done", "ctx.Err()", ctx.Err()) return } log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "from", msg.From, "to", msg.To) err := p.SendPriority(ctx, msg, c.priority) if err != nil { - log.Warn("SendPriority err, so dropping peer", "err", err) - p.Drop(err) + log.Warn("SendPriority error", "err", err) } }() return nil @@ -306,8 +318,7 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) // launch in go routine since GetBatch blocks until new hashes arrive go func() { if err := p.SendOfferedHashes(s, req.From, req.To); err != nil { - log.Warn("SendOfferedHashes dropping peer", "err", err) - p.Drop(err) + log.Warn("SendOfferedHashes error", "err", err) } }() // go p.SendOfferedHashes(s, req.From, req.To) @@ -327,11 +338,7 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) if err != nil { return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err) } - chunk := storage.NewChunk(hash, nil) - chunk.SData = data - if length := len(chunk.SData); length < 9 { - log.Error("Chunk.SData to sync is too short", "len(chunk.SData)", length, "address", chunk.Addr) - } + chunk := storage.NewChunk(hash, data) if err := p.Deliver(ctx, chunk, s.priority); err != nil { return err } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 80b9ab711a..1466a7a9c9 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -33,8 +33,6 @@ import ( opentracing "github.com/opentracing/opentracing-go" ) -var sendTimeout = 30 * time.Second - type notFoundError struct { t string s Stream @@ -83,8 +81,40 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { ctx, cancel := context.WithCancel(context.Background()) go p.pq.Run(ctx, func(i interface{}) { wmsg := i.(WrappedPriorityMsg) - p.Send(wmsg.Context, wmsg.Msg) + err := p.Send(wmsg.Context, wmsg.Msg) + if err != nil { + log.Error("Message send error, dropping peer", "peer", p.ID(), "err", err) + p.Drop(err) + } }) + + // basic monitoring for pq contention + go func(pq *pq.PriorityQueue) { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + var len_maxi int + var cap_maxi int + for k := range pq.Queues { + if len_maxi < len(pq.Queues[k]) { + len_maxi = len(pq.Queues[k]) + } + + if cap_maxi < cap(pq.Queues[k]) { + cap_maxi = cap(pq.Queues[k]) + } + } + + metrics.GetOrRegisterGauge(fmt.Sprintf("pq_len_%s", p.ID().TerminalString()), nil).Update(int64(len_maxi)) + metrics.GetOrRegisterGauge(fmt.Sprintf("pq_cap_%s", p.ID().TerminalString()), nil).Update(int64(cap_maxi)) + case <-p.quit: + return + } + } + }(p.pq) + go func() { <-p.quit cancel() @@ -93,7 +123,7 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { } // Deliver sends a storeRequestMsg protocol message to the peer -func (p *Peer) Deliver(ctx context.Context, chunk *storage.Chunk, priority uint8) error { +func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8) error { var sp opentracing.Span ctx, sp = spancontext.StartSpan( ctx, @@ -101,8 +131,8 @@ func (p *Peer) Deliver(ctx context.Context, chunk *storage.Chunk, priority uint8 defer sp.Finish() msg := &ChunkDeliveryMsg{ - Addr: chunk.Addr, - SData: chunk.SData, + Addr: chunk.Address(), + SData: chunk.Data(), } return p.SendPriority(ctx, msg, priority) } @@ -111,13 +141,16 @@ func (p *Peer) Deliver(ctx context.Context, chunk *storage.Chunk, priority uint8 func (p *Peer) SendPriority(ctx context.Context, msg interface{}, priority uint8) error { defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("peer.sendpriority_t.%d", priority), nil).UpdateSince(time.Now()) metrics.GetOrRegisterCounter(fmt.Sprintf("peer.sendpriority.%d", priority), nil).Inc(1) - cctx, cancel := context.WithTimeout(context.Background(), sendTimeout) - defer cancel() wmsg := WrappedPriorityMsg{ Context: ctx, Msg: msg, } - return p.pq.Push(cctx, wmsg, int(priority)) + err := p.pq.Push(wmsg, int(priority)) + if err == pq.ErrContention { + log.Warn("dropping peer on priority queue contention", "peer", p.ID()) + p.Drop(err) + } + return err } // SendOfferedHashes sends OfferedHashesMsg protocol msg diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 4ff947b215..19eaad34e2 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -124,23 +124,30 @@ func runFileRetrievalTest(nodeCount int) error { return nil, nil, err } bucket.Store(bucketKeyStore, store) - cleanup = func() { - os.RemoveAll(datadir) - store.Close() - } - localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ DoSync: true, SyncUpdateDelay: 3 * time.Second, }) - fileStore := storage.NewFileStore(storage.NewNetStore(localStore, nil), storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) + cleanup = func() { + os.RemoveAll(datadir) + netStore.Close() + r.Close() + } + return r, cleanup, nil }, @@ -267,24 +274,31 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { return nil, nil, err } bucket.Store(bucketKeyStore, store) - cleanup = func() { - os.RemoveAll(datadir) - store.Close() - } - localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ DoSync: true, SyncUpdateDelay: 0, }) - fileStore := storage.NewFileStore(storage.NewNetStore(localStore, nil), storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucketKeyFileStore = simulation.BucketKey("filestore") bucket.Store(bucketKeyFileStore, fileStore) + cleanup = func() { + os.RemoveAll(datadir) + netStore.Close() + r.Close() + } + return r, cleanup, nil }, diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 313019d6a1..7cd09099ce 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "runtime" "sync" "testing" "time" @@ -39,15 +40,20 @@ import ( mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" ) -const testMinProxBinSize = 2 const MaxTimeout = 600 type synctestConfig struct { - addrs [][]byte - hashes []storage.Address - idToChunksMap map[discover.NodeID][]int - chunksToNodesMap map[string][]int - addrToIDMap map[string]discover.NodeID + addrs [][]byte + hashes []storage.Address + idToChunksMap map[discover.NodeID][]int + //chunksToNodesMap map[string][]int + addrToIDMap map[string]discover.NodeID +} + +// Tests in this file should not request chunks from peers. +// This function will panic indicating that there is a problem if request has been made. +func dummyRequestFromPeers(_ context.Context, req *network.Request) (*discover.NodeID, chan struct{}, error) { + panic(fmt.Sprintf("unexpected request: address %s, source %s", req.Addr.String(), req.Source.String())) } //This test is a syncing test for nodes. @@ -58,6 +64,9 @@ type synctestConfig struct { //they are expected to store based on the syncing protocol. //Number of chunks and nodes can be provided via commandline too. func TestSyncingViaGlobalSync(t *testing.T) { + if runtime.GOOS == "darwin" && os.Getenv("TRAVIS") == "true" { + t.Skip("Flaky on mac on travis") + } //if nodes/chunks have been provided via commandline, //run the tests with these values if *nodes != 0 && *chunks != 0 { @@ -86,11 +95,14 @@ func TestSyncingViaGlobalSync(t *testing.T) { } func TestSyncingViaDirectSubscribe(t *testing.T) { + if runtime.GOOS == "darwin" && os.Getenv("TRAVIS") == "true" { + t.Skip("Flaky on mac on travis") + } //if nodes/chunks have been provided via commandline, //run the tests with these values if *nodes != 0 && *chunks != 0 { log.Info(fmt.Sprintf("Running test with %d chunks and %d nodes...", *chunks, *nodes)) - err := testSyncingViaDirectSubscribe(*chunks, *nodes) + err := testSyncingViaDirectSubscribe(t, *chunks, *nodes) if err != nil { t.Fatal(err) } @@ -110,7 +122,7 @@ func TestSyncingViaDirectSubscribe(t *testing.T) { for _, chnk := range chnkCnt { for _, n := range nodeCnt { log.Info(fmt.Sprintf("Long running test with %d chunks and %d nodes...", chnk, n)) - err := testSyncingViaDirectSubscribe(chnk, n) + err := testSyncingViaDirectSubscribe(t, chnk, n) if err != nil { t.Fatal(err) } @@ -130,21 +142,27 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { return nil, nil, err } bucket.Store(bucketKeyStore, store) - cleanup = func() { - os.RemoveAll(datadir) - store.Close() - } localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ DoSync: true, SyncUpdateDelay: 3 * time.Second, }) bucket.Store(bucketKeyRegistry, r) + cleanup = func() { + os.RemoveAll(datadir) + netStore.Close() + r.Close() + } + return r, cleanup, nil }, @@ -166,9 +184,27 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { t.Fatal(err) } - ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) defer cancelSimRun() + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + t.Fatal(err) + } + + disconnections := sim.PeerEvents( + context.Background(), + sim.NodeIDs(), + simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + ) + + go func() { + for d := range disconnections { + log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + t.Fatal("unexpected disconnect") + cancelSimRun() + } + }() + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() for _, n := range nodeIDs { @@ -197,10 +233,6 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { conf.hashes = append(conf.hashes, hashes...) mapKeysToNodes(conf) - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { - return err - } - // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. allSuccess := false @@ -220,6 +252,7 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { }() } for !allSuccess { + allSuccess = true for _, id := range nodeIDs { //for each expected chunk, check if it is in the local store localChunks := conf.idToChunksMap[id] @@ -252,7 +285,10 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) } } - allSuccess = localSuccess + if !localSuccess { + allSuccess = false + break + } } } if !allSuccess { @@ -264,6 +300,7 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { if result.Error != nil { t.Fatal(result.Error) } + log.Info("Simulation ended") } /* @@ -277,7 +314,7 @@ The test loads a snapshot file to construct the swarm network, assuming that the snapshot file identifies a healthy kademlia network. The snapshot should have 'streamer' in its service list. */ -func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { +func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) error { sim := simulation.New(map[string]simulation.ServiceFunc{ "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { @@ -288,28 +325,34 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { return nil, nil, err } bucket.Store(bucketKeyStore, store) - cleanup = func() { - os.RemoveAll(datadir) - store.Close() - } localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), nil) + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), nil) bucket.Store(bucketKeyRegistry, r) - fileStore := storage.NewFileStore(storage.NewNetStore(localStore, nil), storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) + cleanup = func() { + os.RemoveAll(datadir) + netStore.Close() + r.Close() + } + return r, cleanup, nil }, }) defer sim.Close() - ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) defer cancelSimRun() conf := &synctestConfig{} @@ -325,6 +368,24 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { return err } + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + return err + } + + disconnections := sim.PeerEvents( + context.Background(), + sim.NodeIDs(), + simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + ) + + go func() { + for d := range disconnections { + log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + t.Fatal("unexpected disconnect") + cancelSimRun() + } + }() + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() for _, n := range nodeIDs { @@ -402,6 +463,7 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { // or until the timeout is reached. allSuccess := false for !allSuccess { + allSuccess = true for _, id := range nodeIDs { //for each expected chunk, check if it is in the local store localChunks := conf.idToChunksMap[id] @@ -434,7 +496,10 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) } } - allSuccess = localSuccess + if !localSuccess { + allSuccess = false + break + } } } if !allSuccess { @@ -447,7 +512,7 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { return result.Error } - log.Info("Simulation terminated") + log.Info("Simulation ended") return nil } @@ -462,10 +527,9 @@ func startSyncing(r *Registry, conf *synctestConfig) (int, error) { //iterate over each bin and solicit needed subscription to bins kad.EachBin(r.addr.Over(), pof, 0, func(conn *network.Peer, po int) bool { //identify begin and start index of the bin(s) we want to subscribe to - histRange := &Range{} subCnt++ - err = r.RequestSubscription(conf.addrToIDMap[string(conn.Address())], NewStream("SYNC", FormatSyncBinKey(uint8(po)), true), histRange, Top) + err = r.RequestSubscription(conf.addrToIDMap[string(conn.Address())], NewStream("SYNC", FormatSyncBinKey(uint8(po)), true), NewRange(0, 0), High) if err != nil { log.Error(fmt.Sprintf("Error in RequestSubsciption! %v", err)) return false @@ -478,7 +542,6 @@ func startSyncing(r *Registry, conf *synctestConfig) (int, error) { //map chunk keys to addresses which are responsible func mapKeysToNodes(conf *synctestConfig) { - kmap := make(map[string][]int) nodemap := make(map[string][]int) //build a pot for chunk hashes np := pot.NewPot(nil, 0) @@ -487,36 +550,33 @@ func mapKeysToNodes(conf *synctestConfig) { indexmap[string(a)] = i np, _, _ = pot.Add(np, a, pof) } + + var kadMinProxSize = 2 + + ppmap := network.NewPeerPotMap(kadMinProxSize, conf.addrs) + //for each address, run EachNeighbour on the chunk hashes pot to identify closest nodes log.Trace(fmt.Sprintf("Generated hash chunk(s): %v", conf.hashes)) for i := 0; i < len(conf.hashes); i++ { - pl := 256 //highest possible proximity - var nns []int + var a []byte np.EachNeighbour([]byte(conf.hashes[i]), pof, func(val pot.Val, po int) bool { - a := val.([]byte) - if pl < 256 && pl != po { - return false - } - if pl == 256 || pl == po { - log.Trace(fmt.Sprintf("appending %s", conf.addrToIDMap[string(a)])) - nns = append(nns, indexmap[string(a)]) - nodemap[string(a)] = append(nodemap[string(a)], i) - } - if pl == 256 && len(nns) >= testMinProxBinSize { - //maxProxBinSize has been reached at this po, so save it - //we will add all other nodes at the same po - pl = po - } - return true + // take the first address + a = val.([]byte) + return false }) - kmap[string(conf.hashes[i])] = nns + + nns := ppmap[common.Bytes2Hex(a)].NNSet + nns = append(nns, a) + + for _, p := range nns { + nodemap[string(p)] = append(nodemap[string(p)], i) + } } for addr, chunks := range nodemap { //this selects which chunks are expected to be found with the given node conf.idToChunksMap[conf.addrToIDMap[addr]] = chunks } log.Debug(fmt.Sprintf("Map of expected chunks by ID: %v", conf.idToChunksMap)) - conf.chunksToNodesMap = kmap } //upload a file(chunks) to a single local node store diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index deffdfc3f9..1f1f34b7bd 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -32,10 +32,8 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/stream/intervals" "github.com/ethereum/go-ethereum/swarm/pot" - "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" - opentracing "github.com/opentracing/opentracing-go" ) const ( @@ -43,8 +41,8 @@ const ( Mid High Top - PriorityQueue // number of queues - PriorityQueueCap = 32 // queue capacity + PriorityQueue = 4 // number of priority queues - Low, Mid, High, Top + PriorityQueueCap = 128 // queue capacity HashSize = 32 ) @@ -73,7 +71,7 @@ type RegistryOptions struct { } // NewRegistry is Streamer constructor -func NewRegistry(addr *network.BzzAddr, delivery *Delivery, db *storage.DBAPI, intervalsStore state.Store, options *RegistryOptions) *Registry { +func NewRegistry(addr *network.BzzAddr, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry { if options == nil { options = &RegistryOptions{} } @@ -93,13 +91,13 @@ func NewRegistry(addr *network.BzzAddr, delivery *Delivery, db *storage.DBAPI, i streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { - return NewSwarmChunkServer(delivery.db), nil + return NewSwarmChunkServer(delivery.chunkStore), nil }) streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { - return NewSwarmSyncerClient(p, delivery.db, false, NewStream(swarmChunkServerStreamName, t, live)) + return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) }) - RegisterSwarmSyncerServer(streamer, db) - RegisterSwarmSyncerClient(streamer, db) + RegisterSwarmSyncerServer(streamer, syncChunkStore) + RegisterSwarmSyncerClient(streamer, syncChunkStore) if options.DoSync { // latestIntC function ensures that @@ -325,16 +323,6 @@ func (r *Registry) Quit(peerId discover.NodeID, s Stream) error { return peer.Send(context.TODO(), msg) } -func (r *Registry) Retrieve(ctx context.Context, chunk *storage.Chunk) error { - var sp opentracing.Span - ctx, sp = spancontext.StartSpan( - ctx, - "registry.retrieve") - defer sp.Finish() - - return r.delivery.RequestFromPeers(ctx, chunk.Addr[:], r.skipCheck) -} - func (r *Registry) NodeInfo() interface{} { return nil } @@ -557,7 +545,7 @@ func (c client) NextInterval() (start, end uint64, err error) { // Client interface for incoming peer Streamer type Client interface { - NeedData(context.Context, []byte) func() + NeedData(context.Context, []byte) func(context.Context) error BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) Close() } diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index 7523860c9c..06e96b9a97 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -80,15 +80,17 @@ func newTestClient(t string) *testClient { } } -func (self *testClient) NeedData(ctx context.Context, hash []byte) func() { +func (self *testClient) NeedData(ctx context.Context, hash []byte) func(context.Context) error { self.receivedHashes[string(hash)] = hash if bytes.Equal(hash, hash0[:]) { - return func() { + return func(context.Context) error { <-self.wait0 + return nil } } else if bytes.Equal(hash, hash2[:]) { - return func() { + return func(context.Context) error { <-self.wait2 + return nil } } return nil diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index d7febe4a3e..e9811a6785 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -28,7 +28,6 @@ import ( ) const ( - // BatchSize = 2 BatchSize = 128 ) @@ -38,35 +37,37 @@ const ( // * (live/non-live historical) chunk syncing per proximity bin type SwarmSyncerServer struct { po uint8 - db *storage.DBAPI + store storage.SyncChunkStore sessionAt uint64 start uint64 + live bool quit chan struct{} } // NewSwarmSyncerServer is contructor for SwarmSyncerServer -func NewSwarmSyncerServer(live bool, po uint8, db *storage.DBAPI) (*SwarmSyncerServer, error) { - sessionAt := db.CurrentBucketStorageIndex(po) +func NewSwarmSyncerServer(live bool, po uint8, syncChunkStore storage.SyncChunkStore) (*SwarmSyncerServer, error) { + sessionAt := syncChunkStore.BinIndex(po) var start uint64 if live { start = sessionAt } return &SwarmSyncerServer{ po: po, - db: db, + store: syncChunkStore, sessionAt: sessionAt, start: start, + live: live, quit: make(chan struct{}), }, nil } -func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) { +func RegisterSwarmSyncerServer(streamer *Registry, syncChunkStore storage.SyncChunkStore) { streamer.RegisterServerFunc("SYNC", func(p *Peer, t string, live bool) (Server, error) { po, err := ParseSyncBinKey(t) if err != nil { return nil, err } - return NewSwarmSyncerServer(live, po, db) + return NewSwarmSyncerServer(live, po, syncChunkStore) }) // streamer.RegisterServerFunc(stream, func(p *Peer) (Server, error) { // return NewOutgoingProvableSwarmSyncer(po, db) @@ -78,27 +79,35 @@ func (s *SwarmSyncerServer) Close() { close(s.quit) } -// GetSection retrieves the actual chunk from localstore +// GetData retrieves the actual chunk from netstore func (s *SwarmSyncerServer) GetData(ctx context.Context, key []byte) ([]byte, error) { - chunk, err := s.db.Get(ctx, storage.Address(key)) - if err == storage.ErrFetching { - <-chunk.ReqC - } else if err != nil { + chunk, err := s.store.Get(ctx, storage.Address(key)) + if err != nil { return nil, err } - return chunk.SData, nil + return chunk.Data(), nil } // GetBatch retrieves the next batch of hashes from the dbstore func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { var batch []byte i := 0 - if from == 0 { - from = s.start - } - if to <= from || from >= s.sessionAt { - to = math.MaxUint64 + if s.live { + if from == 0 { + from = s.start + } + if to <= from || from >= s.sessionAt { + to = math.MaxUint64 + } + } else { + if (to < from && to != 0) || from > s.sessionAt { + return nil, 0, 0, nil, nil + } + if to == 0 || to > s.sessionAt { + to = s.sessionAt + } } + var ticker *time.Ticker defer func() { if ticker != nil { @@ -119,8 +128,8 @@ func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint6 } metrics.GetOrRegisterCounter("syncer.setnextbatch.iterator", nil).Inc(1) - err := s.db.Iterator(from, to, s.po, func(addr storage.Address, idx uint64) bool { - batch = append(batch, addr[:]...) + err := s.store.Iterator(from, to, s.po, func(key storage.Address, idx uint64) bool { + batch = append(batch, key[:]...) i++ to = idx return i < BatchSize @@ -134,7 +143,7 @@ func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint6 wait = true } - log.Trace("Swarm syncer offer batch", "po", s.po, "len", i, "from", from, "to", to, "current store count", s.db.CurrentBucketStorageIndex(s.po)) + log.Trace("Swarm syncer offer batch", "po", s.po, "len", i, "from", from, "to", to, "current store count", s.store.BinIndex(s.po)) return batch, from, to, nil, nil } @@ -146,28 +155,26 @@ type SwarmSyncerClient struct { sessionReader storage.LazySectionReader retrieveC chan *storage.Chunk storeC chan *storage.Chunk - db *storage.DBAPI + store storage.SyncChunkStore // chunker storage.Chunker - currentRoot storage.Address - requestFunc func(chunk *storage.Chunk) - end, start uint64 - peer *Peer - ignoreExistingRequest bool - stream Stream + currentRoot storage.Address + requestFunc func(chunk *storage.Chunk) + end, start uint64 + peer *Peer + stream Stream } // NewSwarmSyncerClient is a contructor for provable data exchange syncer -func NewSwarmSyncerClient(p *Peer, db *storage.DBAPI, ignoreExistingRequest bool, stream Stream) (*SwarmSyncerClient, error) { +func NewSwarmSyncerClient(p *Peer, store storage.SyncChunkStore, stream Stream) (*SwarmSyncerClient, error) { return &SwarmSyncerClient{ - db: db, - peer: p, - ignoreExistingRequest: ignoreExistingRequest, - stream: stream, + store: store, + peer: p, + stream: stream, }, nil } // // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer -// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *SwarmSyncerClient { +// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Address, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *SwarmSyncerClient { // retrieveC := make(storage.Chunk, chunksCap) // RunChunkRequestor(p, retrieveC) // storeC := make(storage.Chunk, chunksCap) @@ -204,26 +211,15 @@ func NewSwarmSyncerClient(p *Peer, db *storage.DBAPI, ignoreExistingRequest bool // RegisterSwarmSyncerClient registers the client constructor function for // to handle incoming sync streams -func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) { +func RegisterSwarmSyncerClient(streamer *Registry, store storage.SyncChunkStore) { streamer.RegisterClientFunc("SYNC", func(p *Peer, t string, live bool) (Client, error) { - return NewSwarmSyncerClient(p, db, true, NewStream("SYNC", t, live)) + return NewSwarmSyncerClient(p, store, NewStream("SYNC", t, live)) }) } // NeedData -func (s *SwarmSyncerClient) NeedData(ctx context.Context, key []byte) (wait func()) { - chunk, _ := s.db.GetOrCreateRequest(ctx, key) - // TODO: we may want to request from this peer anyway even if the request exists - - // ignoreExistingRequest is temporary commented out until its functionality is verified. - // For now, this optimization can be disabled. - if chunk.ReqC == nil { //|| (s.ignoreExistingRequest && !created) { - return nil - } - // create request and wait until the chunk data arrives and is stored - return func() { - chunk.WaitToStore() - } +func (s *SwarmSyncerClient) NeedData(ctx context.Context, key []byte) (wait func(context.Context) error) { + return s.store.FetchFunc(ctx, key) } // BatchDone diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index f72aa34441..469d520f84 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -102,17 +102,22 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } } localStore := store.(*storage.LocalStore) - db := storage.NewDBAPI(localStore) - bucket.Store(bucketKeyDB, db) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyDB, netStore) kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, db) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + bucket.Store(bucketKeyDelivery, delivery) - r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{ + r := NewRegistry(addr, delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, }) - fileStore := storage.NewFileStore(storage.NewNetStore(localStore, nil), storage.NewFileStoreParams()) + fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) return r, cleanup, nil @@ -197,8 +202,8 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck if !ok { return fmt.Errorf("No DB") } - db := item.(*storage.DBAPI) - db.Iterator(0, math.MaxUint64, po, func(addr storage.Address, index uint64) bool { + netStore := item.(*storage.NetStore) + netStore.Iterator(0, math.MaxUint64, po, func(addr storage.Address, index uint64) bool { hashes[i] = append(hashes[i], addr) totalHashes++ hashCounts[i]++ @@ -216,16 +221,11 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck if !ok { return fmt.Errorf("No DB") } - db := item.(*storage.DBAPI) - chunk, err := db.Get(ctx, key) - if err == storage.ErrFetching { - <-chunk.ReqC - } else if err != nil { - continue + db := item.(*storage.NetStore) + _, err := db.Get(ctx, key) + if err == nil { + found++ } - // needed for leveldb not to be closed? - // chunk.WaitToStore() - found++ } } log.Debug("sync check", "node", node, "index", i, "bin", po, "found", found, "total", total) diff --git a/swarm/network_test.go b/swarm/network_test.go index 9bc6143d84..86a904339c 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -87,10 +87,10 @@ func TestSwarmNetwork(t *testing.T) { }, }, { - name: "100_nodes", + name: "50_nodes", steps: []testSwarmNetworkStep{ { - nodeCount: 100, + nodeCount: 50, }, }, options: &testSwarmNetworkOptions{ @@ -99,10 +99,10 @@ func TestSwarmNetwork(t *testing.T) { disabled: !*longrunning, }, { - name: "100_nodes_skip_check", + name: "50_nodes_skip_check", steps: []testSwarmNetworkStep{ { - nodeCount: 100, + nodeCount: 50, }, }, options: &testSwarmNetworkOptions{ @@ -287,6 +287,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa config.Init(privkey) config.DeliverySkipCheck = o.SkipCheck + config.Port = "" swarm, err := NewSwarm(config, nil) if err != nil { diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 6d805b8e29..40292e88f9 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -22,10 +22,9 @@ import ( "fmt" "io" "sync" - "time" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/spancontext" opentracing "github.com/opentracing/opentracing-go" @@ -67,7 +66,6 @@ The hashing itself does use extra copies and allocation though, since it does ne var ( errAppendOppNotSuported = errors.New("Append operation not supported") - errOperationTimedOut = errors.New("operation timed out") ) type ChunkerParams struct { @@ -133,7 +131,7 @@ type TreeChunker struct { func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader { jp := &JoinerParams{ ChunkerParams: ChunkerParams{ - chunkSize: chunk.DefaultSize, + chunkSize: ch.DefaultSize, hashSize: int64(len(addr)), }, addr: addr, @@ -153,7 +151,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) ( tsp := &TreeSplitterParams{ SplitterParams: SplitterParams{ ChunkerParams: ChunkerParams{ - chunkSize: chunk.DefaultSize, + chunkSize: ch.DefaultSize, hashSize: putter.RefSize(), }, reader: data, @@ -201,11 +199,6 @@ func NewTreeSplitter(params *TreeSplitterParams) *TreeChunker { return tc } -// String() for pretty printing -func (c *Chunk) String() string { - return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", c.Addr.Log(), c.Size, len(c.SData)) -} - type hashJob struct { key Address chunk []byte @@ -236,7 +229,7 @@ func (tc *TreeChunker) Split(ctx context.Context) (k Address, wait func(context. panic("chunker must be initialised") } - tc.runWorker() + tc.runWorker(ctx) depth := 0 treeSize := tc.chunkSize @@ -251,7 +244,7 @@ func (tc *TreeChunker) Split(ctx context.Context) (k Address, wait func(context. // this waitgroup member is released after the root hash is calculated tc.wg.Add(1) //launch actual recursive function passing the waitgroups - go tc.split(depth, treeSize/tc.branches, key, tc.dataSize, tc.wg) + go tc.split(ctx, depth, treeSize/tc.branches, key, tc.dataSize, tc.wg) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -267,14 +260,14 @@ func (tc *TreeChunker) Split(ctx context.Context) (k Address, wait func(context. if err != nil { return nil, nil, err } - case <-time.NewTimer(splitTimeout).C: - return nil, nil, errOperationTimedOut + case <-ctx.Done(): + return nil, nil, ctx.Err() } return key, tc.putter.Wait, nil } -func (tc *TreeChunker) split(depth int, treeSize int64, addr Address, size int64, parentWg *sync.WaitGroup) { +func (tc *TreeChunker) split(ctx context.Context, depth int, treeSize int64, addr Address, size int64, parentWg *sync.WaitGroup) { // @@ -321,10 +314,10 @@ func (tc *TreeChunker) split(depth int, treeSize int64, addr Address, size int64 secSize = treeSize } // the hash of that data - subTreeKey := chunk[8+i*tc.hashSize : 8+(i+1)*tc.hashSize] + subTreeAddress := chunk[8+i*tc.hashSize : 8+(i+1)*tc.hashSize] childrenWg.Add(1) - tc.split(depth-1, treeSize/tc.branches, subTreeKey, secSize, childrenWg) + tc.split(ctx, depth-1, treeSize/tc.branches, subTreeAddress, secSize, childrenWg) i++ pos += treeSize @@ -336,7 +329,7 @@ func (tc *TreeChunker) split(depth int, treeSize int64, addr Address, size int64 worker := tc.getWorkerCount() if int64(len(tc.jobC)) > worker && worker < ChunkProcessors { - tc.runWorker() + tc.runWorker(ctx) } select { @@ -345,7 +338,7 @@ func (tc *TreeChunker) split(depth int, treeSize int64, addr Address, size int64 } } -func (tc *TreeChunker) runWorker() { +func (tc *TreeChunker) runWorker(ctx context.Context) { tc.incrementWorkerCount() go func() { defer tc.decrementWorkerCount() @@ -357,7 +350,7 @@ func (tc *TreeChunker) runWorker() { return } - h, err := tc.putter.Put(tc.ctx, job.chunk) + h, err := tc.putter.Put(ctx, job.chunk) if err != nil { tc.errC <- err return @@ -377,8 +370,8 @@ func (tc *TreeChunker) Append() (Address, func(), error) { // LazyChunkReader implements LazySectionReader type LazyChunkReader struct { - Ctx context.Context - key Address // root key + ctx context.Context + addr Address // root address chunkData ChunkData off int64 // offset chunkSize int64 // inherit from chunker @@ -390,18 +383,18 @@ type LazyChunkReader struct { func (tc *TreeChunker) Join(ctx context.Context) *LazyChunkReader { return &LazyChunkReader{ - key: tc.addr, + addr: tc.addr, chunkSize: tc.chunkSize, branches: tc.branches, hashSize: tc.hashSize, depth: tc.depth, getter: tc.getter, - Ctx: tc.ctx, + ctx: tc.ctx, } } func (r *LazyChunkReader) Context() context.Context { - return r.Ctx + return r.ctx } // Size is meant to be called on the LazySectionReader @@ -415,23 +408,24 @@ func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, e "lcr.size") defer sp.Finish() - log.Debug("lazychunkreader.size", "key", r.key) + log.Debug("lazychunkreader.size", "addr", r.addr) if r.chunkData == nil { - chunkData, err := r.getter.Get(cctx, Reference(r.key)) + chunkData, err := r.getter.Get(cctx, Reference(r.addr)) if err != nil { return 0, err } - if chunkData == nil { - select { - case <-quitC: - return 0, errors.New("aborted") - default: - return 0, fmt.Errorf("root chunk not found for %v", r.key.Hex()) - } - } r.chunkData = chunkData + s := r.chunkData.Size() + log.Debug("lazychunkreader.size", "key", r.addr, "size", s) + if s < 0 { + return 0, errors.New("corrupt size") + } + return int64(s), nil } - return r.chunkData.Size(), nil + s := r.chunkData.Size() + log.Debug("lazychunkreader.size", "key", r.addr, "size", s) + + return int64(s), nil } // read at can be called numerous times @@ -443,7 +437,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { var sp opentracing.Span var cctx context.Context cctx, sp = spancontext.StartSpan( - r.Ctx, + r.ctx, "lcr.read") defer sp.Finish() @@ -460,7 +454,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { quitC := make(chan bool) size, err := r.Size(cctx, quitC) if err != nil { - log.Error("lazychunkreader.readat.size", "size", size, "err", err) + log.Debug("lazychunkreader.readat.size", "size", size, "err", err) return 0, err } @@ -481,7 +475,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { length *= r.chunkSize } wg.Add(1) - go r.join(cctx, b, off, off+length, depth, treeSize/r.branches, r.chunkData, &wg, errC, quitC) + go r.join(b, off, off+length, depth, treeSize/r.branches, r.chunkData, &wg, errC, quitC) go func() { wg.Wait() close(errC) @@ -489,20 +483,22 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { err = <-errC if err != nil { - log.Error("lazychunkreader.readat.errc", "err", err) + log.Debug("lazychunkreader.readat.errc", "err", err) close(quitC) return 0, err } if off+int64(len(b)) >= size { + log.Debug("lazychunkreader.readat.return at end", "size", size, "off", off) return int(size - off), io.EOF } + log.Debug("lazychunkreader.readat.errc", "buff", len(b)) return len(b), nil } -func (r *LazyChunkReader) join(ctx context.Context, b []byte, off int64, eoff int64, depth int, treeSize int64, chunkData ChunkData, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { +func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunkData ChunkData, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { defer parentWg.Done() // find appropriate block level - for chunkData.Size() < treeSize && depth > r.depth { + for chunkData.Size() < uint64(treeSize) && depth > r.depth { treeSize /= r.branches depth-- } @@ -545,19 +541,19 @@ func (r *LazyChunkReader) join(ctx context.Context, b []byte, off int64, eoff in } wg.Add(1) go func(j int64) { - childKey := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize] - chunkData, err := r.getter.Get(ctx, Reference(childKey)) + childAddress := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize] + chunkData, err := r.getter.Get(r.ctx, Reference(childAddress)) if err != nil { - log.Error("lazychunkreader.join", "key", fmt.Sprintf("%x", childKey), "err", err) + log.Debug("lazychunkreader.join", "key", fmt.Sprintf("%x", childAddress), "err", err) select { - case errC <- fmt.Errorf("chunk %v-%v not found; key: %s", off, off+treeSize, fmt.Sprintf("%x", childKey)): + case errC <- fmt.Errorf("chunk %v-%v not found; key: %s", off, off+treeSize, fmt.Sprintf("%x", childAddress)): case <-quitC: } return } if l := len(chunkData); l < 9 { select { - case errC <- fmt.Errorf("chunk %v-%v incomplete; key: %s, data length %v", off, off+treeSize, fmt.Sprintf("%x", childKey), l): + case errC <- fmt.Errorf("chunk %v-%v incomplete; key: %s, data length %v", off, off+treeSize, fmt.Sprintf("%x", childAddress), l): case <-quitC: } return @@ -565,26 +561,26 @@ func (r *LazyChunkReader) join(ctx context.Context, b []byte, off int64, eoff in if soff < off { soff = off } - r.join(ctx, b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/r.branches, chunkData, wg, errC, quitC) + r.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/r.branches, chunkData, wg, errC, quitC) }(i) } //for } // Read keeps a cursor so cannot be called simulateously, see ReadAt func (r *LazyChunkReader) Read(b []byte) (read int, err error) { - log.Debug("lazychunkreader.read", "key", r.key) + log.Debug("lazychunkreader.read", "key", r.addr) metrics.GetOrRegisterCounter("lazychunkreader.read", nil).Inc(1) read, err = r.ReadAt(b, r.off) if err != nil && err != io.EOF { - log.Error("lazychunkreader.readat", "read", read, "err", err) + log.Debug("lazychunkreader.readat", "read", read, "err", err) metrics.GetOrRegisterCounter("lazychunkreader.read.err", nil).Inc(1) } metrics.GetOrRegisterCounter("lazychunkreader.read.bytes", nil).Inc(int64(read)) r.off += int64(read) - return + return read, err } // completely analogous to standard SectionReader implementation @@ -592,7 +588,7 @@ var errWhence = errors.New("Seek: invalid whence") var errOffset = errors.New("Seek: invalid offset") func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { - log.Debug("lazychunkreader.seek", "key", r.key, "offset", offset) + log.Debug("lazychunkreader.seek", "key", r.addr, "offset", offset) switch whence { default: return 0, errWhence @@ -607,7 +603,7 @@ func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { return 0, fmt.Errorf("can't get size: %v", err) } } - offset += r.chunkData.Size() + offset += int64(r.chunkData.Size()) } if offset < 0 { diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index dbcc8700d1..db719ca047 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -21,7 +21,6 @@ import ( "context" "crypto/rand" "encoding/binary" - "errors" "fmt" "io" "testing" @@ -43,27 +42,8 @@ type chunkerTester struct { t test } -// fakeChunkStore doesn't store anything, just implements the ChunkStore interface -// It can be used to inject into a hasherStore if you don't want to actually store data just do the -// hashing -type fakeChunkStore struct { -} - -// Put doesn't store anything it is just here to implement ChunkStore -func (f *fakeChunkStore) Put(context.Context, *Chunk) { -} - -// Gut doesn't store anything it is just here to implement ChunkStore -func (f *fakeChunkStore) Get(context.Context, Address) (*Chunk, error) { - return nil, errors.New("FakeChunkStore doesn't support Get") -} - -// Close doesn't store anything it is just here to implement ChunkStore -func (f *fakeChunkStore) Close() { -} - -func newTestHasherStore(chunkStore ChunkStore, hash string) *hasherStore { - return NewHasherStore(chunkStore, MakeHashFunc(hash), false) +func newTestHasherStore(store ChunkStore, hash string) *hasherStore { + return NewHasherStore(store, MakeHashFunc(hash), false) } func testRandomBrokenData(n int, tester *chunkerTester) { @@ -82,11 +62,12 @@ func testRandomBrokenData(n int, tester *chunkerTester) { putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) expectedError := fmt.Errorf("Broken reader") - addr, _, err := TreeSplit(context.TODO(), brokendata, int64(n), putGetter) + ctx := context.Background() + key, _, err := TreeSplit(ctx, brokendata, int64(n), putGetter) if err == nil || err.Error() != expectedError.Error() { tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) } - tester.t.Logf(" Key = %v\n", addr) + tester.t.Logf(" Address = %v\n", key) } func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) Address { @@ -96,7 +77,7 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = generateRandomData(n) + data, input = GenerateRandomData(n) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -116,13 +97,13 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) if err != nil { tester.t.Fatalf(err.Error()) } - tester.t.Logf(" Key = %v\n", addr) + tester.t.Logf(" Address = %v\n", addr) err = wait(ctx) if err != nil { tester.t.Fatalf(err.Error()) } - reader := TreeJoin(context.TODO(), addr, putGetter, 0) + reader := TreeJoin(ctx, addr, putGetter, 0) output := make([]byte, n) r, err := reader.Read(output) if r != n || err != io.EOF { @@ -196,14 +177,14 @@ func TestDataAppend(t *testing.T) { input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = generateRandomData(n) + data, input = GenerateRandomData(n) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) } - chunkStore := NewMapChunkStore() - putGetter := newTestHasherStore(chunkStore, SHA3Hash) + store := NewMapChunkStore() + putGetter := newTestHasherStore(store, SHA3Hash) ctx := context.TODO() addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) @@ -214,18 +195,17 @@ func TestDataAppend(t *testing.T) { if err != nil { tester.t.Fatalf(err.Error()) } - //create a append data stream appendInput, found := tester.inputs[uint64(m)] var appendData io.Reader if !found { - appendData, appendInput = generateRandomData(m) + appendData, appendInput = GenerateRandomData(m) tester.inputs[uint64(m)] = appendInput } else { appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) } - putGetter = newTestHasherStore(chunkStore, SHA3Hash) + putGetter = newTestHasherStore(store, SHA3Hash) newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter) if err != nil { tester.t.Fatalf(err.Error()) @@ -256,18 +236,18 @@ func TestRandomData(t *testing.T) { tester := &chunkerTester{t: t} for _, s := range sizes { - treeChunkerKey := testRandomData(false, SHA3Hash, s, tester) - pyramidChunkerKey := testRandomData(true, SHA3Hash, s, tester) - if treeChunkerKey.String() != pyramidChunkerKey.String() { - tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) + treeChunkerAddress := testRandomData(false, SHA3Hash, s, tester) + pyramidChunkerAddress := testRandomData(true, SHA3Hash, s, tester) + if treeChunkerAddress.String() != pyramidChunkerAddress.String() { + tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerAddress.String(), pyramidChunkerAddress.String()) } } for _, s := range sizes { - treeChunkerKey := testRandomData(false, BMTHash, s, tester) - pyramidChunkerKey := testRandomData(true, BMTHash, s, tester) - if treeChunkerKey.String() != pyramidChunkerKey.String() { - tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) + treeChunkerAddress := testRandomData(false, BMTHash, s, tester) + pyramidChunkerAddress := testRandomData(true, BMTHash, s, tester) + if treeChunkerAddress.String() != pyramidChunkerAddress.String() { + tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerAddress.String(), pyramidChunkerAddress.String()) } } } @@ -312,12 +292,18 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testDataReader(n) - putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash) + putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) - _, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter) + ctx := context.Background() + _, wait, err := TreeSplit(ctx, data, int64(n), putGetter) if err != nil { t.Fatalf(err.Error()) } + err = wait(ctx) + if err != nil { + t.Fatalf(err.Error()) + } + } } @@ -325,9 +311,32 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testDataReader(n) - putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash) + putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) - _, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter) + ctx := context.Background() + _, wait, err := TreeSplit(ctx, data, int64(n), putGetter) + if err != nil { + t.Fatalf(err.Error()) + } + err = wait(ctx) + if err != nil { + t.Fatalf(err.Error()) + } + } +} + +func benchmarkSplitPyramidBMT(n int, t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + data := testDataReader(n) + putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) + + ctx := context.Background() + _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) + if err != nil { + t.Fatalf(err.Error()) + } + err = wait(ctx) if err != nil { t.Fatalf(err.Error()) } @@ -338,23 +347,14 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { data := testDataReader(n) - putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash) + putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) - _, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter) + ctx := context.Background() + _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) if err != nil { t.Fatalf(err.Error()) } - - } -} - -func benchmarkSplitPyramidBMT(n int, t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - data := testDataReader(n) - putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash) - - _, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter) + err = wait(ctx) if err != nil { t.Fatalf(err.Error()) } @@ -367,10 +367,10 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { data := testDataReader(n) data1 := testDataReader(m) - chunkStore := NewMapChunkStore() - putGetter := newTestHasherStore(chunkStore, SHA3Hash) + store := NewMapChunkStore() + putGetter := newTestHasherStore(store, SHA3Hash) - ctx := context.TODO() + ctx := context.Background() key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) if err != nil { t.Fatalf(err.Error()) @@ -380,7 +380,7 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { t.Fatalf(err.Error()) } - putGetter = newTestHasherStore(chunkStore, SHA3Hash) + putGetter = newTestHasherStore(store, SHA3Hash) _, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter) if err != nil { t.Fatalf(err.Error()) diff --git a/swarm/storage/chunkstore.go b/swarm/storage/chunkstore.go deleted file mode 100644 index 3b4d97a7a7..0000000000 --- a/swarm/storage/chunkstore.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package storage - -import ( - "context" - "sync" -) - -/* -ChunkStore interface is implemented by : - -- MemStore: a memory cache -- DbStore: local disk/db store -- LocalStore: a combination (sequence of) memStore and dbStore -- NetStore: cloud storage abstraction layer -- FakeChunkStore: dummy store which doesn't store anything just implements the interface -*/ -type ChunkStore interface { - Put(context.Context, *Chunk) // effectively there is no error even if there is an error - Get(context.Context, Address) (*Chunk, error) - Close() -} - -// MapChunkStore is a very simple ChunkStore implementation to store chunks in a map in memory. -type MapChunkStore struct { - chunks map[string]*Chunk - mu sync.RWMutex -} - -func NewMapChunkStore() *MapChunkStore { - return &MapChunkStore{ - chunks: make(map[string]*Chunk), - } -} - -func (m *MapChunkStore) Put(ctx context.Context, chunk *Chunk) { - m.mu.Lock() - defer m.mu.Unlock() - m.chunks[chunk.Addr.Hex()] = chunk - chunk.markAsStored() -} - -func (m *MapChunkStore) Get(ctx context.Context, addr Address) (*Chunk, error) { - m.mu.RLock() - defer m.mu.RUnlock() - chunk := m.chunks[addr.Hex()] - if chunk == nil { - return nil, ErrChunkNotFound - } - return chunk, nil -} - -func (m *MapChunkStore) Close() { -} diff --git a/swarm/storage/common.go b/swarm/storage/common.go deleted file mode 100644 index d6352820eb..0000000000 --- a/swarm/storage/common.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . -package storage - -import ( - "context" - "sync" - - "github.com/ethereum/go-ethereum/swarm/log" -) - -// PutChunks adds chunks to localstore -// It waits for receive on the stored channel -// It logs but does not fail on delivery error -func PutChunks(store *LocalStore, chunks ...*Chunk) { - wg := sync.WaitGroup{} - wg.Add(len(chunks)) - go func() { - for _, c := range chunks { - <-c.dbStoredC - if err := c.GetErrored(); err != nil { - log.Error("chunk store fail", "err", err, "key", c.Addr) - } - wg.Done() - } - }() - for _, c := range chunks { - go store.Put(context.TODO(), c) - } - wg.Wait() -} diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index dc1a3ab35f..33133edd74 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -23,16 +23,20 @@ import ( "flag" "fmt" "io" + "io/ioutil" + "os" "sync" "testing" "time" "github.com/ethereum/go-ethereum/log" + ch "github.com/ethereum/go-ethereum/swarm/chunk" colorable "github.com/mattn/go-colorable" ) var ( - loglevel = flag.Int("loglevel", 3, "verbosity of logs") + loglevel = flag.Int("loglevel", 3, "verbosity of logs") + getTimeout = 30 * time.Second ) func init() { @@ -56,47 +60,73 @@ func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader } } -func mputRandomChunks(store ChunkStore, processors int, n int, chunksize int64) (hs []Address) { - return mput(store, processors, n, GenerateRandomChunk) -} - -func mput(store ChunkStore, processors int, n int, f func(i int64) *Chunk) (hs []Address) { - wg := sync.WaitGroup{} - wg.Add(processors) - c := make(chan *Chunk) - for i := 0; i < processors; i++ { - go func() { - defer wg.Done() - for chunk := range c { - wg.Add(1) - chunk := chunk - store.Put(context.TODO(), chunk) - go func() { - defer wg.Done() - <-chunk.dbStoredC - }() - } - }() +func newLDBStore(t *testing.T) (*LDBStore, func()) { + dir, err := ioutil.TempDir("", "bzz-storage-test") + if err != nil { + t.Fatal(err) } - fa := f - if _, ok := store.(*MemStore); ok { - fa = func(i int64) *Chunk { - chunk := f(i) - chunk.markAsStored() - return chunk + log.Trace("memstore.tempdir", "dir", dir) + + ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) + db, err := NewLDBStore(ldbparams) + if err != nil { + t.Fatal(err) + } + + cleanup := func() { + db.Close() + err := os.RemoveAll(dir) + if err != nil { + t.Fatal(err) } } - for i := 0; i < n; i++ { - chunk := fa(int64(i)) - hs = append(hs, chunk.Addr) - c <- chunk - } - close(c) - wg.Wait() - return hs + + return db, cleanup } -func mget(store ChunkStore, hs []Address, f func(h Address, chunk *Chunk) error) error { +func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) { + return mput(store, n, GenerateRandomChunk) +} + +func mputChunks(store ChunkStore, chunks ...Chunk) error { + i := 0 + f := func(n int64) Chunk { + chunk := chunks[i] + i++ + return chunk + } + _, err := mput(store, len(chunks), f) + return err +} + +func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error) { + // put to localstore and wait for stored channel + // does not check delivery error state + errc := make(chan error) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for i := int64(0); i < int64(n); i++ { + chunk := f(ch.DefaultSize) + go func() { + select { + case errc <- store.Put(ctx, chunk): + case <-ctx.Done(): + } + }() + hs = append(hs, chunk) + } + + // wait for all chunks to be stored + for i := 0; i < n; i++ { + err := <-errc + if err != nil { + return nil, err + } + } + return hs, nil +} + +func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error) error { wg := sync.WaitGroup{} wg.Add(len(hs)) errc := make(chan error) @@ -104,6 +134,7 @@ func mget(store ChunkStore, hs []Address, f func(h Address, chunk *Chunk) error) for _, k := range hs { go func(h Address) { defer wg.Done() + // TODO: write timeout with context chunk, err := store.Get(context.TODO(), h) if err != nil { errc <- err @@ -143,57 +174,54 @@ func (r *brokenLimitedReader) Read(buf []byte) (int, error) { return r.lr.Read(buf) } -func generateRandomData(l int) (r io.Reader, slice []byte) { - slice = make([]byte, l) - if _, err := rand.Read(slice); err != nil { - panic("rand error") +func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) { + chunks, err := mputRandomChunks(m, n, chunksize) + if err != nil { + t.Fatalf("expected no error, got %v", err) } - r = io.LimitReader(bytes.NewReader(slice), int64(l)) - return -} - -func testStoreRandom(m ChunkStore, processors int, n int, chunksize int64, t *testing.T) { - hs := mputRandomChunks(m, processors, n, chunksize) - err := mget(m, hs, nil) + err = mget(m, chunkAddresses(chunks), nil) if err != nil { t.Fatalf("testStore failed: %v", err) } } -func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int64, t *testing.T) { - hs := mputRandomChunks(m, processors, n, chunksize) - f := func(h Address, chunk *Chunk) error { - if !bytes.Equal(h, chunk.Addr) { - return fmt.Errorf("key does not match retrieved chunk Key") +func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { + chunks, err := mputRandomChunks(m, n, chunksize) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + f := func(h Address, chunk Chunk) error { + if !bytes.Equal(h, chunk.Address()) { + return fmt.Errorf("key does not match retrieved chunk Address") } hasher := MakeHashFunc(DefaultHash)() - hasher.ResetWithLength(chunk.SData[:8]) - hasher.Write(chunk.SData[8:]) + hasher.ResetWithLength(chunk.SpanBytes()) + hasher.Write(chunk.Payload()) exp := hasher.Sum(nil) if !bytes.Equal(h, exp) { return fmt.Errorf("key is not hash of chunk data") } return nil } - err := mget(m, hs, f) + err = mget(m, chunkAddresses(chunks), f) if err != nil { t.Fatalf("testStore failed: %v", err) } } -func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int64, b *testing.B) { - chunks := make([]*Chunk, n) +func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) { + chunks := make([]Chunk, n) i := 0 - f := func(dataSize int64) *Chunk { + f := func(dataSize int64) Chunk { chunk := GenerateRandomChunk(dataSize) chunks[i] = chunk i++ return chunk } - mput(store, processors, n, f) + mput(store, n, f) - f = func(dataSize int64) *Chunk { + f = func(dataSize int64) Chunk { chunk := chunks[i] i++ return chunk @@ -204,18 +232,62 @@ func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int64, for j := 0; j < b.N; j++ { i = 0 - mput(store, processors, n, f) + mput(store, n, f) } } -func benchmarkStoreGet(store ChunkStore, processors int, n int, chunksize int64, b *testing.B) { - hs := mputRandomChunks(store, processors, n, chunksize) +func benchmarkStoreGet(store ChunkStore, n int, chunksize int64, b *testing.B) { + chunks, err := mputRandomChunks(store, n, chunksize) + if err != nil { + b.Fatalf("expected no error, got %v", err) + } b.ReportAllocs() b.ResetTimer() + addrs := chunkAddresses(chunks) for i := 0; i < b.N; i++ { - err := mget(store, hs, nil) + err := mget(store, addrs, nil) if err != nil { b.Fatalf("mget failed: %v", err) } } } + +// MapChunkStore is a very simple ChunkStore implementation to store chunks in a map in memory. +type MapChunkStore struct { + chunks map[string]Chunk + mu sync.RWMutex +} + +func NewMapChunkStore() *MapChunkStore { + return &MapChunkStore{ + chunks: make(map[string]Chunk), + } +} + +func (m *MapChunkStore) Put(_ context.Context, ch Chunk) error { + m.mu.Lock() + defer m.mu.Unlock() + m.chunks[ch.Address().Hex()] = ch + return nil +} + +func (m *MapChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { + m.mu.RLock() + defer m.mu.RUnlock() + chunk := m.chunks[ref.Hex()] + if chunk == nil { + return nil, ErrChunkNotFound + } + return chunk, nil +} + +func (m *MapChunkStore) Close() { +} + +func chunkAddresses(chunks []Chunk) []Address { + addrs := make([]Address, len(chunks)) + for i, ch := range chunks { + addrs[i] = ch.Address() + } + return addrs +} diff --git a/swarm/storage/dbapi.go b/swarm/storage/dbapi.go deleted file mode 100644 index dd71752eb2..0000000000 --- a/swarm/storage/dbapi.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package storage - -import "context" - -// wrapper of db-s to provide mockable custom local chunk store access to syncer -type DBAPI struct { - db *LDBStore - loc *LocalStore -} - -func NewDBAPI(loc *LocalStore) *DBAPI { - return &DBAPI{loc.DbStore, loc} -} - -// to obtain the chunks from address or request db entry only -func (d *DBAPI) Get(ctx context.Context, addr Address) (*Chunk, error) { - return d.loc.Get(ctx, addr) -} - -// current storage counter of chunk db -func (d *DBAPI) CurrentBucketStorageIndex(po uint8) uint64 { - return d.db.CurrentBucketStorageIndex(po) -} - -// iteration storage counter and proximity order -func (d *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error { - return d.db.SyncIterator(from, to, po, f) -} - -// to obtain the chunks from address or request db entry only -func (d *DBAPI) GetOrCreateRequest(ctx context.Context, addr Address) (*Chunk, bool) { - return d.loc.GetOrCreateRequest(ctx, addr) -} - -// to obtain the chunks from key or request db entry only -func (d *DBAPI) Put(ctx context.Context, chunk *Chunk) { - d.loc.Put(ctx, chunk) -} diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index f3f5972558..d79efb530d 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -49,11 +49,11 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { fileStore := NewFileStore(localStore, NewFileStoreParams()) defer os.RemoveAll("/tmp/bzz") - reader, slice := generateRandomData(testDataSize) + reader, slice := GenerateRandomData(testDataSize) ctx := context.TODO() key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) if err != nil { - t.Errorf("Store error: %v", err) + t.Fatalf("Store error: %v", err) } err = wait(ctx) if err != nil { @@ -66,13 +66,13 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { - t.Errorf("Retrieve error: %v", err) + t.Fatalf("Retrieve error: %v", err) } if n != len(slice) { - t.Errorf("Slice size error got %d, expected %d.", n, len(slice)) + t.Fatalf("Slice size error got %d, expected %d.", n, len(slice)) } if !bytes.Equal(slice, resultSlice) { - t.Errorf("Comparison error.") + t.Fatalf("Comparison error.") } ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666) ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666) @@ -86,13 +86,13 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { } n, err = resultReader.ReadAt(resultSlice, 0) if err != io.EOF { - t.Errorf("Retrieve error after removing memStore: %v", err) + t.Fatalf("Retrieve error after removing memStore: %v", err) } if n != len(slice) { - t.Errorf("Slice size error after removing memStore got %d, expected %d.", n, len(slice)) + t.Fatalf("Slice size error after removing memStore got %d, expected %d.", n, len(slice)) } if !bytes.Equal(slice, resultSlice) { - t.Errorf("Comparison error after removing memStore.") + t.Fatalf("Comparison error after removing memStore.") } } @@ -114,7 +114,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { DbStore: db, } fileStore := NewFileStore(localStore, NewFileStoreParams()) - reader, slice := generateRandomData(testDataSize) + reader, slice := GenerateRandomData(testDataSize) ctx := context.TODO() key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) if err != nil { @@ -122,7 +122,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { } err = wait(ctx) if err != nil { - t.Errorf("Store error: %v", err) + t.Fatalf("Store error: %v", err) } resultReader, isEncrypted := fileStore.Retrieve(context.TODO(), key) if isEncrypted != toEncrypt { @@ -131,13 +131,13 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { - t.Errorf("Retrieve error: %v", err) + t.Fatalf("Retrieve error: %v", err) } if n != len(slice) { - t.Errorf("Slice size error got %d, expected %d.", n, len(slice)) + t.Fatalf("Slice size error got %d, expected %d.", n, len(slice)) } if !bytes.Equal(slice, resultSlice) { - t.Errorf("Comparison error.") + t.Fatalf("Comparison error.") } // Clear memStore memStore.setCapacity(0) @@ -148,7 +148,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) } if _, err = resultReader.ReadAt(resultSlice, 0); err == nil { - t.Errorf("Was able to read %d bytes from an empty memStore.", len(slice)) + t.Fatalf("Was able to read %d bytes from an empty memStore.", len(slice)) } // check how it works with localStore fileStore.ChunkStore = localStore @@ -162,12 +162,12 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { } n, err = resultReader.ReadAt(resultSlice, 0) if err != io.EOF { - t.Errorf("Retrieve error after clearing memStore: %v", err) + t.Fatalf("Retrieve error after clearing memStore: %v", err) } if n != len(slice) { - t.Errorf("Slice size error after clearing memStore got %d, expected %d.", n, len(slice)) + t.Fatalf("Slice size error after clearing memStore got %d, expected %d.", n, len(slice)) } if !bytes.Equal(slice, resultSlice) { - t.Errorf("Comparison error after clearing memStore.") + t.Fatalf("Comparison error after clearing memStore.") } } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 766207eae0..879622b9a0 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -19,10 +19,10 @@ package storage import ( "context" "fmt" - "sync" + "sync/atomic" "github.com/ethereum/go-ethereum/crypto/sha3" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/encryption" ) @@ -30,31 +30,36 @@ type hasherStore struct { store ChunkStore toEncrypt bool hashFunc SwarmHasher - hashSize int // content hash size - refSize int64 // reference size (content hash + possibly encryption key) - wg *sync.WaitGroup - closed chan struct{} + hashSize int // content hash size + refSize int64 // reference size (content hash + possibly encryption key) + nrChunks uint64 // number of chunks to store + errC chan error // global error channel + doneC chan struct{} // closed by Close() call to indicate that count is the final number of chunks + quitC chan struct{} // closed to quit unterminated routines } // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. // With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore // and the hasherStore will take core of encryption/decryption of data if necessary -func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { +func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { hashSize := hashFunc().Size() refSize := int64(hashSize) if toEncrypt { refSize += encryption.KeyLength } - return &hasherStore{ - store: chunkStore, + h := &hasherStore{ + store: store, toEncrypt: toEncrypt, hashFunc: hashFunc, hashSize: hashSize, refSize: refSize, - wg: &sync.WaitGroup{}, - closed: make(chan struct{}), + errC: make(chan error), + doneC: make(chan struct{}), + quitC: make(chan struct{}), } + + return h } // Put stores the chunkData into the ChunkStore of the hasherStore and returns the reference. @@ -62,7 +67,6 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) // Asynchronous function, the data will not necessarily be stored when it returns. func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, error) { c := chunkData - size := chunkData.Size() var encryptionKey encryption.Key if h.toEncrypt { var err error @@ -71,29 +75,28 @@ func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, return nil, err } } - chunk := h.createChunk(c, size) - + chunk := h.createChunk(c) h.storeChunk(ctx, chunk) - return Reference(append(chunk.Addr, encryptionKey...)), nil + return Reference(append(chunk.Address(), encryptionKey...)), nil } // Get returns data of the chunk with the given reference (retrieved from the ChunkStore of hasherStore). // If the data is encrypted and the reference contains an encryption key, it will be decrypted before // return. func (h *hasherStore) Get(ctx context.Context, ref Reference) (ChunkData, error) { - key, encryptionKey, err := parseReference(ref, h.hashSize) + addr, encryptionKey, err := parseReference(ref, h.hashSize) if err != nil { return nil, err } + + chunk, err := h.store.Get(ctx, addr) + if err != nil { + return nil, err + } + + chunkData := ChunkData(chunk.Data()) toDecrypt := (encryptionKey != nil) - - chunk, err := h.store.Get(ctx, key) - if err != nil { - return nil, err - } - - chunkData := chunk.SData if toDecrypt { var err error chunkData, err = h.decryptChunkData(chunkData, encryptionKey) @@ -107,16 +110,40 @@ func (h *hasherStore) Get(ctx context.Context, ref Reference) (ChunkData, error) // Close indicates that no more chunks will be put with the hasherStore, so the Wait // function can return when all the previously put chunks has been stored. func (h *hasherStore) Close() { - close(h.closed) + close(h.doneC) } // Wait returns when // 1) the Close() function has been called and // 2) all the chunks which has been Put has been stored func (h *hasherStore) Wait(ctx context.Context) error { - <-h.closed - h.wg.Wait() - return nil + defer close(h.quitC) + var nrStoredChunks uint64 // number of stored chunks + var done bool + doneC := h.doneC + for { + select { + // if context is done earlier, just return with the error + case <-ctx.Done(): + return ctx.Err() + // doneC is closed if all chunks have been submitted, from then we just wait until all of them are also stored + case <-doneC: + done = true + doneC = nil + // a chunk has been stored, if err is nil, then successfully, so increase the stored chunk counter + case err := <-h.errC: + if err != nil { + return err + } + nrStoredChunks++ + } + // if all the chunks have been submitted and all of them are stored, then we can return + if done { + if nrStoredChunks >= atomic.LoadUint64(&h.nrChunks) { + return nil + } + } + } } func (h *hasherStore) createHash(chunkData ChunkData) Address { @@ -126,12 +153,9 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { return hasher.Sum(nil) } -func (h *hasherStore) createChunk(chunkData ChunkData, chunkSize int64) *Chunk { +func (h *hasherStore) createChunk(chunkData ChunkData) *chunk { hash := h.createHash(chunkData) - chunk := NewChunk(hash, nil) - chunk.SData = chunkData - chunk.Size = chunkSize - + chunk := NewChunk(hash, chunkData) return chunk } @@ -162,10 +186,10 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp // removing extra bytes which were just added for padding length := ChunkData(decryptedSpan).Size() - for length > chunk.DefaultSize { - length = length + (chunk.DefaultSize - 1) - length = length / chunk.DefaultSize - length *= h.refSize + for length > ch.DefaultSize { + length = length + (ch.DefaultSize - 1) + length = length / ch.DefaultSize + length *= uint64(h.refSize) } c := make(ChunkData, length+8) @@ -205,32 +229,32 @@ func (h *hasherStore) decrypt(chunkData ChunkData, key encryption.Key) ([]byte, } func (h *hasherStore) newSpanEncryption(key encryption.Key) encryption.Encryption { - return encryption.New(key, 0, uint32(chunk.DefaultSize/h.refSize), sha3.NewKeccak256) + return encryption.New(key, 0, uint32(ch.DefaultSize/h.refSize), sha3.NewKeccak256) } func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryption { - return encryption.New(key, int(chunk.DefaultSize), 0, sha3.NewKeccak256) + return encryption.New(key, int(ch.DefaultSize), 0, sha3.NewKeccak256) } -func (h *hasherStore) storeChunk(ctx context.Context, chunk *Chunk) { - h.wg.Add(1) +func (h *hasherStore) storeChunk(ctx context.Context, chunk *chunk) { + atomic.AddUint64(&h.nrChunks, 1) go func() { - <-chunk.dbStoredC - h.wg.Done() + select { + case h.errC <- h.store.Put(ctx, chunk): + case <-h.quitC: + } }() - h.store.Put(ctx, chunk) } func parseReference(ref Reference, hashSize int) (Address, encryption.Key, error) { - encryptedKeyLength := hashSize + encryption.KeyLength + encryptedRefLength := hashSize + encryption.KeyLength switch len(ref) { - case KeyLength: + case AddressLength: return Address(ref), nil, nil - case encryptedKeyLength: + case encryptedRefLength: encKeyIdx := len(ref) - encryption.KeyLength return Address(ref[:encKeyIdx]), encryption.Key(ref[encKeyIdx:]), nil default: - return nil, nil, fmt.Errorf("Invalid reference length, expected %v or %v got %v", hashSize, encryptedKeyLength, len(ref)) + return nil, nil, fmt.Errorf("Invalid reference length, expected %v or %v got %v", hashSize, encryptedRefLength, len(ref)) } - } diff --git a/swarm/storage/hasherstore_test.go b/swarm/storage/hasherstore_test.go index ddf1c39b08..22cf98d0e8 100644 --- a/swarm/storage/hasherstore_test.go +++ b/swarm/storage/hasherstore_test.go @@ -46,14 +46,16 @@ func TestHasherStore(t *testing.T) { hasherStore := NewHasherStore(chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt) // Put two random chunks into the hasherStore - chunkData1 := GenerateRandomChunk(int64(tt.chunkLength)).SData - key1, err := hasherStore.Put(context.TODO(), chunkData1) + chunkData1 := GenerateRandomChunk(int64(tt.chunkLength)).Data() + ctx, cancel := context.WithTimeout(context.Background(), getTimeout) + defer cancel() + key1, err := hasherStore.Put(ctx, chunkData1) if err != nil { t.Fatalf("Expected no error got \"%v\"", err) } - chunkData2 := GenerateRandomChunk(int64(tt.chunkLength)).SData - key2, err := hasherStore.Put(context.TODO(), chunkData2) + chunkData2 := GenerateRandomChunk(int64(tt.chunkLength)).Data() + key2, err := hasherStore.Put(ctx, chunkData2) if err != nil { t.Fatalf("Expected no error got \"%v\"", err) } @@ -61,13 +63,13 @@ func TestHasherStore(t *testing.T) { hasherStore.Close() // Wait until chunks are really stored - err = hasherStore.Wait(context.TODO()) + err = hasherStore.Wait(ctx) if err != nil { t.Fatalf("Expected no error got \"%v\"", err) } // Get the first chunk - retrievedChunkData1, err := hasherStore.Get(context.TODO(), key1) + retrievedChunkData1, err := hasherStore.Get(ctx, key1) if err != nil { t.Fatalf("Expected no error, got \"%v\"", err) } @@ -78,7 +80,7 @@ func TestHasherStore(t *testing.T) { } // Get the second chunk - retrievedChunkData2, err := hasherStore.Get(context.TODO(), key2) + retrievedChunkData2, err := hasherStore.Get(ctx, key2) if err != nil { t.Fatalf("Expected no error, got \"%v\"", err) } @@ -105,12 +107,12 @@ func TestHasherStore(t *testing.T) { } // Check if chunk data in store is encrypted or not - chunkInStore, err := chunkStore.Get(context.TODO(), hash1) + chunkInStore, err := chunkStore.Get(ctx, hash1) if err != nil { t.Fatalf("Expected no error got \"%v\"", err) } - chunkDataInStore := chunkInStore.SData + chunkDataInStore := chunkInStore.Data() if tt.toEncrypt && bytes.Equal(chunkData1, chunkDataInStore) { t.Fatalf("Chunk expected to be encrypted but it is stored without encryption") diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 675b5de01a..8ab7e60b36 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -28,6 +28,7 @@ import ( "context" "encoding/binary" "encoding/hex" + "errors" "fmt" "io" "io/ioutil" @@ -36,7 +37,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" @@ -62,6 +63,10 @@ var ( keyDistanceCnt = byte(7) ) +var ( + ErrDBClosed = errors.New("LDBStore closed") +) + type gcItem struct { idx uint64 value uint64 @@ -99,18 +104,29 @@ type LDBStore struct { batchC chan bool batchesC chan struct{} - batch *leveldb.Batch + closed bool + batch *dbBatch lock sync.RWMutex quit chan struct{} // Functions encodeDataFunc is used to bypass // the default functionality of DbStore with // mock.NodeStore for testing purposes. - encodeDataFunc func(chunk *Chunk) []byte + encodeDataFunc func(chunk Chunk) []byte // If getDataFunc is defined, it will be used for // retrieving the chunk data instead from the local // LevelDB database. - getDataFunc func(addr Address) (data []byte, err error) + getDataFunc func(key Address) (data []byte, err error) +} + +type dbBatch struct { + *leveldb.Batch + err error + c chan struct{} +} + +func newBatch() *dbBatch { + return &dbBatch{Batch: new(leveldb.Batch), c: make(chan struct{})} } // TODO: Instead of passing the distance function, just pass the address from which distances are calculated @@ -121,10 +137,9 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { s.hashfunc = params.Hash s.quit = make(chan struct{}) - s.batchC = make(chan bool) s.batchesC = make(chan struct{}, 1) go s.writeBatches() - s.batch = new(leveldb.Batch) + s.batch = newBatch() // associate encodeData with default functionality s.encodeDataFunc = encodeData @@ -143,7 +158,6 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { k[1] = uint8(i) cnt, _ := s.db.Get(k) s.bucketCnt[i] = BytesToU64(cnt) - s.bucketCnt[i]++ } data, _ := s.db.Get(keyEntryCnt) s.entryCnt = BytesToU64(data) @@ -202,14 +216,6 @@ func getIndexKey(hash Address) []byte { return key } -func getOldDataKey(idx uint64) []byte { - key := make([]byte, 9) - key[0] = keyOldData - binary.BigEndian.PutUint64(key[1:9], idx) - - return key -} - func getDataKey(idx uint64, po uint8) []byte { key := make([]byte, 10) key[0] = keyData @@ -224,12 +230,12 @@ func encodeIndex(index *dpaDBIndex) []byte { return data } -func encodeData(chunk *Chunk) []byte { +func encodeData(chunk Chunk) []byte { // Always create a new underlying array for the returned byte slice. - // The chunk.Key array may be used in the returned slice which + // The chunk.Address array may be used in the returned slice which // may be changed later in the code or by the LevelDB, resulting - // that the Key is changed as well. - return append(append([]byte{}, chunk.Addr[:]...), chunk.SData...) + // that the Address is changed as well. + return append(append([]byte{}, chunk.Address()[:]...), chunk.Data()...) } func decodeIndex(data []byte, index *dpaDBIndex) error { @@ -237,14 +243,8 @@ func decodeIndex(data []byte, index *dpaDBIndex) error { return dec.Decode(index) } -func decodeData(data []byte, chunk *Chunk) { - chunk.SData = data[32:] - chunk.Size = int64(binary.BigEndian.Uint64(data[32:40])) -} - -func decodeOldData(data []byte, chunk *Chunk) { - chunk.SData = data - chunk.Size = int64(binary.BigEndian.Uint64(data[0:8])) +func decodeData(addr Address, data []byte) (*chunk, error) { + return NewChunk(addr, data[32:]), nil } func (s *LDBStore) collectGarbage(ratio float32) { @@ -347,44 +347,75 @@ func (s *LDBStore) Export(out io.Writer) (int64, error) { func (s *LDBStore) Import(in io.Reader) (int64, error) { tr := tar.NewReader(in) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + countC := make(chan int64) + errC := make(chan error) var count int64 - var wg sync.WaitGroup + go func() { + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + select { + case errC <- err: + case <-ctx.Done(): + } + } + + if len(hdr.Name) != 64 { + log.Warn("ignoring non-chunk file", "name", hdr.Name) + continue + } + + keybytes, err := hex.DecodeString(hdr.Name) + if err != nil { + log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) + continue + } + + data, err := ioutil.ReadAll(tr) + if err != nil { + select { + case errC <- err: + case <-ctx.Done(): + } + } + key := Address(keybytes) + chunk := NewChunk(key, data[32:]) + + go func() { + select { + case errC <- s.Put(ctx, chunk): + case <-ctx.Done(): + } + }() + + count++ + } + countC <- count + }() + + // wait for all chunks to be stored + i := int64(0) + var total int64 for { - hdr, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return count, err + select { + case err := <-errC: + if err != nil { + return count, err + } + i++ + case total = <-countC: + case <-ctx.Done(): + return i, ctx.Err() } - - if len(hdr.Name) != 64 { - log.Warn("ignoring non-chunk file", "name", hdr.Name) - continue + if total > 0 && i == total { + return total, nil } - - keybytes, err := hex.DecodeString(hdr.Name) - if err != nil { - log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) - continue - } - - data, err := ioutil.ReadAll(tr) - if err != nil { - return count, err - } - key := Address(keybytes) - chunk := NewChunk(key, nil) - chunk.SData = data[32:] - s.Put(context.TODO(), chunk) - wg.Add(1) - go func() { - defer wg.Done() - <-chunk.dbStoredC - }() - count++ } - wg.Wait() - return count, nil } func (s *LDBStore) Cleanup() { @@ -430,15 +461,18 @@ func (s *LDBStore) Cleanup() { } } - c := &Chunk{} ck := data[:32] - decodeData(data, c) + c, err := decodeData(ck, data) + if err != nil { + log.Error("decodeData error", "err", err) + continue + } - cs := int64(binary.LittleEndian.Uint64(c.SData[:8])) - log.Trace("chunk", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.SData), "size", cs) + cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) + log.Trace("chunk", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) - if len(c.SData) > chunk.DefaultSize+8 { - log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.SData), "size", cs) + if len(c.sdata) > ch.DefaultSize+8 { + log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) s.delete(index.Idx, getIndexKey(key[1:]), po) removed++ errorsFound++ @@ -511,7 +545,6 @@ func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { batch.Delete(getDataKey(idx, po)) s.entryCnt-- dbEntryCount.Dec(1) - s.bucketCnt[po]-- cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po @@ -520,10 +553,9 @@ func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { s.db.Write(batch) } -func (s *LDBStore) CurrentBucketStorageIndex(po uint8) uint64 { +func (s *LDBStore) BinIndex(po uint8) uint64 { s.lock.RLock() defer s.lock.RUnlock() - return s.bucketCnt[po] } @@ -539,43 +571,53 @@ func (s *LDBStore) CurrentStorageIndex() uint64 { return s.dataIdx } -func (s *LDBStore) Put(ctx context.Context, chunk *Chunk) { +func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { metrics.GetOrRegisterCounter("ldbstore.put", nil).Inc(1) - log.Trace("ldbstore.put", "key", chunk.Addr) + log.Trace("ldbstore.put", "key", chunk.Address()) - ikey := getIndexKey(chunk.Addr) + ikey := getIndexKey(chunk.Address()) var index dpaDBIndex - po := s.po(chunk.Addr) - s.lock.Lock() - defer s.lock.Unlock() + po := s.po(chunk.Address()) - log.Trace("ldbstore.put: s.db.Get", "key", chunk.Addr, "ikey", fmt.Sprintf("%x", ikey)) + s.lock.Lock() + + if s.closed { + s.lock.Unlock() + return ErrDBClosed + } + batch := s.batch + + log.Trace("ldbstore.put: s.db.Get", "key", chunk.Address(), "ikey", fmt.Sprintf("%x", ikey)) idata, err := s.db.Get(ikey) if err != nil { s.doPut(chunk, &index, po) - batchC := s.batchC - go func() { - <-batchC - chunk.markAsStored() - }() } else { - log.Trace("ldbstore.put: chunk already exists, only update access", "key", chunk.Addr) + log.Trace("ldbstore.put: chunk already exists, only update access", "key", chunk.Address) decodeIndex(idata, &index) - chunk.markAsStored() } index.Access = s.accessCnt s.accessCnt++ idata = encodeIndex(&index) s.batch.Put(ikey, idata) + + s.lock.Unlock() + select { case s.batchesC <- struct{}{}: default: } + + select { + case <-batch.c: + return batch.err + case <-ctx.Done(): + return ctx.Err() + } } // force putting into db, does not check access index -func (s *LDBStore) doPut(chunk *Chunk, index *dpaDBIndex, po uint8) { +func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { data := s.encodeDataFunc(chunk) dkey := getDataKey(s.dataIdx, po) s.batch.Put(dkey, data) @@ -592,58 +634,64 @@ func (s *LDBStore) doPut(chunk *Chunk, index *dpaDBIndex, po uint8) { } func (s *LDBStore) writeBatches() { -mainLoop: for { select { case <-s.quit: - break mainLoop + log.Debug("DbStore: quit batch write loop") + return case <-s.batchesC: - s.lock.Lock() - b := s.batch - e := s.entryCnt - d := s.dataIdx - a := s.accessCnt - c := s.batchC - s.batchC = make(chan bool) - s.batch = new(leveldb.Batch) - err := s.writeBatch(b, e, d, a) - // TODO: set this error on the batch, then tell the chunk + err := s.writeCurrentBatch() if err != nil { - log.Error(fmt.Sprintf("spawn batch write (%d entries): %v", b.Len(), err)) + log.Debug("DbStore: quit batch write loop", "err", err.Error()) + return } - close(c) - for e > s.capacity { - log.Trace("for >", "e", e, "s.capacity", s.capacity) - // Collect garbage in a separate goroutine - // to be able to interrupt this loop by s.quit. - done := make(chan struct{}) - go func() { - s.collectGarbage(gcArrayFreeRatio) - log.Trace("collectGarbage closing done") - close(done) - }() - - select { - case <-s.quit: - s.lock.Unlock() - break mainLoop - case <-done: - } - e = s.entryCnt - } - s.lock.Unlock() } } - log.Trace(fmt.Sprintf("DbStore: quit batch write loop")) + +} + +func (s *LDBStore) writeCurrentBatch() error { + s.lock.Lock() + defer s.lock.Unlock() + b := s.batch + l := b.Len() + if l == 0 { + return nil + } + e := s.entryCnt + d := s.dataIdx + a := s.accessCnt + s.batch = newBatch() + b.err = s.writeBatch(b, e, d, a) + close(b.c) + for e > s.capacity { + log.Trace("for >", "e", e, "s.capacity", s.capacity) + // Collect garbage in a separate goroutine + // to be able to interrupt this loop by s.quit. + done := make(chan struct{}) + go func() { + s.collectGarbage(gcArrayFreeRatio) + log.Trace("collectGarbage closing done") + close(done) + }() + + select { + case <-s.quit: + return errors.New("CollectGarbage terminated due to quit") + case <-done: + } + e = s.entryCnt + } + return nil } // must be called non concurrently -func (s *LDBStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) error { +func (s *LDBStore) writeBatch(b *dbBatch, entryCnt, dataIdx, accessCnt uint64) error { b.Put(keyEntryCnt, U64ToBytes(entryCnt)) b.Put(keyDataIdx, U64ToBytes(dataIdx)) b.Put(keyAccessCnt, U64ToBytes(accessCnt)) l := b.Len() - if err := s.db.Write(b); err != nil { + if err := s.db.Write(b.Batch); err != nil { return fmt.Errorf("unable to write batch: %v", err) } log.Trace(fmt.Sprintf("batch write (%d entries)", l)) @@ -654,12 +702,12 @@ func (s *LDBStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uin // to a mock store to bypass the default functionality encodeData. // The constructed function always returns the nil data, as DbStore does // not need to store the data, but still need to create the index. -func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk *Chunk) []byte { - return func(chunk *Chunk) []byte { - if err := mockStore.Put(chunk.Addr, encodeData(chunk)); err != nil { - log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Addr.Log(), err)) +func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { + return func(chunk Chunk) []byte { + if err := mockStore.Put(chunk.Address(), encodeData(chunk)); err != nil { + log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Address().Log(), err)) } - return chunk.Addr[:] + return chunk.Address()[:] } } @@ -682,7 +730,7 @@ func (s *LDBStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { return true } -func (s *LDBStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error) { +func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) { metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1) log.Trace("ldbstore.get", "key", addr) @@ -691,9 +739,11 @@ func (s *LDBStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err err return s.get(addr) } -func (s *LDBStore) get(addr Address) (chunk *Chunk, err error) { +func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { var indx dpaDBIndex - + if s.closed { + return nil, ErrDBClosed + } if s.tryAccessIdx(getIndexKey(addr), &indx) { var data []byte if s.getDataFunc != nil { @@ -716,9 +766,7 @@ func (s *LDBStore) get(addr Address) (chunk *Chunk, err error) { } } - chunk = NewChunk(addr, nil) - chunk.markAsStored() - decodeData(data, chunk) + return decodeData(addr, data) } else { err = ErrChunkNotFound } @@ -772,6 +820,12 @@ func (s *LDBStore) setCapacity(c uint64) { func (s *LDBStore) Close() { close(s.quit) + s.lock.Lock() + s.closed = true + s.lock.Unlock() + // force writing out current batch + s.writeCurrentBatch() + close(s.batchesC) s.db.Close() } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 9694a724e4..ae70ee2592 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -22,13 +22,12 @@ import ( "fmt" "io/ioutil" "os" - "sync" "testing" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" ldberrors "github.com/syndtr/goleveldb/leveldb/errors" @@ -86,70 +85,54 @@ func (db *testDbStore) close() { } } -func testDbStoreRandom(n int, processors int, chunksize int64, mock bool, t *testing.T) { +func testDbStoreRandom(n int, chunksize int64, mock bool, t *testing.T) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { t.Fatalf("init dbStore failed: %v", err) } - testStoreRandom(db, processors, n, chunksize, t) + testStoreRandom(db, n, chunksize, t) } -func testDbStoreCorrect(n int, processors int, chunksize int64, mock bool, t *testing.T) { +func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) { db, cleanup, err := newTestDbStore(mock, false) defer cleanup() if err != nil { t.Fatalf("init dbStore failed: %v", err) } - testStoreCorrect(db, processors, n, chunksize, t) + testStoreCorrect(db, n, chunksize, t) } func TestDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 1, 0, false, t) + testDbStoreRandom(1, 0, false, t) } func TestDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 1, 4096, false, t) + testDbStoreCorrect(1, 4096, false, t) } -func TestDbStoreRandom_1_5k(t *testing.T) { - testDbStoreRandom(8, 5000, 0, false, t) +func TestDbStoreRandom_5k(t *testing.T) { + testDbStoreRandom(5000, 0, false, t) } -func TestDbStoreRandom_8_5k(t *testing.T) { - testDbStoreRandom(8, 5000, 0, false, t) -} - -func TestDbStoreCorrect_1_5k(t *testing.T) { - testDbStoreCorrect(1, 5000, 4096, false, t) -} - -func TestDbStoreCorrect_8_5k(t *testing.T) { - testDbStoreCorrect(8, 5000, 4096, false, t) +func TestDbStoreCorrect_5k(t *testing.T) { + testDbStoreCorrect(5000, 4096, false, t) } func TestMockDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 1, 0, true, t) + testDbStoreRandom(1, 0, true, t) } func TestMockDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 1, 4096, true, t) + testDbStoreCorrect(1, 4096, true, t) } -func TestMockDbStoreRandom_1_5k(t *testing.T) { - testDbStoreRandom(8, 5000, 0, true, t) +func TestMockDbStoreRandom_5k(t *testing.T) { + testDbStoreRandom(5000, 0, true, t) } -func TestMockDbStoreRandom_8_5k(t *testing.T) { - testDbStoreRandom(8, 5000, 0, true, t) -} - -func TestMockDbStoreCorrect_1_5k(t *testing.T) { - testDbStoreCorrect(1, 5000, 4096, true, t) -} - -func TestMockDbStoreCorrect_8_5k(t *testing.T) { - testDbStoreCorrect(8, 5000, 4096, true, t) +func TestMockDbStoreCorrect_5k(t *testing.T) { + testDbStoreCorrect(5000, 4096, true, t) } func testDbStoreNotFound(t *testing.T, mock bool) { @@ -185,26 +168,19 @@ func testIterator(t *testing.T, mock bool) { t.Fatalf("init dbStore failed: %v", err) } - chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount) + chunks := GenerateRandomChunks(ch.DefaultSize, chunkcount) - wg := &sync.WaitGroup{} - wg.Add(len(chunks)) for i = 0; i < len(chunks); i++ { - db.Put(context.TODO(), chunks[i]) - chunkkeys[i] = chunks[i].Addr - j := i - go func() { - defer wg.Done() - <-chunks[j].dbStoredC - }() + chunkkeys[i] = chunks[i].Address() + err := db.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatalf("dbStore.Put failed: %v", err) + } } - //testSplit(m, l, 128, chunkkeys, t) - for i = 0; i < len(chunkkeys); i++ { log.Trace(fmt.Sprintf("Chunk array pos %d/%d: '%v'", i, chunkcount, chunkkeys[i])) } - wg.Wait() i = 0 for poc = 0; poc <= 255; poc++ { err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Address, n uint64) bool { @@ -239,7 +215,7 @@ func benchmarkDbStorePut(n int, processors int, chunksize int64, mock bool, b *t if err != nil { b.Fatalf("init dbStore failed: %v", err) } - benchmarkStorePut(db, processors, n, chunksize, b) + benchmarkStorePut(db, n, chunksize, b) } func benchmarkDbStoreGet(n int, processors int, chunksize int64, mock bool, b *testing.B) { @@ -248,7 +224,7 @@ func benchmarkDbStoreGet(n int, processors int, chunksize int64, mock bool, b *t if err != nil { b.Fatalf("init dbStore failed: %v", err) } - benchmarkStoreGet(db, processors, n, chunksize, b) + benchmarkStoreGet(db, n, chunksize, b) } func BenchmarkDbStorePut_1_500(b *testing.B) { @@ -293,35 +269,22 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks := []*Chunk{} - for i := 0; i < n; i++ { - c := GenerateRandomChunk(chunk.DefaultSize) - chunks = append(chunks, c) - log.Trace("generate random chunk", "idx", i, "chunk", c) - } - - for i := 0; i < n; i++ { - go ldb.Put(context.TODO(), chunks[i]) - } - - // wait for all chunks to be stored - for i := 0; i < n; i++ { - <-chunks[i].dbStoredC + chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) } log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) - for i := 0; i < n; i++ { - ret, err := ldb.Get(context.TODO(), chunks[i].Addr) + for _, ch := range chunks { + ret, err := ldb.Get(context.TODO(), ch.Address()) if err != nil { t.Fatal(err) } - if !bytes.Equal(ret.SData, chunks[i].SData) { + if !bytes.Equal(ret.Data(), ch.Data()) { t.Fatal("expected to get the same data back, but got smth else") } - - log.Info("got back chunk", "chunk", ret) } if ldb.entryCnt != uint64(n) { @@ -343,30 +306,18 @@ func TestLDBStoreCollectGarbage(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks := []*Chunk{} - for i := 0; i < n; i++ { - c := GenerateRandomChunk(chunk.DefaultSize) - chunks = append(chunks, c) - log.Trace("generate random chunk", "idx", i, "chunk", c) + chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) } - - for i := 0; i < n; i++ { - ldb.Put(context.TODO(), chunks[i]) - } - - // wait for all chunks to be stored - for i := 0; i < n; i++ { - <-chunks[i].dbStoredC - } - log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) // wait for garbage collection to kick in on the responsible actor time.Sleep(1 * time.Second) var missing int - for i := 0; i < n; i++ { - ret, err := ldb.Get(context.TODO(), chunks[i].Addr) + for _, ch := range chunks { + ret, err := ldb.Get(context.Background(), ch.Address()) if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { missing++ continue @@ -375,7 +326,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) { t.Fatal(err) } - if !bytes.Equal(ret.SData, chunks[i].SData) { + if !bytes.Equal(ret.Data(), ch.Data()) { t.Fatal("expected to get the same data back, but got smth else") } @@ -396,38 +347,27 @@ func TestLDBStoreAddRemove(t *testing.T) { defer cleanup() n := 100 - - chunks := []*Chunk{} - for i := 0; i < n; i++ { - c := GenerateRandomChunk(chunk.DefaultSize) - chunks = append(chunks, c) - log.Trace("generate random chunk", "idx", i, "chunk", c) - } - - for i := 0; i < n; i++ { - go ldb.Put(context.TODO(), chunks[i]) - } - - // wait for all chunks to be stored before continuing - for i := 0; i < n; i++ { - <-chunks[i].dbStoredC + chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + if err != nil { + t.Fatalf(err.Error()) } for i := 0; i < n; i++ { // delete all even index chunks if i%2 == 0 { - ldb.Delete(chunks[i].Addr) + ldb.Delete(chunks[i].Address()) } } log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) for i := 0; i < n; i++ { - ret, err := ldb.Get(context.TODO(), chunks[i].Addr) + ret, err := ldb.Get(nil, chunks[i].Address()) if i%2 == 0 { // expect even chunks to be missing - if err == nil || ret != nil { + if err == nil { + // if err != ErrChunkNotFound { t.Fatal("expected chunk to be missing, but got no error") } } else { @@ -436,7 +376,7 @@ func TestLDBStoreAddRemove(t *testing.T) { t.Fatalf("expected no error, but got %s", err) } - if !bytes.Equal(ret.SData, chunks[i].SData) { + if !bytes.Equal(ret.Data(), chunks[i].Data()) { t.Fatal("expected to get the same data back, but got smth else") } } @@ -446,15 +386,16 @@ func TestLDBStoreAddRemove(t *testing.T) { // TestLDBStoreRemoveThenCollectGarbage tests that we can delete chunks and that we can trigger garbage collection func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { capacity := 11 + surplus := 4 ldb, cleanup := newLDBStore(t) ldb.setCapacity(uint64(capacity)) - n := 11 + n := capacity - chunks := []*Chunk{} - for i := 0; i < capacity; i++ { - c := GenerateRandomChunk(chunk.DefaultSize) + chunks := []Chunk{} + for i := 0; i < n+surplus; i++ { + c := GenerateRandomChunk(ch.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -463,53 +404,54 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { ldb.Put(context.TODO(), chunks[i]) } - // wait for all chunks to be stored before continuing - for i := 0; i < n; i++ { - <-chunks[i].dbStoredC - } - // delete all chunks for i := 0; i < n; i++ { - ldb.Delete(chunks[i].Addr) + ldb.Delete(chunks[i].Address()) } log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) + if ldb.entryCnt != 0 { + t.Fatalf("ldb.entrCnt expected 0 got %v", ldb.entryCnt) + } + + expAccessCnt := uint64(n * 2) + if ldb.accessCnt != expAccessCnt { + t.Fatalf("ldb.accessCnt expected %v got %v", expAccessCnt, ldb.entryCnt) + } + cleanup() ldb, cleanup = newLDBStore(t) capacity = 10 ldb.setCapacity(uint64(capacity)) + defer cleanup() - n = 11 + n = capacity + surplus for i := 0; i < n; i++ { ldb.Put(context.TODO(), chunks[i]) } - // wait for all chunks to be stored before continuing - for i := 0; i < n; i++ { - <-chunks[i].dbStoredC - } - // wait for garbage collection time.Sleep(1 * time.Second) - // expect for first chunk to be missing, because it has the smallest access value - idx := 0 - ret, err := ldb.Get(context.TODO(), chunks[idx].Addr) - if err == nil || ret != nil { - t.Fatal("expected first chunk to be missing, but got no error") + // expect first surplus chunks to be missing, because they have the smallest access value + for i := 0; i < surplus; i++ { + _, err := ldb.Get(context.TODO(), chunks[i].Address()) + if err == nil { + t.Fatal("expected surplus chunk to be missing, but got no error") + } } - // expect for last chunk to be present, as it has the largest access value - idx = 9 - ret, err = ldb.Get(context.TODO(), chunks[idx].Addr) - if err != nil { - t.Fatalf("expected no error, but got %s", err) - } - - if !bytes.Equal(ret.SData, chunks[idx].SData) { - t.Fatal("expected to get the same data back, but got smth else") + // expect last chunks to be present, as they have the largest access value + for i := surplus; i < surplus+capacity; i++ { + ret, err := ldb.Get(context.TODO(), chunks[i].Address()) + if err != nil { + t.Fatalf("chunk %v: expected no error, but got %s", i, err) + } + if !bytes.Equal(ret.Data(), chunks[i].Data()) { + t.Fatal("expected to get the same data back, but got smth else") + } } } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 9e34749797..04701ee69a 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -18,8 +18,6 @@ package storage import ( "context" - "encoding/binary" - "fmt" "path/filepath" "sync" @@ -97,123 +95,89 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { // when the chunk is stored in memstore. // After the LDBStore.Put, it is ensured that the MemStore // contains the chunk with the same data, but nil ReqC channel. -func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk) { +func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { valid := true // ls.Validators contains a list of one validator per chunk type. // if one validator succeeds, then the chunk is valid for _, v := range ls.Validators { - if valid = v.Validate(chunk.Addr, chunk.SData); valid { + if valid = v.Validate(chunk.Address(), chunk.Data()); valid { break } } if !valid { - log.Trace("invalid chunk", "addr", chunk.Addr, "len", len(chunk.SData)) - chunk.SetErrored(ErrChunkInvalid) - chunk.markAsStored() - return + return ErrChunkInvalid } - log.Trace("localstore.put", "addr", chunk.Addr) - + log.Trace("localstore.put", "key", chunk.Address()) ls.mu.Lock() defer ls.mu.Unlock() - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - - memChunk, err := ls.memStore.Get(ctx, chunk.Addr) - switch err { - case nil: - if memChunk.ReqC == nil { - chunk.markAsStored() - return - } - case ErrChunkNotFound: - default: - chunk.SetErrored(err) - return + _, err := ls.memStore.Get(ctx, chunk.Address()) + if err == nil { + return nil } - - ls.DbStore.Put(ctx, chunk) - - // chunk is no longer a request, but a chunk with data, so replace it in memStore - newc := NewChunk(chunk.Addr, nil) - newc.SData = chunk.SData - newc.Size = chunk.Size - newc.dbStoredC = chunk.dbStoredC - - ls.memStore.Put(ctx, newc) - - if memChunk != nil && memChunk.ReqC != nil { - close(memChunk.ReqC) + if err != nil && err != ErrChunkNotFound { + return err } + ls.memStore.Put(ctx, chunk) + err = ls.DbStore.Put(ctx, chunk) + return err } // Get(chunk *Chunk) looks up a chunk in the local stores // This method is blocking until the chunk is retrieved // so additional timeout may be needed to wrap this call if // ChunkStores are remote and can have long latency -func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error) { +func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk Chunk, err error) { ls.mu.Lock() defer ls.mu.Unlock() return ls.get(ctx, addr) } -func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk *Chunk, err error) { +func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err error) { chunk, err = ls.memStore.Get(ctx, addr) - if err == nil { - if chunk.ReqC != nil { - select { - case <-chunk.ReqC: - default: - metrics.GetOrRegisterCounter("localstore.get.errfetching", nil).Inc(1) - return chunk, ErrFetching - } - } - metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) - return + + if err != nil && err != ErrChunkNotFound { + metrics.GetOrRegisterCounter("localstore.get.error", nil).Inc(1) + return nil, err } + + if err == nil { + metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) + return chunk, nil + } + metrics.GetOrRegisterCounter("localstore.get.cachemiss", nil).Inc(1) chunk, err = ls.DbStore.Get(ctx, addr) if err != nil { metrics.GetOrRegisterCounter("localstore.get.error", nil).Inc(1) - return + return nil, err } - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + ls.memStore.Put(ctx, chunk) - return + return chunk, nil } -// retrieve logic common for local and network chunk retrieval requests -func (ls *LocalStore) GetOrCreateRequest(ctx context.Context, addr Address) (chunk *Chunk, created bool) { - metrics.GetOrRegisterCounter("localstore.getorcreaterequest", nil).Inc(1) - +func (ls *LocalStore) FetchFunc(ctx context.Context, addr Address) func(context.Context) error { ls.mu.Lock() defer ls.mu.Unlock() - var err error - chunk, err = ls.get(ctx, addr) - if err == nil && chunk.GetErrored() == nil { - metrics.GetOrRegisterCounter("localstore.getorcreaterequest.hit", nil).Inc(1) - log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", addr)) - return chunk, false + _, err := ls.get(ctx, addr) + if err == nil { + return nil } - if err == ErrFetching && chunk.GetErrored() == nil { - metrics.GetOrRegisterCounter("localstore.getorcreaterequest.errfetching", nil).Inc(1) - log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", addr, chunk.ReqC)) - return chunk, false + return func(context.Context) error { + return err } - // no data and no request status - metrics.GetOrRegisterCounter("localstore.getorcreaterequest.miss", nil).Inc(1) - log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v not found locally. open new request", addr)) - chunk = NewChunk(addr, make(chan bool)) - ls.memStore.Put(ctx, chunk) - return chunk, true } -// RequestsCacheLen returns the current number of outgoing requests stored in the cache -func (ls *LocalStore) RequestsCacheLen() int { - return ls.memStore.requests.Len() +func (ls *LocalStore) BinIndex(po uint8) uint64 { + return ls.DbStore.BinIndex(po) +} + +func (ls *LocalStore) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error { + return ls.DbStore.SyncIterator(from, to, po, f) } // Close the local store diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index ae62218fe8..814d270d38 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -17,11 +17,12 @@ package storage import ( + "context" "io/ioutil" "os" "testing" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" ) var ( @@ -50,29 +51,29 @@ func TestValidator(t *testing.T) { chunks := GenerateRandomChunks(259, 2) goodChunk := chunks[0] badChunk := chunks[1] - copy(badChunk.SData, goodChunk.SData) + copy(badChunk.Data(), goodChunk.Data()) - PutChunks(store, goodChunk, badChunk) - if err := goodChunk.GetErrored(); err != nil { + errs := putChunks(store, goodChunk, badChunk) + if errs[0] != nil { t.Fatalf("expected no error on good content address chunk in spite of no validation, but got: %s", err) } - if err := badChunk.GetErrored(); err != nil { + if errs[1] != nil { t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err) } // add content address validator and check puts // bad should fail, good should pass store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc)) - chunks = GenerateRandomChunks(chunk.DefaultSize, 2) + chunks = GenerateRandomChunks(ch.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] - copy(badChunk.SData, goodChunk.SData) + copy(badChunk.Data(), goodChunk.Data()) - PutChunks(store, goodChunk, badChunk) - if err := goodChunk.GetErrored(); err != nil { + errs = putChunks(store, goodChunk, badChunk) + if errs[0] != nil { t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) } - if err := badChunk.GetErrored(); err == nil { + if errs[1] == nil { t.Fatal("expected error on bad content address chunk with content address validator only, but got nil") } @@ -81,16 +82,16 @@ func TestValidator(t *testing.T) { var negV boolTestValidator store.Validators = append(store.Validators, negV) - chunks = GenerateRandomChunks(chunk.DefaultSize, 2) + chunks = GenerateRandomChunks(ch.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] - copy(badChunk.SData, goodChunk.SData) + copy(badChunk.Data(), goodChunk.Data()) - PutChunks(store, goodChunk, badChunk) - if err := goodChunk.GetErrored(); err != nil { + errs = putChunks(store, goodChunk, badChunk) + if errs[0] != nil { t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) } - if err := badChunk.GetErrored(); err == nil { + if errs[1] == nil { t.Fatal("expected error on bad content address chunk with content address validator only, but got nil") } @@ -99,18 +100,19 @@ func TestValidator(t *testing.T) { var posV boolTestValidator = true store.Validators = append(store.Validators, posV) - chunks = GenerateRandomChunks(chunk.DefaultSize, 2) + chunks = GenerateRandomChunks(ch.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] - copy(badChunk.SData, goodChunk.SData) + copy(badChunk.Data(), goodChunk.Data()) - PutChunks(store, goodChunk, badChunk) - if err := goodChunk.GetErrored(); err != nil { + errs = putChunks(store, goodChunk, badChunk) + if errs[0] != nil { t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) } - if err := badChunk.GetErrored(); err != nil { - t.Fatalf("expected no error on bad content address chunk with content address validator only, but got: %s", err) + if errs[1] != nil { + t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err) } + } type boolTestValidator bool @@ -118,3 +120,27 @@ type boolTestValidator bool func (self boolTestValidator) Validate(addr Address, data []byte) bool { return bool(self) } + +// putChunks adds chunks to localstore +// It waits for receive on the stored channel +// It logs but does not fail on delivery error +func putChunks(store *LocalStore, chunks ...Chunk) []error { + i := 0 + f := func(n int64) Chunk { + chunk := chunks[i] + i++ + return chunk + } + _, errs := put(store, len(chunks), f) + return errs +} + +func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs []error) { + for i := int64(0); i < int64(n); i++ { + chunk := f(ch.DefaultSize) + err := store.Put(context.TODO(), chunk) + errs = append(errs, err) + hs = append(hs, chunk.Address()) + } + return hs, errs +} diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 55cfcbfeaf..36b1e00d9b 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -20,24 +20,17 @@ package storage import ( "context" - "sync" lru "github.com/hashicorp/golang-lru" ) type MemStore struct { cache *lru.Cache - requests *lru.Cache - mu sync.RWMutex disabled bool } -//NewMemStore is instantiating a MemStore cache. We are keeping a record of all outgoing requests for chunks, that -//should later be delivered by peer nodes, in the `requests` LRU cache. We are also keeping all frequently requested +//NewMemStore is instantiating a MemStore cache keeping all frequently requested //chunks in the `cache` LRU cache. -// -//`requests` LRU cache capacity should ideally never be reached, this is why for the time being it should be initialised -//with the same value as the LDBStore capacity. func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { if params.CacheCapacity == 0 { return &MemStore{ @@ -45,102 +38,48 @@ func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { } } - onEvicted := func(key interface{}, value interface{}) { - v := value.(*Chunk) - <-v.dbStoredC - } - c, err := lru.NewWithEvict(int(params.CacheCapacity), onEvicted) - if err != nil { - panic(err) - } - - requestEvicted := func(key interface{}, value interface{}) { - // temporary remove of the error log, until we figure out the problem, as it is too spamy - //log.Error("evict called on outgoing request") - } - r, err := lru.NewWithEvict(int(params.ChunkRequestsCacheCapacity), requestEvicted) + c, err := lru.New(int(params.CacheCapacity)) if err != nil { panic(err) } return &MemStore{ - cache: c, - requests: r, + cache: c, } } -func (m *MemStore) Get(ctx context.Context, addr Address) (*Chunk, error) { +func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { if m.disabled { return nil, ErrChunkNotFound } - m.mu.RLock() - defer m.mu.RUnlock() - - r, ok := m.requests.Get(string(addr)) - // it is a request - if ok { - return r.(*Chunk), nil - } - - // it is not a request c, ok := m.cache.Get(string(addr)) if !ok { return nil, ErrChunkNotFound } - return c.(*Chunk), nil + return c.(*chunk), nil } -func (m *MemStore) Put(ctx context.Context, c *Chunk) { +func (m *MemStore) Put(_ context.Context, c Chunk) error { if m.disabled { - return + return nil } - m.mu.Lock() - defer m.mu.Unlock() - - // it is a request - if c.ReqC != nil { - select { - case <-c.ReqC: - if c.GetErrored() != nil { - m.requests.Remove(string(c.Addr)) - return - } - m.cache.Add(string(c.Addr), c) - m.requests.Remove(string(c.Addr)) - default: - m.requests.Add(string(c.Addr), c) - } - return - } - - // it is not a request - m.cache.Add(string(c.Addr), c) - m.requests.Remove(string(c.Addr)) + m.cache.Add(string(c.Address()), c) + return nil } func (m *MemStore) setCapacity(n int) { if n <= 0 { m.disabled = true } else { - onEvicted := func(key interface{}, value interface{}) { - v := value.(*Chunk) - <-v.dbStoredC - } - c, err := lru.NewWithEvict(n, onEvicted) + c, err := lru.New(n) if err != nil { panic(err) } - r, err := lru.New(defaultChunkRequestsCacheCapacity) - if err != nil { - panic(err) - } - - m = &MemStore{ - cache: c, - requests: r, + *m = MemStore{ + cache: c, } } } diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 2c1b0e89e6..6b370d2b41 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -18,11 +18,6 @@ package storage import ( "context" - "crypto/rand" - "encoding/binary" - "io/ioutil" - "os" - "sync" "testing" "github.com/ethereum/go-ethereum/swarm/log" @@ -33,40 +28,32 @@ func newTestMemStore() *MemStore { return NewMemStore(storeparams, nil) } -func testMemStoreRandom(n int, processors int, chunksize int64, t *testing.T) { +func testMemStoreRandom(n int, chunksize int64, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreRandom(m, processors, n, chunksize, t) + testStoreRandom(m, n, chunksize, t) } -func testMemStoreCorrect(n int, processors int, chunksize int64, t *testing.T) { +func testMemStoreCorrect(n int, chunksize int64, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreCorrect(m, processors, n, chunksize, t) + testStoreCorrect(m, n, chunksize, t) } func TestMemStoreRandom_1(t *testing.T) { - testMemStoreRandom(1, 1, 0, t) + testMemStoreRandom(1, 0, t) } func TestMemStoreCorrect_1(t *testing.T) { - testMemStoreCorrect(1, 1, 4104, t) + testMemStoreCorrect(1, 4104, t) } -func TestMemStoreRandom_1_1k(t *testing.T) { - testMemStoreRandom(1, 1000, 0, t) +func TestMemStoreRandom_1k(t *testing.T) { + testMemStoreRandom(1000, 0, t) } -func TestMemStoreCorrect_1_1k(t *testing.T) { - testMemStoreCorrect(1, 100, 4096, t) -} - -func TestMemStoreRandom_8_1k(t *testing.T) { - testMemStoreRandom(8, 1000, 0, t) -} - -func TestMemStoreCorrect_8_1k(t *testing.T) { - testMemStoreCorrect(8, 1000, 4096, t) +func TestMemStoreCorrect_1k(t *testing.T) { + testMemStoreCorrect(100, 4096, t) } func TestMemStoreNotFound(t *testing.T) { @@ -82,13 +69,13 @@ func TestMemStoreNotFound(t *testing.T) { func benchmarkMemStorePut(n int, processors int, chunksize int64, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStorePut(m, processors, n, chunksize, b) + benchmarkStorePut(m, n, chunksize, b) } func benchmarkMemStoreGet(n int, processors int, chunksize int64, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStoreGet(m, processors, n, chunksize, b) + benchmarkStoreGet(m, n, chunksize, b) } func BenchmarkMemStorePut_1_500(b *testing.B) { @@ -107,104 +94,70 @@ func BenchmarkMemStoreGet_8_500(b *testing.B) { benchmarkMemStoreGet(500, 8, 4096, b) } -func newLDBStore(t *testing.T) (*LDBStore, func()) { - dir, err := ioutil.TempDir("", "bzz-storage-test") - if err != nil { - t.Fatal(err) - } - log.Trace("memstore.tempdir", "dir", dir) - - ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) - db, err := NewLDBStore(ldbparams) - if err != nil { - t.Fatal(err) - } - - cleanup := func() { - db.Close() - err := os.RemoveAll(dir) - if err != nil { - t.Fatal(err) - } - } - - return db, cleanup -} - func TestMemStoreAndLDBStore(t *testing.T) { ldb, cleanup := newLDBStore(t) ldb.setCapacity(4000) defer cleanup() cacheCap := 200 - requestsCap := 200 - memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil) + memStore := NewMemStore(NewStoreParams(4000, 200, nil, nil), nil) tests := []struct { - n int // number of chunks to push to memStore - chunkSize uint64 // size of chunk (by default in Swarm - 4096) - request bool // whether or not to set the ReqC channel on the random chunks + n int // number of chunks to push to memStore + chunkSize int64 // size of chunk (by default in Swarm - 4096) }{ { n: 1, chunkSize: 4096, - request: false, }, { n: 201, chunkSize: 4096, - request: false, }, { n: 501, chunkSize: 4096, - request: false, }, { n: 3100, chunkSize: 4096, - request: false, }, { n: 100, chunkSize: 4096, - request: true, }, } for i, tt := range tests { log.Info("running test", "idx", i, "tt", tt) - var chunks []*Chunk + var chunks []Chunk for i := 0; i < tt.n; i++ { - var c *Chunk - if tt.request { - c = NewRandomRequestChunk(tt.chunkSize) - } else { - c = NewRandomChunk(tt.chunkSize) - } - + c := GenerateRandomChunk(tt.chunkSize) chunks = append(chunks, c) } for i := 0; i < tt.n; i++ { - go ldb.Put(context.TODO(), chunks[i]) - memStore.Put(context.TODO(), chunks[i]) + err := ldb.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatal(err) + } + err = memStore.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatal(err) + } if got := memStore.cache.Len(); got > cacheCap { t.Fatalf("expected to get cache capacity less than %v, but got %v", cacheCap, got) } - if got := memStore.requests.Len(); got > requestsCap { - t.Fatalf("expected to get requests capacity less than %v, but got %v", requestsCap, got) - } } for i := 0; i < tt.n; i++ { - _, err := memStore.Get(context.TODO(), chunks[i].Addr) + _, err := memStore.Get(context.TODO(), chunks[i].Address()) if err != nil { if err == ErrChunkNotFound { - _, err := ldb.Get(context.TODO(), chunks[i].Addr) + _, err := ldb.Get(context.TODO(), chunks[i].Address()) if err != nil { t.Fatalf("couldn't get chunk %v from ldb, got error: %v", i, err) } @@ -213,37 +166,5 @@ func TestMemStoreAndLDBStore(t *testing.T) { } } } - - // wait for all chunks to be stored before ending the test are cleaning up - for i := 0; i < tt.n; i++ { - <-chunks[i].dbStoredC - } } } - -func NewRandomChunk(chunkSize uint64) *Chunk { - c := &Chunk{ - Addr: make([]byte, 32), - ReqC: nil, - SData: make([]byte, chunkSize+8), // SData should be chunkSize + 8 bytes reserved for length - dbStoredC: make(chan bool), - dbStoredMu: &sync.Mutex{}, - } - - rand.Read(c.SData) - - binary.LittleEndian.PutUint64(c.SData[:8], chunkSize) - - hasher := MakeHashFunc(SHA3Hash)() - hasher.Write(c.SData) - copy(c.Addr, hasher.Sum(nil)) - - return c -} - -func NewRandomRequestChunk(chunkSize uint64) *Chunk { - c := NewRandomChunk(chunkSize) - c.ReqC = make(chan bool) - - return c -} diff --git a/swarm/storage/mru/handler.go b/swarm/storage/mru/handler.go index 57561fd14b..18c667f14e 100644 --- a/swarm/storage/mru/handler.go +++ b/swarm/storage/mru/handler.go @@ -187,12 +187,12 @@ func (h *Handler) New(ctx context.Context, request *Request) error { return err } if request.metaHash != nil && !bytes.Equal(request.metaHash, metaHash) || - request.rootAddr != nil && !bytes.Equal(request.rootAddr, chunk.Addr) { + request.rootAddr != nil && !bytes.Equal(request.rootAddr, chunk.Address()) { return NewError(ErrInvalidValue, "metaHash in UpdateRequest does not match actual metadata") } request.metaHash = metaHash - request.rootAddr = chunk.Addr + request.rootAddr = chunk.Address() h.chunkStore.Put(ctx, chunk) log.Debug("new resource", "name", request.metadata.Name, "startTime", request.metadata.StartTime, "frequency", request.metadata.Frequency, "owner", request.metadata.Owner) @@ -202,14 +202,14 @@ func (h *Handler) New(ctx context.Context, request *Request) error { resourceUpdate: resourceUpdate{ updateHeader: updateHeader{ UpdateLookup: UpdateLookup{ - rootAddr: chunk.Addr, + rootAddr: chunk.Address(), }, }, }, ResourceMetadata: request.metadata, updated: time.Now(), } - h.set(chunk.Addr, rsrc) + h.set(chunk.Address(), rsrc) return nil } @@ -348,7 +348,11 @@ func (h *Handler) lookup(rsrc *resource, params *LookupParams) (*resource, error return nil, NewErrorf(ErrPeriodDepth, "Lookup exceeded max period hops (%d)", lp.Limit) } updateAddr := lp.UpdateAddr() - chunk, err := h.chunkStore.GetWithTimeout(context.TODO(), updateAddr, defaultRetrieveTimeout) + + ctx, cancel := context.WithTimeout(context.Background(), defaultRetrieveTimeout) + defer cancel() + + chunk, err := h.chunkStore.Get(ctx, updateAddr) if err == nil { if specificversion { return h.updateIndex(rsrc, chunk) @@ -358,7 +362,11 @@ func (h *Handler) lookup(rsrc *resource, params *LookupParams) (*resource, error for { newversion := lp.version + 1 updateAddr := lp.UpdateAddr() - newchunk, err := h.chunkStore.GetWithTimeout(context.TODO(), updateAddr, defaultRetrieveTimeout) + + ctx, cancel := context.WithTimeout(context.Background(), defaultRetrieveTimeout) + defer cancel() + + newchunk, err := h.chunkStore.Get(ctx, updateAddr) if err != nil { return h.updateIndex(rsrc, chunk) } @@ -380,7 +388,10 @@ func (h *Handler) lookup(rsrc *resource, params *LookupParams) (*resource, error // Load retrieves the Mutable Resource metadata chunk stored at rootAddr // Upon retrieval it creates/updates the index entry for it with metadata corresponding to the chunk contents func (h *Handler) Load(ctx context.Context, rootAddr storage.Address) (*resource, error) { - chunk, err := h.chunkStore.GetWithTimeout(ctx, rootAddr, defaultRetrieveTimeout) + //TODO: Maybe add timeout to context, defaultRetrieveTimeout? + ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout) + defer cancel() + chunk, err := h.chunkStore.Get(ctx, rootAddr) if err != nil { return nil, NewError(ErrNotFound, err.Error()) } @@ -388,11 +399,11 @@ func (h *Handler) Load(ctx context.Context, rootAddr storage.Address) (*resource // create the index entry rsrc := &resource{} - if err := rsrc.ResourceMetadata.binaryGet(chunk.SData); err != nil { // Will fail if this is not really a metadata chunk + if err := rsrc.ResourceMetadata.binaryGet(chunk.Data()); err != nil { // Will fail if this is not really a metadata chunk return nil, err } - rsrc.rootAddr, rsrc.metaHash = metadataHash(chunk.SData) + rsrc.rootAddr, rsrc.metaHash = metadataHash(chunk.Data()) if !bytes.Equal(rsrc.rootAddr, rootAddr) { return nil, NewError(ErrCorruptData, "Corrupt metadata chunk") } @@ -402,17 +413,17 @@ func (h *Handler) Load(ctx context.Context, rootAddr storage.Address) (*resource } // update mutable resource index map with specified content -func (h *Handler) updateIndex(rsrc *resource, chunk *storage.Chunk) (*resource, error) { +func (h *Handler) updateIndex(rsrc *resource, chunk storage.Chunk) (*resource, error) { // retrieve metadata from chunk data and check that it matches this mutable resource var r SignedResourceUpdate - if err := r.fromChunk(chunk.Addr, chunk.SData); err != nil { + if err := r.fromChunk(chunk.Address(), chunk.Data()); err != nil { return nil, err } - log.Trace("resource index update", "name", rsrc.ResourceMetadata.Name, "updatekey", chunk.Addr, "period", r.period, "version", r.version) + log.Trace("resource index update", "name", rsrc.ResourceMetadata.Name, "updatekey", chunk.Address(), "period", r.period, "version", r.version) // update our rsrcs entry map - rsrc.lastKey = chunk.Addr + rsrc.lastKey = chunk.Address() rsrc.period = r.period rsrc.version = r.version rsrc.updated = time.Now() @@ -420,8 +431,8 @@ func (h *Handler) updateIndex(rsrc *resource, chunk *storage.Chunk) (*resource, rsrc.multihash = r.multihash copy(rsrc.data, r.data) rsrc.Reader = bytes.NewReader(rsrc.data) - log.Debug("resource synced", "name", rsrc.ResourceMetadata.Name, "updateAddr", chunk.Addr, "period", rsrc.period, "version", rsrc.version) - h.set(chunk.Addr, rsrc) + log.Debug("resource synced", "name", rsrc.ResourceMetadata.Name, "updateAddr", chunk.Address(), "period", rsrc.period, "version", rsrc.version) + h.set(chunk.Address(), rsrc) return rsrc, nil } @@ -457,7 +468,7 @@ func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAd // send the chunk h.chunkStore.Put(ctx, chunk) - log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.SData, "multihash", r.multihash) + log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.Data(), "multihash", r.multihash) // update our resources map entry if the new update is older than the one we have, if we have it. if rsrc != nil && (r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version)) { @@ -475,7 +486,7 @@ func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAd // Retrieves the resource index value for the given nameHash func (h *Handler) get(rootAddr storage.Address) *resource { - if len(rootAddr) < storage.KeyLength { + if len(rootAddr) < storage.AddressLength { log.Warn("Handler.get with invalid rootAddr") return nil } @@ -488,7 +499,7 @@ func (h *Handler) get(rootAddr storage.Address) *resource { // Sets the resource index value for the given nameHash func (h *Handler) set(rootAddr storage.Address, rsrc *resource) { - if len(rootAddr) < storage.KeyLength { + if len(rootAddr) < storage.AddressLength { log.Warn("Handler.set with invalid rootAddr") return } diff --git a/swarm/storage/mru/lookup.go b/swarm/storage/mru/lookup.go index eb28336e12..b52cd5b4ff 100644 --- a/swarm/storage/mru/lookup.go +++ b/swarm/storage/mru/lookup.go @@ -72,7 +72,7 @@ type UpdateLookup struct { // 4 bytes period // 4 bytes version // storage.Keylength for rootAddr -const updateLookupLength = 4 + 4 + storage.KeyLength +const updateLookupLength = 4 + 4 + storage.AddressLength // UpdateAddr calculates the resource update chunk address corresponding to this lookup key func (u *UpdateLookup) UpdateAddr() (updateAddr storage.Address) { @@ -90,7 +90,7 @@ func (u *UpdateLookup) binaryPut(serializedData []byte) error { if len(serializedData) != updateLookupLength { return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize UpdateLookup. Expected %d, got %d", updateLookupLength, len(serializedData)) } - if len(u.rootAddr) != storage.KeyLength { + if len(u.rootAddr) != storage.AddressLength { return NewError(ErrInvalidValue, "UpdateLookup.binaryPut called without rootAddr set") } binary.LittleEndian.PutUint32(serializedData[:4], u.period) @@ -111,7 +111,7 @@ func (u *UpdateLookup) binaryGet(serializedData []byte) error { } u.period = binary.LittleEndian.Uint32(serializedData[:4]) u.version = binary.LittleEndian.Uint32(serializedData[4:8]) - u.rootAddr = storage.Address(make([]byte, storage.KeyLength)) + u.rootAddr = storage.Address(make([]byte, storage.AddressLength)) copy(u.rootAddr[:], serializedData[8:]) return nil } diff --git a/swarm/storage/mru/metadata.go b/swarm/storage/mru/metadata.go index 0ab0ed1d9e..5091148959 100644 --- a/swarm/storage/mru/metadata.go +++ b/swarm/storage/mru/metadata.go @@ -142,7 +142,7 @@ func (r *ResourceMetadata) serializeAndHash() (rootAddr, metaHash []byte, chunkD } // creates a metadata chunk out of a resourceMetadata structure -func (metadata *ResourceMetadata) newChunk() (chunk *storage.Chunk, metaHash []byte, err error) { +func (metadata *ResourceMetadata) newChunk() (chunk storage.Chunk, metaHash []byte, err error) { // the metadata chunk contains a timestamp of when the resource starts to be valid // and also how frequently it is expected to be updated // from this we know at what time we should look for updates, and how often @@ -157,9 +157,7 @@ func (metadata *ResourceMetadata) newChunk() (chunk *storage.Chunk, metaHash []b } // make the chunk and send it to swarm - chunk = storage.NewChunk(rootAddr, nil) - chunk.SData = chunkData - chunk.Size = int64(len(chunkData)) + chunk = storage.NewChunk(rootAddr, chunkData) return chunk, metaHash, nil } diff --git a/swarm/storage/mru/request.go b/swarm/storage/mru/request.go index dd71f855d6..af2ccf5c76 100644 --- a/swarm/storage/mru/request.go +++ b/swarm/storage/mru/request.go @@ -182,7 +182,7 @@ func (r *Request) fromJSON(j *updateRequestJSON) error { var declaredRootAddr storage.Address var declaredMetaHash []byte - declaredRootAddr, err = decodeHexSlice(j.RootAddr, storage.KeyLength, "rootAddr") + declaredRootAddr, err = decodeHexSlice(j.RootAddr, storage.AddressLength, "rootAddr") if err != nil { return err } diff --git a/swarm/storage/mru/resource_test.go b/swarm/storage/mru/resource_test.go index 76d7c58a1e..0fb465bb0f 100644 --- a/swarm/storage/mru/resource_test.go +++ b/swarm/storage/mru/resource_test.go @@ -87,8 +87,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { resourceUpdate: resourceUpdate{ updateHeader: updateHeader{ UpdateLookup: UpdateLookup{ - - rootAddr: make([]byte, 79), // put the wrong length, should be storage.KeyLength + rootAddr: make([]byte, 79), // put the wrong length, should be storage.AddressLength }, metaHash: nil, multihash: false, @@ -99,8 +98,8 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { if err == nil { t.Fatal("Expected newUpdateChunk to fail when rootAddr or metaHash have the wrong length") } - r.rootAddr = make([]byte, storage.KeyLength) - r.metaHash = make([]byte, storage.KeyLength) + r.rootAddr = make([]byte, storage.AddressLength) + r.metaHash = make([]byte, storage.AddressLength) _, err = r.toChunk() if err == nil { t.Fatal("Expected newUpdateChunk to fail when there is no data") @@ -197,7 +196,7 @@ func TestReverse(t *testing.T) { // check that we can recover the owner account from the update chunk's signature var checkUpdate SignedResourceUpdate - if err := checkUpdate.fromChunk(chunk.Addr, chunk.SData); err != nil { + if err := checkUpdate.fromChunk(chunk.Address(), chunk.Data()); err != nil { t.Fatal(err) } checkdigest, err := checkUpdate.GetDigest() @@ -215,8 +214,8 @@ func TestReverse(t *testing.T) { t.Fatalf("addresses dont match: %x != %x", originaladdress, recoveredaddress) } - if !bytes.Equal(key[:], chunk.Addr[:]) { - t.Fatalf("Expected chunk key '%x', was '%x'", key, chunk.Addr) + if !bytes.Equal(key[:], chunk.Address()[:]) { + t.Fatalf("Expected chunk key '%x', was '%x'", key, chunk.Address()) } if period != checkUpdate.period { t.Fatalf("Expected period '%d', was '%d'", period, checkUpdate.period) @@ -270,16 +269,16 @@ func TestResourceHandler(t *testing.T) { t.Fatal(err) } - chunk, err := rh.chunkStore.Get(context.TODO(), storage.Address(request.rootAddr)) + chunk, err := rh.chunkStore.Get(ctx, storage.Address(request.rootAddr)) if err != nil { t.Fatal(err) - } else if len(chunk.SData) < 16 { - t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData)) + } else if len(chunk.Data()) < 16 { + t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.Data())) } var recoveredMetadata ResourceMetadata - recoveredMetadata.binaryGet(chunk.SData) + recoveredMetadata.binaryGet(chunk.Data()) if err != nil { t.Fatal(err) } @@ -704,7 +703,7 @@ func TestValidator(t *testing.T) { if err != nil { t.Fatal(err) } - if !rh.Validate(chunk.Addr, chunk.SData) { + if !rh.Validate(chunk.Address(), chunk.Data()) { t.Fatal("Chunk validator fail on update chunk") } @@ -724,7 +723,7 @@ func TestValidator(t *testing.T) { t.Fatal(err) } - if rh.Validate(chunk.Addr, chunk.SData) { + if rh.Validate(chunk.Address(), chunk.Data()) { t.Fatal("Chunk validator did not fail on update chunk with false address") } @@ -742,7 +741,7 @@ func TestValidator(t *testing.T) { t.Fatal(err) } - if !rh.Validate(chunk.Addr, chunk.SData) { + if !rh.Validate(chunk.Address(), chunk.Data()) { t.Fatal("Chunk validator fail on metadata chunk") } } @@ -783,8 +782,7 @@ func TestValidatorInStore(t *testing.T) { // create content addressed chunks, one good, one faulty chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk := chunks[0] - badChunk := chunks[1] - badChunk.SData = goodChunk.SData + badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data()) metadata := &ResourceMetadata{ StartTime: startTime, @@ -801,7 +799,7 @@ func TestValidatorInStore(t *testing.T) { updateLookup := UpdateLookup{ period: 42, version: 1, - rootAddr: rootChunk.Addr, + rootAddr: rootChunk.Address(), } updateAddr := updateLookup.UpdateAddr() @@ -826,16 +824,16 @@ func TestValidatorInStore(t *testing.T) { } // put the chunks in the store and check their error status - storage.PutChunks(store, goodChunk) - if goodChunk.GetErrored() == nil { + err = store.Put(context.Background(), goodChunk) + if err == nil { t.Fatal("expected error on good content address chunk with resource validator only, but got nil") } - storage.PutChunks(store, badChunk) - if badChunk.GetErrored() == nil { + err = store.Put(context.Background(), badChunk) + if err == nil { t.Fatal("expected error on bad content address chunk with resource validator only, but got nil") } - storage.PutChunks(store, uglyChunk) - if err := uglyChunk.GetErrored(); err != nil { + err = store.Put(context.Background(), uglyChunk) + if err != nil { t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err) } } @@ -897,7 +895,7 @@ func getUpdateDirect(rh *Handler, addr storage.Address) ([]byte, error) { return nil, err } var r SignedResourceUpdate - if err := r.fromChunk(addr, chunk.SData); err != nil { + if err := r.fromChunk(addr, chunk.Data()); err != nil { return nil, err } return r.data, nil diff --git a/swarm/storage/mru/signedupdate.go b/swarm/storage/mru/signedupdate.go index 1c6d02e82a..41a5a5e631 100644 --- a/swarm/storage/mru/signedupdate.go +++ b/swarm/storage/mru/signedupdate.go @@ -96,7 +96,7 @@ func (r *SignedResourceUpdate) Sign(signer Signer) error { } // create an update chunk. -func (r *SignedResourceUpdate) toChunk() (*storage.Chunk, error) { +func (r *SignedResourceUpdate) toChunk() (storage.Chunk, error) { // Check that the update is signed and serialized // For efficiency, data is serialized during signature and cached in @@ -105,14 +105,11 @@ func (r *SignedResourceUpdate) toChunk() (*storage.Chunk, error) { return nil, NewError(ErrInvalidSignature, "newUpdateChunk called without a valid signature or payload data. Call .Sign() first.") } - chunk := storage.NewChunk(r.updateAddr, nil) resourceUpdateLength := r.resourceUpdate.binaryLength() - chunk.SData = r.binaryData - // signature is the last item in the chunk data - copy(chunk.SData[resourceUpdateLength:], r.signature[:]) + copy(r.binaryData[resourceUpdateLength:], r.signature[:]) - chunk.Size = int64(len(chunk.SData)) + chunk := storage.NewChunk(r.updateAddr, r.binaryData) return chunk, nil } diff --git a/swarm/storage/mru/testutil.go b/swarm/storage/mru/testutil.go index 6efcba9aba..a30baaa1d7 100644 --- a/swarm/storage/mru/testutil.go +++ b/swarm/storage/mru/testutil.go @@ -17,8 +17,12 @@ package mru import ( + "context" "fmt" "path/filepath" + "sync" + + "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -35,6 +39,17 @@ func (t *TestHandler) Close() { t.chunkStore.Close() } +type mockNetFetcher struct{} + +func (m *mockNetFetcher) Request(ctx context.Context) { +} +func (m *mockNetFetcher) Offer(ctx context.Context, source *discover.NodeID) { +} + +func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetFetcher { + return &mockNetFetcher{} +} + // NewTestHandler creates Handler object to be used for testing purposes. func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) { path := filepath.Join(datadir, testDbDirName) @@ -47,7 +62,11 @@ func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) } localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(resourceHashAlgorithm))) localStore.Validators = append(localStore.Validators, rh) - netStore := storage.NewNetStore(localStore, nil) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, err + } + netStore.NewNetFetcherFunc = newFakeNetFetcher rh.SetStore(netStore) return &TestHandler{rh}, nil } diff --git a/swarm/storage/mru/updateheader.go b/swarm/storage/mru/updateheader.go index 3ac20c1890..f0039eaf66 100644 --- a/swarm/storage/mru/updateheader.go +++ b/swarm/storage/mru/updateheader.go @@ -27,7 +27,7 @@ type updateHeader struct { metaHash []byte // SHA3 hash of the metadata chunk (less ownerAddr). Used to prove ownerhsip of the resource. } -const metaHashLength = storage.KeyLength +const metaHashLength = storage.AddressLength // updateLookupLength bytes // 1 byte flags (multihash bool for now) @@ -76,7 +76,7 @@ func (h *updateHeader) binaryGet(serializedData []byte) error { } cursor := updateLookupLength h.metaHash = make([]byte, metaHashLength) - copy(h.metaHash[:storage.KeyLength], serializedData[cursor:cursor+storage.KeyLength]) + copy(h.metaHash[:storage.AddressLength], serializedData[cursor:cursor+storage.AddressLength]) cursor += metaHashLength flags := serializedData[cursor] diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 96a7e51f77..de2d82d2b7 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -18,181 +18,275 @@ package storage import ( "context" + "encoding/hex" + "fmt" + "sync" + "sync/atomic" "time" + "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/spancontext" - opentracing "github.com/opentracing/opentracing-go" + + lru "github.com/hashicorp/golang-lru" ) -var ( - // NetStore.Get timeout for get and get retries - // This is the maximum period that the Get will block. - // If it is reached, Get will return ErrChunkNotFound. - netStoreRetryTimeout = 30 * time.Second - // Minimal period between calling get method on NetStore - // on retry. It protects calling get very frequently if - // it returns ErrChunkNotFound very fast. - netStoreMinRetryDelay = 3 * time.Second - // Timeout interval before retrieval is timed out. - // It is used in NetStore.get on waiting for ReqC to be - // closed on a single retrieve request. - searchTimeout = 10 * time.Second +type ( + NewNetFetcherFunc func(ctx context.Context, addr Address, peers *sync.Map) NetFetcher ) -// NetStore implements the ChunkStore interface, -// this chunk access layer assumed 2 chunk stores -// local storage eg. LocalStore and network storage eg., NetStore -// access by calling network is blocking with a timeout +type NetFetcher interface { + Request(ctx context.Context) + Offer(ctx context.Context, source *discover.NodeID) +} + +// NetStore is an extension of local storage +// it implements the ChunkStore interface +// on request it initiates remote cloud retrieval using a fetcher +// fetchers are unique to a chunk and are stored in fetchers LRU memory cache +// fetchFuncFactory is a factory object to create a fetch function for a specific chunk address type NetStore struct { - localStore *LocalStore - retrieve func(ctx context.Context, chunk *Chunk) error + mu sync.Mutex + store SyncChunkStore + fetchers *lru.Cache + NewNetFetcherFunc NewNetFetcherFunc + closeC chan struct{} } -func NewNetStore(localStore *LocalStore, retrieve func(ctx context.Context, chunk *Chunk) error) *NetStore { - return &NetStore{localStore, retrieve} +// NewNetStore creates a new NetStore object using the given local store. newFetchFunc is a +// constructor function that can create a fetch function for a specific chunk address. +func NewNetStore(store SyncChunkStore, nnf NewNetFetcherFunc) (*NetStore, error) { + fetchers, err := lru.New(defaultChunkRequestsCacheCapacity) + if err != nil { + return nil, err + } + return &NetStore{ + store: store, + fetchers: fetchers, + NewNetFetcherFunc: nnf, + closeC: make(chan struct{}), + }, nil } -// Get is the entrypoint for local retrieve requests -// waits for response or times out -// -// Get uses get method to retrieve request, but retries if the -// ErrChunkNotFound is returned by get, until the netStoreRetryTimeout -// is reached. -func (ns *NetStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error) { +// Put stores a chunk in localstore, and delivers to all requestor peers using the fetcher stored in +// the fetchers cache +func (n *NetStore) Put(ctx context.Context, ch Chunk) error { + n.mu.Lock() + defer n.mu.Unlock() - var sp opentracing.Span - ctx, sp = spancontext.StartSpan( - ctx, - "netstore.get.global") - defer sp.Finish() - - timer := time.NewTimer(netStoreRetryTimeout) - defer timer.Stop() - - // result and resultC provide results from the goroutine - // where NetStore.get is called. - type result struct { - chunk *Chunk - err error + // put to the chunk to the store, there should be no error + err := n.store.Put(ctx, ch) + if err != nil { + return err } - resultC := make(chan result) - // quitC ensures that retring goroutine is terminated - // when this function returns. - quitC := make(chan struct{}) - defer close(quitC) - - // do retries in a goroutine so that the timer can - // force this method to return after the netStoreRetryTimeout. - go func() { - // limiter ensures that NetStore.get is not called more frequently - // then netStoreMinRetryDelay. If NetStore.get takes longer - // then netStoreMinRetryDelay, the next retry call will be - // without a delay. - limiter := time.NewTimer(netStoreMinRetryDelay) - defer limiter.Stop() - - for { - chunk, err := ns.get(ctx, addr, 0) - if err != ErrChunkNotFound { - // break retry only if the error is nil - // or other error then ErrChunkNotFound - select { - case <-quitC: - // Maybe NetStore.Get function has returned - // by the timer.C while we were waiting for the - // results. Terminate this goroutine. - case resultC <- result{chunk: chunk, err: err}: - // Send the result to the parrent goroutine. - } - return - - } - select { - case <-quitC: - // NetStore.Get function has returned, possibly - // by the timer.C, which makes this goroutine - // not needed. - return - case <-limiter.C: - } - // Reset the limiter for the next iteration. - limiter.Reset(netStoreMinRetryDelay) - log.Debug("NetStore.Get retry chunk", "key", addr) - } - }() - - select { - case r := <-resultC: - return r.chunk, r.err - case <-timer.C: - return nil, ErrChunkNotFound + // if chunk is now put in the store, check if there was an active fetcher and call deliver on it + // (this delivers the chunk to requestors via the fetcher) + if f := n.getFetcher(ch.Address()); f != nil { + f.deliver(ctx, ch) } + return nil } -// GetWithTimeout makes a single retrieval attempt for a chunk with a explicit timeout parameter -func (ns *NetStore) GetWithTimeout(ctx context.Context, addr Address, timeout time.Duration) (chunk *Chunk, err error) { - return ns.get(ctx, addr, timeout) +// Get retrieves the chunk from the NetStore DPA synchronously. +// It calls NetStore.get, and if the chunk is not in local Storage +// it calls fetch with the request, which blocks until the chunk +// arrived or context is done +func (n *NetStore) Get(rctx context.Context, ref Address) (Chunk, error) { + chunk, fetch, err := n.get(rctx, ref) + if err != nil { + return nil, err + } + if chunk != nil { + return chunk, nil + } + return fetch(rctx) } -func (ns *NetStore) get(ctx context.Context, addr Address, timeout time.Duration) (chunk *Chunk, err error) { - if timeout == 0 { - timeout = searchTimeout - } - - var sp opentracing.Span - ctx, sp = spancontext.StartSpan( - ctx, - "netstore.get") - defer sp.Finish() - - if ns.retrieve == nil { - chunk, err = ns.localStore.Get(ctx, addr) - if err == nil { - return chunk, nil - } - if err != ErrFetching { - return nil, err - } - } else { - var created bool - chunk, created = ns.localStore.GetOrCreateRequest(ctx, addr) - - if chunk.ReqC == nil { - return chunk, nil - } - - if created { - err := ns.retrieve(ctx, chunk) - if err != nil { - // mark chunk request as failed so that we can retry it later - chunk.SetErrored(ErrChunkUnavailable) - return nil, err - } - } - } - - t := time.NewTicker(timeout) - defer t.Stop() - - select { - case <-t.C: - // mark chunk request as failed so that we can retry - chunk.SetErrored(ErrChunkNotFound) - return nil, ErrChunkNotFound - case <-chunk.ReqC: - } - chunk.SetErrored(nil) - return chunk, nil +func (n *NetStore) BinIndex(po uint8) uint64 { + return n.store.BinIndex(po) } -// Put is the entrypoint for local store requests coming from storeLoop -func (ns *NetStore) Put(ctx context.Context, chunk *Chunk) { - ns.localStore.Put(ctx, chunk) +func (n *NetStore) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error { + return n.store.Iterator(from, to, po, f) +} + +// FetchFunc returns nil if the store contains the given address. Otherwise it returns a wait function, +// which returns after the chunk is available or the context is done +func (n *NetStore) FetchFunc(ctx context.Context, ref Address) func(context.Context) error { + chunk, fetch, _ := n.get(ctx, ref) + if chunk != nil { + return nil + } + return func(ctx context.Context) error { + _, err := fetch(ctx) + return err + } } // Close chunk store -func (ns *NetStore) Close() { - ns.localStore.Close() +func (n *NetStore) Close() { + close(n.closeC) + n.store.Close() + // TODO: loop through fetchers to cancel them +} + +// get attempts at retrieving the chunk from LocalStore +// If it is not found then using getOrCreateFetcher: +// 1. Either there is already a fetcher to retrieve it +// 2. A new fetcher is created and saved in the fetchers cache +// From here on, all Get will hit on this fetcher until the chunk is delivered +// or all fetcher contexts are done. +// It returns a chunk, a fetcher function and an error +// If chunk is nil, the returned fetch function needs to be called with a context to return the chunk. +func (n *NetStore) get(ctx context.Context, ref Address) (Chunk, func(context.Context) (Chunk, error), error) { + n.mu.Lock() + defer n.mu.Unlock() + + chunk, err := n.store.Get(ctx, ref) + if err != nil { + if err != ErrChunkNotFound { + log.Debug("Received error from LocalStore other than ErrNotFound", "err", err) + } + // The chunk is not available in the LocalStore, let's get the fetcher for it, or create a new one + // if it doesn't exist yet + f := n.getOrCreateFetcher(ref) + // If the caller needs the chunk, it has to use the returned fetch function to get it + return nil, f.Fetch, nil + } + + return chunk, nil, nil +} + +// getOrCreateFetcher attempts at retrieving an existing fetchers +// if none exists, creates one and saves it in the fetchers cache +// caller must hold the lock +func (n *NetStore) getOrCreateFetcher(ref Address) *fetcher { + if f := n.getFetcher(ref); f != nil { + return f + } + + // no fetcher for the given address, we have to create a new one + key := hex.EncodeToString(ref) + // create the context during which fetching is kept alive + ctx, cancel := context.WithCancel(context.Background()) + // destroy is called when all requests finish + destroy := func() { + // remove fetcher from fetchers + n.fetchers.Remove(key) + // stop fetcher by cancelling context called when + // all requests cancelled/timedout or chunk is delivered + cancel() + } + // peers always stores all the peers which have an active request for the chunk. It is shared + // between fetcher and the NewFetchFunc function. It is needed by the NewFetchFunc because + // the peers which requested the chunk should not be requested to deliver it. + peers := &sync.Map{} + + fetcher := newFetcher(ref, n.NewNetFetcherFunc(ctx, ref, peers), destroy, peers, n.closeC) + n.fetchers.Add(key, fetcher) + + return fetcher +} + +// getFetcher retrieves the fetcher for the given address from the fetchers cache if it exists, +// otherwise it returns nil +func (n *NetStore) getFetcher(ref Address) *fetcher { + key := hex.EncodeToString(ref) + f, ok := n.fetchers.Get(key) + if ok { + return f.(*fetcher) + } + return nil +} + +// RequestsCacheLen returns the current number of outgoing requests stored in the cache +func (n *NetStore) RequestsCacheLen() int { + return n.fetchers.Len() +} + +// One fetcher object is responsible to fetch one chunk for one address, and keep track of all the +// peers who have requested it and did not receive it yet. +type fetcher struct { + addr Address // address of chunk + chunk Chunk // fetcher can set the chunk on the fetcher + deliveredC chan struct{} // chan signalling chunk delivery to requests + cancelledC chan struct{} // chan signalling the fetcher has been cancelled (removed from fetchers in NetStore) + netFetcher NetFetcher // remote fetch function to be called with a request source taken from the context + cancel func() // cleanup function for the remote fetcher to call when all upstream contexts are called + peers *sync.Map // the peers which asked for the chunk + requestCnt int32 // number of requests on this chunk. If all the requests are done (delivered or context is done) the cancel function is called + deliverOnce *sync.Once // guarantees that we only close deliveredC once +} + +// newFetcher creates a new fetcher object for the fiven addr. fetch is the function which actually +// does the retrieval (in non-test cases this is coming from the network package). cancel function is +// called either +// 1. when the chunk has been fetched all peers have been either notified or their context has been done +// 2. the chunk has not been fetched but all context from all the requests has been done +// The peers map stores all the peers which have requested chunk. +func newFetcher(addr Address, nf NetFetcher, cancel func(), peers *sync.Map, closeC chan struct{}) *fetcher { + cancelOnce := &sync.Once{} // cancel should only be called once + return &fetcher{ + addr: addr, + deliveredC: make(chan struct{}), + deliverOnce: &sync.Once{}, + cancelledC: closeC, + netFetcher: nf, + cancel: func() { + cancelOnce.Do(func() { + cancel() + }) + }, + peers: peers, + } +} + +// Fetch fetches the chunk synchronously, it is called by NetStore.Get is the chunk is not available +// locally. +func (f *fetcher) Fetch(rctx context.Context) (Chunk, error) { + atomic.AddInt32(&f.requestCnt, 1) + defer func() { + // if all the requests are done the fetcher can be cancelled + if atomic.AddInt32(&f.requestCnt, -1) == 0 { + f.cancel() + } + }() + + // The peer asking for the chunk. Store in the shared peers map, but delete after the request + // has been delivered + peer := rctx.Value("peer") + if peer != nil { + f.peers.Store(peer, time.Now()) + defer f.peers.Delete(peer) + } + + // If there is a source in the context then it is an offer, otherwise a request + sourceIF := rctx.Value("source") + if sourceIF != nil { + var source *discover.NodeID + id := discover.MustHexID(sourceIF.(string)) + source = &id + f.netFetcher.Offer(rctx, source) + } else { + f.netFetcher.Request(rctx) + } + + // wait until either the chunk is delivered or the context is done + select { + case <-rctx.Done(): + return nil, rctx.Err() + case <-f.deliveredC: + return f.chunk, nil + case <-f.cancelledC: + return nil, fmt.Errorf("fetcher cancelled") + } +} + +// deliver is called by NetStore.Put to notify all pending requests +func (f *fetcher) deliver(ctx context.Context, ch Chunk) { + f.deliverOnce.Do(func() { + f.chunk = ch + // closing the deliveredC channel will terminate ongoing requests + close(f.deliveredC) + }) } diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index 7babbf5e0c..f08968f0ea 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -17,107 +17,622 @@ package storage import ( + "bytes" "context" - "encoding/hex" - "errors" + "crypto/rand" "io/ioutil" + "sync" "testing" "time" - "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/p2p/discover" + ch "github.com/ethereum/go-ethereum/swarm/chunk" + + "github.com/ethereum/go-ethereum/common" ) -var ( - errUnknown = errors.New("unknown error") -) +var sourcePeerID = discover.MustHexID("2dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439") -type mockRetrieve struct { - requests map[string]int +type mockNetFetcher struct { + peers *sync.Map + sources []*discover.NodeID + peersPerRequest [][]Address + requestCalled bool + offerCalled bool + quit <-chan struct{} + ctx context.Context } -func NewMockRetrieve() *mockRetrieve { - return &mockRetrieve{requests: make(map[string]int)} +func (m *mockNetFetcher) Offer(ctx context.Context, source *discover.NodeID) { + m.offerCalled = true + m.sources = append(m.sources, source) } -func newDummyChunk(addr Address) *Chunk { - chunk := NewChunk(addr, make(chan bool)) - chunk.SData = []byte{3, 4, 5} - chunk.Size = 3 - - return chunk +func (m *mockNetFetcher) Request(ctx context.Context) { + m.requestCalled = true + var peers []Address + m.peers.Range(func(key interface{}, _ interface{}) bool { + peers = append(peers, common.FromHex(key.(string))) + return true + }) + m.peersPerRequest = append(m.peersPerRequest, peers) } -func (m *mockRetrieve) retrieve(ctx context.Context, chunk *Chunk) error { - hkey := hex.EncodeToString(chunk.Addr) - m.requests[hkey] += 1 - - // on second call return error - if m.requests[hkey] == 2 { - return errUnknown - } - - // on third call return data - if m.requests[hkey] == 3 { - *chunk = *newDummyChunk(chunk.Addr) - go func() { - time.Sleep(100 * time.Millisecond) - close(chunk.ReqC) - }() - - return nil - } - - return nil +type mockNetFetchFuncFactory struct { + fetcher *mockNetFetcher } -func TestNetstoreFailedRequest(t *testing.T) { - searchTimeout = 300 * time.Millisecond +func (m *mockNetFetchFuncFactory) newMockNetFetcher(ctx context.Context, _ Address, peers *sync.Map) NetFetcher { + m.fetcher.peers = peers + m.fetcher.quit = ctx.Done() + m.fetcher.ctx = ctx + return m.fetcher +} - // setup - addr := network.RandomAddr() // tested peers peer address +func mustNewNetStore(t *testing.T) *NetStore { + netStore, _ := mustNewNetStoreWithFetcher(t) + return netStore +} + +func mustNewNetStoreWithFetcher(t *testing.T) (*NetStore, *mockNetFetcher) { + t.Helper() - // temp datadir datadir, err := ioutil.TempDir("", "netstore") if err != nil { t.Fatal(err) } + naddr := make([]byte, 32) params := NewDefaultLocalStoreParams() params.Init(datadir) - params.BaseKey = addr.Over() + params.BaseKey = naddr localStore, err := NewTestLocalStoreForAddr(params) if err != nil { t.Fatal(err) } - r := NewMockRetrieve() - netStore := NewNetStore(localStore, r.retrieve) - - key := Address{} - - // first call is done by the retry on ErrChunkNotFound, no need to do it here - // _, err = netStore.Get(key) - // if err == nil || err != ErrChunkNotFound { - // t.Fatalf("expected to get ErrChunkNotFound, but got: %s", err) - // } - - // second call - _, err = netStore.Get(context.TODO(), key) - if got := r.requests[hex.EncodeToString(key)]; got != 2 { - t.Fatalf("expected to have called retrieve two times, but got: %v", got) + fetcher := &mockNetFetcher{} + mockNetFetchFuncFactory := &mockNetFetchFuncFactory{ + fetcher: fetcher, } - if err != errUnknown { - t.Fatalf("expected to get an unknown error, but got: %s", err) + netStore, err := NewNetStore(localStore, mockNetFetchFuncFactory.newMockNetFetcher) + if err != nil { + t.Fatal(err) + } + return netStore, fetcher +} + +// TestNetStoreGetAndPut tests calling NetStore.Get which is blocked until the same chunk is Put. +// After the Put there should no active fetchers, and the context created for the fetcher should +// be cancelled. +func TestNetStoreGetAndPut(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + c := make(chan struct{}) // this channel ensures that the gouroutine with the Put does not run earlier than the Get + go func() { + <-c // wait for the Get to be called + time.Sleep(200 * time.Millisecond) // and a little more so it is surely called + + // check if netStore created a fetcher in the Get call for the unavailable chunk + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatal("Expected netStore to use a fetcher for the Get call") + } + + err := netStore.Put(ctx, chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + }() + + close(c) + recChunk, err := netStore.Get(ctx, chunk.Address()) // this is blocked until the Put above is done + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + // the retrieved chunk should be the same as what we Put + if !bytes.Equal(recChunk.Address(), chunk.Address()) || !bytes.Equal(recChunk.Data(), chunk.Data()) { + t.Fatalf("Different chunk received than what was put") + } + // the chunk is already available locally, so there should be no active fetchers waiting for it + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after delivery") } - // third call - chunk, err := netStore.Get(context.TODO(), key) - if got := r.requests[hex.EncodeToString(key)]; got != 3 { - t.Fatalf("expected to have called retrieve three times, but got: %v", got) + // A fetcher was created when the Get was called (and the chunk was not available). The chunk + // was delivered with the Put call, so the fetcher should be cancelled now. + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") } - if err != nil || chunk == nil { - t.Fatalf("expected to get a chunk but got: %v, %s", chunk, err) + +} + +// TestNetStoreGetAndPut tests calling NetStore.Put and then NetStore.Get. +// After the Put the chunk is available locally, so the Get can just retrieve it from LocalStore, +// there is no need to create fetchers. +func TestNetStoreGetAfterPut(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + // First we Put the chunk, so the chunk will be available locally + err := netStore.Put(ctx, chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) } - if len(chunk.SData) != 3 { - t.Fatalf("expected to get a chunk with size 3, but got: %v", chunk.SData) + + // Get should retrieve the chunk from LocalStore, without creating fetcher + recChunk, err := netStore.Get(ctx, chunk.Address()) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + // the retrieved chunk should be the same as what we Put + if !bytes.Equal(recChunk.Address(), chunk.Address()) || !bytes.Equal(recChunk.Data(), chunk.Data()) { + t.Fatalf("Different chunk received than what was put") + } + // no fetcher offer or request should be created for a locally available chunk + if fetcher.offerCalled || fetcher.requestCalled { + t.Fatal("NetFetcher.offerCalled or requestCalled not expected to be called") + } + // no fetchers should be created for a locally available chunk + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to not have fetcher") + } + +} + +// TestNetStoreGetTimeout tests a Get call for an unavailable chunk and waits for timeout +func TestNetStoreGetTimeout(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + c := make(chan struct{}) // this channel ensures that the gouroutine does not run earlier than the Get + go func() { + <-c // wait for the Get to be called + time.Sleep(200 * time.Millisecond) // and a little more so it is surely called + + // check if netStore created a fetcher in the Get call for the unavailable chunk + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatal("Expected netStore to use a fetcher for the Get call") + } + }() + + close(c) + // We call Get on this chunk, which is not in LocalStore. We don't Put it at all, so there will + // be a timeout + _, err := netStore.Get(ctx, chunk.Address()) + + // Check if the timeout happened + if err != context.DeadlineExceeded { + t.Fatalf("Expected context.DeadLineExceeded err got %v", err) + } + + // A fetcher was created, check if it has been removed after timeout + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after timeout") + } + + // Check if the fetcher context has been cancelled after the timeout + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") } } + +// TestNetStoreGetCancel tests a Get call for an unavailable chunk, then cancels the context and checks +// the errors +func TestNetStoreGetCancel(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + + c := make(chan struct{}) // this channel ensures that the gouroutine with the cancel does not run earlier than the Get + go func() { + <-c // wait for the Get to be called + time.Sleep(200 * time.Millisecond) // and a little more so it is surely called + // check if netStore created a fetcher in the Get call for the unavailable chunk + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatal("Expected netStore to use a fetcher for the Get call") + } + cancel() + }() + + close(c) + // We call Get with an unavailable chunk, so it will create a fetcher and wait for delivery + _, err := netStore.Get(ctx, chunk.Address()) + + // After the context is cancelled above Get should return with an error + if err != context.Canceled { + t.Fatalf("Expected context.Canceled err got %v", err) + } + + // A fetcher was created, check if it has been removed after cancel + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after cancel") + } + + // Check if the fetcher context has been cancelled after the request context cancel + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") + } +} + +// TestNetStoreMultipleGetAndPut tests four Get calls for the same unavailable chunk. The chunk is +// delivered with a Put, we have to make sure all Get calls return, and they use a single fetcher +// for the chunk retrieval +func TestNetStoreMultipleGetAndPut(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + go func() { + // sleep to make sure Put is called after all the Get + time.Sleep(500 * time.Millisecond) + // check if netStore created exactly one fetcher for all Get calls + if netStore.fetchers.Len() != 1 { + t.Fatal("Expected netStore to use one fetcher for all Get calls") + } + err := netStore.Put(ctx, chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + }() + + // call Get 4 times for the same unavailable chunk. The calls will be blocked until the Put above. + getWG := sync.WaitGroup{} + for i := 0; i < 4; i++ { + getWG.Add(1) + go func() { + defer getWG.Done() + recChunk, err := netStore.Get(ctx, chunk.Address()) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + if !bytes.Equal(recChunk.Address(), chunk.Address()) || !bytes.Equal(recChunk.Data(), chunk.Data()) { + t.Fatalf("Different chunk received than what was put") + } + }() + } + + finishedC := make(chan struct{}) + go func() { + getWG.Wait() + close(finishedC) + }() + + // The Get calls should return after Put, so no timeout expected + select { + case <-finishedC: + case <-time.After(1 * time.Second): + t.Fatalf("Timeout waiting for Get calls to return") + } + + // A fetcher was created, check if it has been removed after cancel + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after delivery") + } + + // A fetcher was created, check if it has been removed after delivery + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") + } + +} + +// TestNetStoreFetchFuncTimeout tests a FetchFunc call for an unavailable chunk and waits for timeout +func TestNetStoreFetchFuncTimeout(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + // FetchFunc is called for an unavaible chunk, so the returned wait function should not be nil + wait := netStore.FetchFunc(ctx, chunk.Address()) + if wait == nil { + t.Fatal("Expected wait function to be not nil") + } + + // There should an active fetcher for the chunk after the FetchFunc call + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatalf("Expected netStore to have one fetcher for the requested chunk") + } + + // wait function should timeout because we don't deliver the chunk with a Put + err := wait(ctx) + if err != context.DeadlineExceeded { + t.Fatalf("Expected context.DeadLineExceeded err got %v", err) + } + + // the fetcher should be removed after timeout + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after timeout") + } + + // the fetcher context should be cancelled after timeout + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") + } +} + +// TestNetStoreFetchFuncAfterPut tests that the FetchFunc should return nil for a locally available chunk +func TestNetStoreFetchFuncAfterPut(t *testing.T) { + netStore := mustNewNetStore(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + // We deliver the created the chunk with a Put + err := netStore.Put(ctx, chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + + // FetchFunc should return nil, because the chunk is available locally, no need to fetch it + wait := netStore.FetchFunc(ctx, chunk.Address()) + if wait != nil { + t.Fatal("Expected wait to be nil") + } + + // No fetchers should be created at all + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to not have fetcher") + } +} + +// TestNetStoreGetCallsRequest tests if Get created a request on the NetFetcher for an unavailable chunk +func TestNetStoreGetCallsRequest(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + // We call get for a not available chunk, it will timeout because the chunk is not delivered + _, err := netStore.Get(ctx, chunk.Address()) + + if err != context.DeadlineExceeded { + t.Fatalf("Expected context.DeadlineExceeded err got %v", err) + } + + // NetStore should call NetFetcher.Request and wait for the chunk + if !fetcher.requestCalled { + t.Fatal("Expected NetFetcher.Request to be called") + } +} + +// TestNetStoreGetCallsOffer tests if Get created a request on the NetFetcher for an unavailable chunk +// in case of a source peer provided in the context. +func TestNetStoreGetCallsOffer(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + // If a source peer is added to the context, NetStore will handle it as an offer + ctx := context.WithValue(context.Background(), "source", sourcePeerID.String()) + ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + + // We call get for a not available chunk, it will timeout because the chunk is not delivered + chunk, err := netStore.Get(ctx, chunk.Address()) + + if err != context.DeadlineExceeded { + t.Fatalf("Expect error %v got %v", context.DeadlineExceeded, err) + } + + // NetStore should call NetFetcher.Offer with the source peer + if !fetcher.offerCalled { + t.Fatal("Expected NetFetcher.Request to be called") + } + + if len(fetcher.sources) != 1 { + t.Fatalf("Expected fetcher sources length 1 got %v", len(fetcher.sources)) + } + + if fetcher.sources[0].String() != sourcePeerID.String() { + t.Fatalf("Expected fetcher source %v got %v", sourcePeerID, fetcher.sources[0]) + } + +} + +// TestNetStoreFetcherCountPeers tests multiple NetStore.Get calls with peer in the context. +// There is no Put call, so the Get calls timeout +func TestNetStoreFetcherCountPeers(t *testing.T) { + + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + addr := randomAddr() + peers := []string{randomAddr().Hex(), randomAddr().Hex(), randomAddr().Hex()} + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + errC := make(chan error) + nrGets := 3 + + // Call Get 3 times with a peer in context + for i := 0; i < nrGets; i++ { + peer := peers[i] + go func() { + ctx := context.WithValue(ctx, "peer", peer) + _, err := netStore.Get(ctx, addr) + errC <- err + }() + } + + // All 3 Get calls should timeout + for i := 0; i < nrGets; i++ { + err := <-errC + if err != context.DeadlineExceeded { + t.Fatalf("Expected \"%v\" error got \"%v\"", context.DeadlineExceeded, err) + } + } + + // fetcher should be closed after timeout + select { + case <-fetcher.quit: + case <-time.After(3 * time.Second): + t.Fatalf("mockNetFetcher not closed after timeout") + } + + // All 3 peers should be given to NetFetcher after the 3 Get calls + if len(fetcher.peersPerRequest) != nrGets { + t.Fatalf("Expected 3 got %v", len(fetcher.peersPerRequest)) + } + + for i, peers := range fetcher.peersPerRequest { + if len(peers) < i+1 { + t.Fatalf("Expected at least %v got %v", i+1, len(peers)) + } + } +} + +// TestNetStoreFetchFuncCalledMultipleTimes calls the wait function given by FetchFunc three times, +// and checks there is still exactly one fetcher for one chunk. Afthe chunk is delivered, it checks +// if the fetcher is closed. +func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + // FetchFunc should return a non-nil wait function, because the chunk is not available + wait := netStore.FetchFunc(ctx, chunk.Address()) + if wait == nil { + t.Fatal("Expected wait function to be not nil") + } + + // There should be exactly one fetcher for the chunk + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatalf("Expected netStore to have one fetcher for the requested chunk") + } + + // Call wait three times parallelly + wg := sync.WaitGroup{} + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + err := wait(ctx) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + wg.Done() + }() + } + + // sleep a little so the wait functions are called above + time.Sleep(100 * time.Millisecond) + + // there should be still only one fetcher, because all wait calls are for the same chunk + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatal("Expected netStore to have one fetcher for the requested chunk") + } + + // Deliver the chunk with a Put + err := netStore.Put(ctx, chunk) + if err != nil { + t.Fatalf("Expected no err got %v", err) + } + + // wait until all wait calls return (because the chunk is delivered) + wg.Wait() + + // There should be no more fetchers for the delivered chunk + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after delivery") + } + + // The context for the fetcher should be cancelled after delivery + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") + } +} + +// TestNetStoreFetcherLifeCycleWithTimeout is similar to TestNetStoreFetchFuncCalledMultipleTimes, +// the only difference is that we don't deilver the chunk, just wait for timeout +func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) { + netStore, fetcher := mustNewNetStoreWithFetcher(t) + + chunk := GenerateRandomChunk(ch.DefaultSize) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + // FetchFunc should return a non-nil wait function, because the chunk is not available + wait := netStore.FetchFunc(ctx, chunk.Address()) + if wait == nil { + t.Fatal("Expected wait function to be not nil") + } + + // There should be exactly one fetcher for the chunk + if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { + t.Fatalf("Expected netStore to have one fetcher for the requested chunk") + } + + // Call wait three times parallelly + wg := sync.WaitGroup{} + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + rctx, rcancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer rcancel() + err := wait(rctx) + if err != context.DeadlineExceeded { + t.Fatalf("Expected err %v got %v", context.DeadlineExceeded, err) + } + }() + } + + // wait until all wait calls timeout + wg.Wait() + + // There should be no more fetchers after timeout + if netStore.fetchers.Len() != 0 { + t.Fatal("Expected netStore to remove the fetcher after delivery") + } + + // The context for the fetcher should be cancelled after timeout + select { + case <-fetcher.ctx.Done(): + default: + t.Fatal("Expected fetcher context to be cancelled") + } +} + +func randomAddr() Address { + addr := make([]byte, 32) + rand.Read(addr) + return Address(addr) +} diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 36ff66d045..f74eef06bc 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" ) @@ -57,7 +57,7 @@ import ( When certain no of data chunks are created (defaultBranches), a signal is sent to create a tree entry. When the level 0 tree entries reaches certain threshold (defaultBranches), another signal is sent to a tree entry one level up.. and so on... until only the data is exhausted AND only one - tree entry is present in certain level. The key of tree entry is given out as the rootKey of the file. + tree entry is present in certain level. The key of tree entry is given out as the rootAddress of the file. */ @@ -98,15 +98,15 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get } /* - When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes. + When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Address), the root hash of the entire content will fill this once processing finishes. New chunks to store are store using the putter which the caller provides. */ func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, ch.DefaultSize)).Split(ctx) } func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, ch.DefaultSize)).Append(ctx) } // Entry to create a tree node @@ -153,7 +153,7 @@ type PyramidChunker struct { wg *sync.WaitGroup errC chan error quitC chan bool - rootKey []byte + rootAddress []byte chunkLevel [][]*TreeEntry } @@ -171,14 +171,14 @@ func NewPyramidSplitter(params *PyramidSplitterParams) (pc *PyramidChunker) { pc.wg = &sync.WaitGroup{} pc.errC = make(chan error) pc.quitC = make(chan bool) - pc.rootKey = make([]byte, pc.hashSize) + pc.rootAddress = make([]byte, pc.hashSize) pc.chunkLevel = make([][]*TreeEntry, pc.branches) return } func (pc *PyramidChunker) Join(addr Address, getter Getter, depth int) LazySectionReader { return &LazyChunkReader{ - key: addr, + addr: addr, depth: depth, chunkSize: pc.chunkSize, branches: pc.branches, @@ -209,7 +209,7 @@ func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(conte log.Debug("pyramid.chunker: Split()") pc.wg.Add(1) - pc.prepareChunks(false) + pc.prepareChunks(ctx, false) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -231,19 +231,21 @@ func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(conte if err != nil { return nil, nil, err } - case <-time.NewTimer(splitTimeout).C: + case <-ctx.Done(): + _ = pc.putter.Wait(ctx) //??? + return nil, nil, ctx.Err() } - return pc.rootKey, pc.putter.Wait, nil + return pc.rootAddress, pc.putter.Wait, nil } func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(context.Context) error, err error) { log.Debug("pyramid.chunker: Append()") // Load the right most unfinished tree chunks in every level - pc.loadTree() + pc.loadTree(ctx) pc.wg.Add(1) - pc.prepareChunks(true) + pc.prepareChunks(ctx, true) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -265,11 +267,11 @@ func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(cont case <-time.NewTimer(splitTimeout).C: } - return pc.rootKey, pc.putter.Wait, nil + return pc.rootAddress, pc.putter.Wait, nil } -func (pc *PyramidChunker) processor(id int64) { +func (pc *PyramidChunker) processor(ctx context.Context, id int64) { defer pc.decrementWorkerCount() for { select { @@ -278,19 +280,22 @@ func (pc *PyramidChunker) processor(id int64) { if !ok { return } - pc.processChunk(id, job) + pc.processChunk(ctx, id, job) case <-pc.quitC: return } } } -func (pc *PyramidChunker) processChunk(id int64, job *chunkJob) { +func (pc *PyramidChunker) processChunk(ctx context.Context, id int64, job *chunkJob) { log.Debug("pyramid.chunker: processChunk()", "id", id) - ref, err := pc.putter.Put(context.TODO(), job.chunk) + ref, err := pc.putter.Put(ctx, job.chunk) if err != nil { - pc.errC <- err + select { + case pc.errC <- err: + case <-pc.quitC: + } } // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) @@ -300,14 +305,14 @@ func (pc *PyramidChunker) processChunk(id int64, job *chunkJob) { job.parentWg.Done() } -func (pc *PyramidChunker) loadTree() error { +func (pc *PyramidChunker) loadTree(ctx context.Context) error { log.Debug("pyramid.chunker: loadTree()") // Get the root chunk to get the total size - chunkData, err := pc.getter.Get(context.TODO(), Reference(pc.key)) + chunkData, err := pc.getter.Get(ctx, Reference(pc.key)) if err != nil { return errLoadingTreeRootChunk } - chunkSize := chunkData.Size() + chunkSize := int64(chunkData.Size()) log.Trace("pyramid.chunker: root chunk", "chunk.Size", chunkSize, "pc.chunkSize", pc.chunkSize) //if data size is less than a chunk... add a parent with update as pending @@ -356,7 +361,7 @@ func (pc *PyramidChunker) loadTree() error { branchCount = int64(len(ent.chunk)-8) / pc.hashSize for i := int64(0); i < branchCount; i++ { key := ent.chunk[8+(i*pc.hashSize) : 8+((i+1)*pc.hashSize)] - newChunkData, err := pc.getter.Get(context.TODO(), Reference(key)) + newChunkData, err := pc.getter.Get(ctx, Reference(key)) if err != nil { return errLoadingTreeChunk } @@ -365,7 +370,7 @@ func (pc *PyramidChunker) loadTree() error { newEntry := &TreeEntry{ level: lvl - 1, branchCount: bewBranchCount, - subtreeSize: uint64(newChunkSize), + subtreeSize: newChunkSize, chunk: newChunkData, key: key, index: 0, @@ -385,7 +390,7 @@ func (pc *PyramidChunker) loadTree() error { return nil } -func (pc *PyramidChunker) prepareChunks(isAppend bool) { +func (pc *PyramidChunker) prepareChunks(ctx context.Context, isAppend bool) { log.Debug("pyramid.chunker: prepareChunks", "isAppend", isAppend) defer pc.wg.Done() @@ -393,11 +398,11 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) { pc.incrementWorkerCount() - go pc.processor(pc.workerCount) + go pc.processor(ctx, pc.workerCount) parent := NewTreeEntry(pc) var unfinishedChunkData ChunkData - var unfinishedChunkSize int64 + var unfinishedChunkSize uint64 if isAppend && len(pc.chunkLevel[0]) != 0 { lastIndex := len(pc.chunkLevel[0]) - 1 @@ -415,16 +420,16 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) { } lastBranch := parent.branchCount - 1 - lastKey := parent.chunk[8+lastBranch*pc.hashSize : 8+(lastBranch+1)*pc.hashSize] + lastAddress := parent.chunk[8+lastBranch*pc.hashSize : 8+(lastBranch+1)*pc.hashSize] var err error - unfinishedChunkData, err = pc.getter.Get(context.TODO(), lastKey) + unfinishedChunkData, err = pc.getter.Get(ctx, lastAddress) if err != nil { pc.errC <- err } unfinishedChunkSize = unfinishedChunkData.Size() - if unfinishedChunkSize < pc.chunkSize { - parent.subtreeSize = parent.subtreeSize - uint64(unfinishedChunkSize) + if unfinishedChunkSize < uint64(pc.chunkSize) { + parent.subtreeSize = parent.subtreeSize - unfinishedChunkSize parent.branchCount = parent.branchCount - 1 } else { unfinishedChunkData = nil @@ -468,8 +473,8 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) { if parent.branchCount == 1 && (pc.depth() == 0 || isAppend) { // Data is exactly one chunk.. pick the last chunk key as root chunkWG.Wait() - lastChunksKey := parent.chunk[8 : 8+pc.hashSize] - copy(pc.rootKey, lastChunksKey) + lastChunksAddress := parent.chunk[8 : 8+pc.hashSize] + copy(pc.rootAddress, lastChunksAddress) break } } else { @@ -502,7 +507,7 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) { // No need to build the tree if the depth is 0 // or we are appending. // Just use the last key. - copy(pc.rootKey, pkey) + copy(pc.rootAddress, pkey) } else { // We need to build the tree and and provide the lonely // chunk key to replace the last tree chunk key. @@ -525,7 +530,7 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) { workers := pc.getWorkerCount() if int64(len(pc.jobC)) > workers && workers < ChunkProcessors { pc.incrementWorkerCount() - go pc.processor(pc.workerCount) + go pc.processor(ctx, pc.workerCount) } } @@ -558,7 +563,7 @@ func (pc *PyramidChunker) buildTree(isAppend bool, ent *TreeEntry, chunkWG *sync lvlCount := int64(len(pc.chunkLevel[lvl])) if lvlCount == 1 && last { - copy(pc.rootKey, pc.chunkLevel[lvl][0].key) + copy(pc.rootAddress, pc.chunkLevel[lvl][0].key) return } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 53e3af485a..bc2af2cd78 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -25,16 +25,16 @@ import ( "fmt" "hash" "io" - "sync" + "io/ioutil" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/swarm/bmt" - "github.com/ethereum/go-ethereum/swarm/chunk" + ch "github.com/ethereum/go-ethereum/swarm/chunk" ) const MaxPO = 16 -const KeyLength = 32 +const AddressLength = 32 type Hasher func() hash.Hash type SwarmHasher func() SwarmHash @@ -116,7 +116,7 @@ func MakeHashFunc(hash string) SwarmHasher { return func() SwarmHash { hasher := sha3.NewKeccak256 hasherSize := hasher().Size() - segmentCount := chunk.DefaultSize / hasherSize + segmentCount := ch.DefaultSize / hasherSize pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize) return bmt.New(pool) } @@ -169,88 +169,88 @@ func (c AddressCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } -// Chunk also serves as a request object passed to ChunkStores -// in case it is a retrieval request, Data is nil and Size is 0 -// Note that Size is not the size of the data chunk, which is Data.Size() -// but the size of the subtree encoded in the chunk -// 0 if request, to be supplied by the dpa -type Chunk struct { - Addr Address // always - SData []byte // nil if request, to be supplied by dpa - Size int64 // size of the data covered by the subtree encoded in this chunk - //Source Peer // peer - C chan bool // to signal data delivery by the dpa - ReqC chan bool // to signal the request done - dbStoredC chan bool // never remove a chunk from memStore before it is written to dbStore - dbStored bool - dbStoredMu *sync.Mutex - errored error // flag which is set when the chunk request has errored or timeouted - erroredMu sync.Mutex +// Chunk interface implemented by context.Contexts and data chunks +type Chunk interface { + Address() Address + Payload() []byte + SpanBytes() []byte + Span() int64 + Data() []byte } -func (c *Chunk) SetErrored(err error) { - c.erroredMu.Lock() - defer c.erroredMu.Unlock() - - c.errored = err +type chunk struct { + addr Address + sdata []byte + span int64 } -func (c *Chunk) GetErrored() error { - c.erroredMu.Lock() - defer c.erroredMu.Unlock() - - return c.errored -} - -func NewChunk(addr Address, reqC chan bool) *Chunk { - return &Chunk{ - Addr: addr, - ReqC: reqC, - dbStoredC: make(chan bool), - dbStoredMu: &sync.Mutex{}, +func NewChunk(addr Address, data []byte) *chunk { + return &chunk{ + addr: addr, + sdata: data, + span: -1, } } -func (c *Chunk) markAsStored() { - c.dbStoredMu.Lock() - defer c.dbStoredMu.Unlock() +func (c *chunk) Address() Address { + return c.addr +} - if !c.dbStored { - close(c.dbStoredC) - c.dbStored = true +func (c *chunk) SpanBytes() []byte { + return c.sdata[:8] +} + +func (c *chunk) Span() int64 { + if c.span == -1 { + c.span = int64(binary.LittleEndian.Uint64(c.sdata[:8])) } + return c.span } -func (c *Chunk) WaitToStore() error { - <-c.dbStoredC - return c.GetErrored() +func (c *chunk) Data() []byte { + return c.sdata } -func GenerateRandomChunk(dataSize int64) *Chunk { - return GenerateRandomChunks(dataSize, 1)[0] +func (c *chunk) Payload() []byte { + return c.sdata[8:] } -func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) { - var i int +// String() for pretty printing +func (self *chunk) String() string { + return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata)) +} + +func GenerateRandomChunk(dataSize int64) Chunk { hasher := MakeHashFunc(DefaultHash)() - if dataSize > chunk.DefaultSize { - dataSize = chunk.DefaultSize - } + sdata := make([]byte, dataSize+8) + rand.Read(sdata[8:]) + binary.LittleEndian.PutUint64(sdata[:8], uint64(dataSize)) + hasher.ResetWithLength(sdata[:8]) + hasher.Write(sdata[8:]) + return NewChunk(hasher.Sum(nil), sdata) +} - for i = 0; i < count; i++ { - chunks = append(chunks, NewChunk(nil, nil)) - chunks[i].SData = make([]byte, dataSize+8) - rand.Read(chunks[i].SData) - binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(dataSize)) - hasher.ResetWithLength(chunks[i].SData[:8]) - hasher.Write(chunks[i].SData[8:]) - chunks[i].Addr = make([]byte, 32) - copy(chunks[i].Addr, hasher.Sum(nil)) +func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { + if dataSize > ch.DefaultSize { + dataSize = ch.DefaultSize + } + for i := 0; i < count; i++ { + ch := GenerateRandomChunk(ch.DefaultSize) + chunks = append(chunks, ch) } - return chunks } +func GenerateRandomData(l int) (r io.Reader, slice []byte) { + slice, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(l))) + if err != nil { + panic("rand error") + } + // log.Warn("generate random data", "len", len(slice), "data", common.Bytes2Hex(slice)) + r = io.LimitReader(bytes.NewReader(slice), int64(l)) + return r, slice +} + // Size, Seek, Read, ReadAt type LazySectionReader interface { Context() context.Context @@ -273,18 +273,17 @@ func (r *LazyTestSectionReader) Context() context.Context { } type StoreParams struct { - Hash SwarmHasher `toml:"-"` - DbCapacity uint64 - CacheCapacity uint - ChunkRequestsCacheCapacity uint - BaseKey []byte + Hash SwarmHasher `toml:"-"` + DbCapacity uint64 + CacheCapacity uint + BaseKey []byte } func NewDefaultStoreParams() *StoreParams { - return NewStoreParams(defaultLDBCapacity, defaultCacheCapacity, defaultChunkRequestsCacheCapacity, nil, nil) + return NewStoreParams(defaultLDBCapacity, defaultCacheCapacity, nil, nil) } -func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHasher, basekey []byte) *StoreParams { +func NewStoreParams(ldbCap uint64, cacheCap uint, hash SwarmHasher, basekey []byte) *StoreParams { if basekey == nil { basekey = make([]byte, 32) } @@ -292,11 +291,10 @@ func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHa hash = MakeHashFunc(DefaultHash) } return &StoreParams{ - Hash: hash, - DbCapacity: ldbCap, - CacheCapacity: cacheCap, - ChunkRequestsCacheCapacity: requestsCap, - BaseKey: basekey, + Hash: hash, + DbCapacity: ldbCap, + CacheCapacity: cacheCap, + BaseKey: basekey, } } @@ -321,8 +319,8 @@ type Getter interface { } // NOTE: this returns invalid data if chunk is encrypted -func (c ChunkData) Size() int64 { - return int64(binary.LittleEndian.Uint64(c[:8])) +func (c ChunkData) Size() uint64 { + return binary.LittleEndian.Uint64(c[:8]) } func (c ChunkData) Data() []byte { @@ -348,7 +346,8 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { // Validate that the given key is a valid content address for the given data func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { - if l := len(data); l < 9 || l > chunk.DefaultSize+8 { + if l := len(data); l < 9 || l > ch.DefaultSize+8 { + // log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l) return false } @@ -359,3 +358,37 @@ func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { return bytes.Equal(hash, addr[:]) } + +type ChunkStore interface { + Put(ctx context.Context, ch Chunk) (err error) + Get(rctx context.Context, ref Address) (ch Chunk, err error) + Close() +} + +// SyncChunkStore is a ChunkStore which supports syncing +type SyncChunkStore interface { + ChunkStore + BinIndex(po uint8) uint64 + Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error + FetchFunc(ctx context.Context, ref Address) func(context.Context) error +} + +// FakeChunkStore doesn't store anything, just implements the ChunkStore interface +// It can be used to inject into a hasherStore if you don't want to actually store data just do the +// hashing +type FakeChunkStore struct { +} + +// Put doesn't store anything it is just here to implement ChunkStore +func (f *FakeChunkStore) Put(_ context.Context, ch Chunk) error { + return nil +} + +// Gut doesn't store anything it is just here to implement ChunkStore +func (f *FakeChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { + panic("FakeChunkStore doesn't support Get") +} + +// Close doesn't store anything it is just here to implement ChunkStore +func (f *FakeChunkStore) Close() { +} diff --git a/swarm/swarm.go b/swarm/swarm.go index baf71b9623..13aa1125d3 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -75,8 +75,8 @@ type Swarm struct { privateKey *ecdsa.PrivateKey corsString string swapEnabled bool - lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped - sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit + netStore *storage.NetStore + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit ps *pss.Pss tracerClose io.Closer @@ -164,37 +164,40 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e self.dns = resolver } - self.lstore, err = storage.NewLocalStore(config.LocalStoreParams, mockStore) + lstore, err := storage.NewLocalStore(config.LocalStoreParams, mockStore) if err != nil { - return + return nil, err + } + + self.netStore, err = storage.NewNetStore(lstore, nil) + if err != nil { + return nil, err } - db := storage.NewDBAPI(self.lstore) to := network.NewKademlia( common.FromHex(config.BzzKey), network.NewKadParams(), ) - delivery := stream.NewDelivery(to, db) + delivery := stream.NewDelivery(to, self.netStore) + self.netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, config.DeliverySkipCheck).New - self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{ - SkipCheck: config.DeliverySkipCheck, + self.streamer = stream.NewRegistry(addr, delivery, self.netStore, stateStore, &stream.RegistryOptions{ + SkipCheck: config.SyncingSkipCheck, DoSync: config.SyncEnabled, DoRetrieve: true, SyncUpdateDelay: config.SyncUpdateDelay, }) - // set up NetStore, the cloud storage local access layer - netStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage - self.fileStore = storage.NewFileStore(netStore, self.config.FileStoreParams) + self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) var resourceHandler *mru.Handler rhparams := &mru.HandlerParams{} resourceHandler = mru.NewHandler(rhparams) - resourceHandler.SetStore(netStore) + resourceHandler.SetStore(self.netStore) - self.lstore.Validators = []storage.ChunkValidator{ + lstore.Validators = []storage.ChunkValidator{ storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)), resourceHandler, } @@ -399,7 +402,7 @@ func (self *Swarm) periodicallyUpdateGauges() { func (self *Swarm) updateGauges() { uptimeGauge.Update(time.Since(startTime).Nanoseconds()) - requestsCacheGauge.Update(int64(self.lstore.RequestsCacheLen())) + requestsCacheGauge.Update(int64(self.netStore.RequestsCacheLen())) } // implements the node.Service interface @@ -420,8 +423,8 @@ func (self *Swarm) Stop() error { ch.Save() } - if self.lstore != nil { - self.lstore.DbStore.Close() + if self.netStore != nil { + self.netStore.Close() } self.sfs.Stop() stopCounter.Inc(1) @@ -478,21 +481,6 @@ func (self *Swarm) APIs() []rpc.API { Service: self.sfs, Public: false, }, - // storage APIs - // DEPRECATED: Use the HTTP API instead - { - Namespace: "bzz", - Version: "0.1", - Service: api.NewStorage(self.api), - Public: true, - }, - { - Namespace: "bzz", - Version: "0.1", - Service: api.NewFileSystem(self.api), - Public: false, - }, - // {Namespace, Version, api.NewAdmin(self), false}, } apis = append(apis, self.bzz.APIs()...) diff --git a/swarm/swarm_test.go b/swarm/swarm_test.go index 0827748ae2..c6569e37bf 100644 --- a/swarm/swarm_test.go +++ b/swarm/swarm_test.go @@ -82,8 +82,8 @@ func TestNewSwarm(t *testing.T) { if s.dns != nil { t.Error("dns initialized, but it should not be") } - if s.lstore == nil { - t.Error("localstore not initialized") + if s.netStore == nil { + t.Error("netStore not initialized") } if s.streamer == nil { t.Error("streamer not initialized") @@ -91,9 +91,6 @@ func TestNewSwarm(t *testing.T) { if s.fileStore == nil { t.Error("fileStore not initialized") } - if s.lstore.Validators == nil { - t.Error("localstore validators not initialized") - } if s.bzz == nil { t.Error("bzz not initialized") } From 72c820c49e231b9048b7b46ced109efc09396e8e Mon Sep 17 00:00:00 2001 From: Liang ZOU Date: Thu, 13 Sep 2018 17:48:15 +0800 Subject: [PATCH 116/126] core/vm: fix typo 'EVM EVM' ==> 'EVM' (#17654) --- core/vm/interface.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/interface.go b/core/vm/interface.go index 1ef91cf1d6..d176f5b397 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -64,7 +64,7 @@ type StateDB interface { ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) } -// CallContext provides a basic interface for the EVM calling conventions. The EVM EVM +// CallContext provides a basic interface for the EVM calling conventions. The EVM // depends on this context being implemented for doing subcalls and initialising new EVM contracts. type CallContext interface { // Call another contract From 44a1764f9ca182d9d7c13742f07fa33f6e4f3197 Mon Sep 17 00:00:00 2001 From: EOS Classic Date: Fri, 14 Sep 2018 22:16:10 +0900 Subject: [PATCH 117/126] README: Change gitter badge to discord --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c6bc91af17..f308fb1011 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874 )](https://godoc.org/github.com/ethereum/go-ethereum) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) -[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) Automated builds are available for stable releases and the unstable master branch. Binary archives are published at https://geth.ethereum.org/downloads/. From 86a03f97d30c11a5321fa2f0fd37cbc4c7f63a32 Mon Sep 17 00:00:00 2001 From: Emil Date: Fri, 14 Sep 2018 23:07:13 +0300 Subject: [PATCH 118/126] all: simplify s[:] to s where s is a slice (#17673) --- cmd/evm/disasm.go | 2 +- common/bytes.go | 2 +- core/chain_indexer.go | 2 +- core/types/bloom9.go | 4 ++-- eth/tracers/tracer.go | 8 ++++---- p2p/discover/table.go | 2 +- p2p/discv5/table.go | 2 +- swarm/api/http/server_test.go | 14 +++++++------- swarm/pss/pss.go | 2 +- swarm/storage/ldbstore.go | 10 +++++----- swarm/storage/ldbstore_test.go | 2 +- swarm/storage/types.go | 2 +- whisper/whisperv5/filter_test.go | 2 +- whisper/whisperv6/filter_test.go | 2 +- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go index 4a442cf784..69f611e39b 100644 --- a/cmd/evm/disasm.go +++ b/cmd/evm/disasm.go @@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error { return err } - code := strings.TrimSpace(string(in[:])) + code := strings.TrimSpace(string(in)) fmt.Printf("%v\n", code) return asm.PrintDisassembled(code) } diff --git a/common/bytes.go b/common/bytes.go index cbab2c3fa9..0c257a1ee0 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -100,7 +100,7 @@ func Hex2BytesFixed(str string, flen int) []byte { return h[len(h)-flen:] } hh := make([]byte, flen) - copy(hh[flen-len(h):flen], h[:]) + copy(hh[flen-len(h):flen], h) return hh } diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 89ee75eb29..4bdd4ba1c8 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -445,7 +445,7 @@ func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) { func (c *ChainIndexer) loadValidSections() { data, _ := c.indexDb.Get([]byte("count")) if len(data) == 8 { - c.storedSections = binary.BigEndian.Uint64(data[:]) + c.storedSections = binary.BigEndian.Uint64(data) } } diff --git a/core/types/bloom9.go b/core/types/bloom9.go index a76b6f33c5..d045c9e667 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -113,7 +113,7 @@ func LogsBloom(logs []*Log) *big.Int { } func bloom9(b []byte) *big.Int { - b = crypto.Keccak256(b[:]) + b = crypto.Keccak256(b) r := new(big.Int) @@ -130,7 +130,7 @@ var Bloom9 = bloom9 func BloomLookup(bin Bloom, topic bytesBacked) bool { bloom := bin.Big() - cmp := bloom9(topic.Bytes()[:]) + cmp := bloom9(topic.Bytes()) return bloom.And(bloom, cmp).Cmp(cmp) == 0 } diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index feb57e0601..b519236f2d 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -124,7 +124,7 @@ func (mw *memoryWrapper) pushObject(vm *duktape.Context) { ctx.Pop2() ptr := ctx.PushFixedBuffer(len(blob)) - copy(makeSlice(ptr, uint(len(blob))), blob[:]) + copy(makeSlice(ptr, uint(len(blob))), blob) return 1 }) vm.PutPropString(obj, "slice") @@ -204,7 +204,7 @@ func (dw *dbWrapper) pushObject(vm *duktape.Context) { code := dw.db.GetCode(common.BytesToAddress(popSlice(ctx))) ptr := ctx.PushFixedBuffer(len(code)) - copy(makeSlice(ptr, uint(len(code))), code[:]) + copy(makeSlice(ptr, uint(len(code))), code) return 1 }) vm.PutPropString(obj, "getCode") @@ -268,7 +268,7 @@ func (cw *contractWrapper) pushObject(vm *duktape.Context) { blob := cw.contract.Input ptr := ctx.PushFixedBuffer(len(blob)) - copy(makeSlice(ptr, uint(len(blob))), blob[:]) + copy(makeSlice(ptr, uint(len(blob))), blob) return 1 }) vm.PutPropString(obj, "getInput") @@ -584,7 +584,7 @@ func (jst *Tracer) GetResult() (json.RawMessage, error) { case []byte: ptr := jst.vm.PushFixedBuffer(len(val)) - copy(makeSlice(ptr, uint(len(val))), val[:]) + copy(makeSlice(ptr, uint(len(val))), val) case common.Address: ptr := jst.vm.PushFixedBuffer(20) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 0a554bbeb4..a130b5494f 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -162,7 +162,7 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) { var buckets [][]*Node for _, b := range &tab.buckets { if len(b.entries) > 0 { - buckets = append(buckets, b.entries[:]) + buckets = append(buckets, b.entries) } } if len(buckets) == 0 { diff --git a/p2p/discv5/table.go b/p2p/discv5/table.go index c793be5082..4f4b2426f4 100644 --- a/p2p/discv5/table.go +++ b/p2p/discv5/table.go @@ -123,7 +123,7 @@ func (tab *Table) readRandomNodes(buf []*Node) (n int) { var buckets [][]*Node for _, b := range &tab.buckets { if len(b.entries) > 0 { - buckets = append(buckets, b.entries[:]) + buckets = append(buckets, b.entries) } } if len(buckets) == 0 { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 2acdbd1ad4..4a3ca0429a 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -501,7 +501,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { var respbody []byte url := srv.URL + "/bzz-raw:/" - if k[:] != "" { + if k != "" { url += rootRef + "/" + k[1:] + "?content_type=text/plain" } resp, err = http.Get(url) @@ -515,7 +515,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { isexpectedfailrequest := false for _, r := range expectedfailrequests { - if k[:] == r { + if k == r { isexpectedfailrequest = true } } @@ -530,7 +530,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { var respbody []byte url := srv.URL + "/bzz-hash:/" - if k[:] != "" { + if k != "" { url += rootRef + "/" + k[1:] } resp, err = http.Get(url) @@ -547,7 +547,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { isexpectedfailrequest := false for _, r := range expectedfailrequests { - if k[:] == r { + if k == r { isexpectedfailrequest = true } } @@ -599,7 +599,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } { k := c.path url := srv.URL + "/bzz-list:/" - if k[:] != "" { + if k != "" { url += rootRef + "/" + k[1:] } t.Run("json list "+c.path, func(t *testing.T) { @@ -618,7 +618,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { isexpectedfailrequest := false for _, r := range expectedfailrequests { - if k[:] == r { + if k == r { isexpectedfailrequest = true } } @@ -650,7 +650,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { isexpectedfailrequest := false for _, r := range expectedfailrequests { - if k[:] == r { + if k == r { isexpectedfailrequest = true } } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index b55c97fddf..b96649fea1 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -448,7 +448,7 @@ func (p *Pss) isSelfRecipient(msg *PssMsg) bool { // test match of leftmost bytes in given message to node's Kademlia address func (p *Pss) isSelfPossibleRecipient(msg *PssMsg) bool { local := p.Kademlia.BaseAddr() - return bytes.Equal(msg.To[:], local[:len(msg.To)]) + return bytes.Equal(msg.To, local[:len(msg.To)]) } ///////////////////////////////////////////////////////////////////// diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 8ab7e60b36..bde627394e 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -85,7 +85,7 @@ func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams { return &LDBStoreParams{ StoreParams: storeparams, Path: path, - Po: func(k Address) (ret uint8) { return uint8(Proximity(storeparams.BaseKey[:], k[:])) }, + Po: func(k Address) (ret uint8) { return uint8(Proximity(storeparams.BaseKey, k[:])) }, } } @@ -322,7 +322,7 @@ func (s *LDBStore) Export(out io.Writer) (int64, error) { log.Trace("store.export", "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po) data, err := s.db.Get(datakey) if err != nil { - log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) + log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key, err)) continue } @@ -455,7 +455,7 @@ func (s *LDBStore) Cleanup() { } if !found { - log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key[:])) + log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key)) errorsFound++ continue } @@ -469,10 +469,10 @@ func (s *LDBStore) Cleanup() { } cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) - log.Trace("chunk", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) + log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) if len(c.sdata) > ch.DefaultSize+8 { - log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) + log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) s.delete(index.Idx, getIndexKey(key[1:]), po) removed++ errorsFound++ diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index ae70ee2592..75b5d6aa95 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -74,7 +74,7 @@ func newTestDbStore(mock bool, trusted bool) (*testDbStore, func(), error) { func testPoFunc(k Address) (ret uint8) { basekey := make([]byte, 32) - return uint8(Proximity(basekey[:], k[:])) + return uint8(Proximity(basekey, k[:])) } func (db *testDbStore) close() { diff --git a/swarm/storage/types.go b/swarm/storage/types.go index bc2af2cd78..8c70f45848 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -136,7 +136,7 @@ func (a Address) Log() string { } func (a Address) String() string { - return fmt.Sprintf("%064x", []byte(a)[:]) + return fmt.Sprintf("%064x", []byte(a)) } func (a Address) MarshalJSON() (out []byte, err error) { diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index c01c22668c..33138fb1a4 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -56,7 +56,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { f.Topics = make([][]byte, topicNum) for i := 0; i < topicNum; i++ { f.Topics[i] = make([]byte, 4) - mrand.Read(f.Topics[i][:]) + mrand.Read(f.Topics[i]) f.Topics[i][0] = 0x01 } diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 82e4aa0241..5ce99d9f6c 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -56,7 +56,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { f.Topics = make([][]byte, topicNum) for i := 0; i < topicNum; i++ { f.Topics[i] = make([]byte, 4) - mrand.Read(f.Topics[i][:]) + mrand.Read(f.Topics[i]) f.Topics[i][0] = 0x01 } From d4a28a13ca4c36019d23da6342e7e55b1d406fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 14 Sep 2018 22:14:29 +0200 Subject: [PATCH 119/126] les: fix distReq.sentChn double close bug (#17639) --- les/distributor.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/les/distributor.go b/les/distributor.go index d3f6b21d18..f90765b624 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -114,7 +114,9 @@ func (d *requestDistributor) loop() { d.lock.Lock() elem := d.reqQueue.Front() for elem != nil { - close(elem.Value.(*distReq).sentChn) + req := elem.Value.(*distReq) + close(req.sentChn) + req.sentChn = nil elem = elem.Next() } d.lock.Unlock() From 3df7df0386a78a6d818fb6dfbbbcf07ddedc658f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 15 Sep 2018 23:44:25 +0200 Subject: [PATCH 120/126] ethash: less copy-paste for EIP 1234 --- consensus/ethash/consensus.go | 179 ++++++++++++---------------------- 1 file changed, 65 insertions(+), 114 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 874f098195..0394c37bb7 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -43,6 +43,16 @@ var ( ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople maxUncles = 2 // Maximum number of uncles allowed in a single block allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks + + // calcDifficultyConstantinople is the difficulty adjustment algorithm for Constantinople. + // It returns the difficulty that a new block should have when created at time given the + // parent block's time and difficulty. The calculation uses the Byzantium rules, but with bomb offset 5M. + calcDifficultyConstantinople = makeDifficultyCalculator(big.NewInt(4999999)) + + // calcDifficultyByzantium is the difficulty adjustment algorithm. It returns + // the difficulty that a new block should have when created at time given the + // parent block's time and difficulty. The calculation uses the Byzantium rules. + calcDifficultyByzantium = makeDifficultyCalculator(big.NewInt(2999999)) ) // Various error messages to mark blocks invalid. These should be private to @@ -319,126 +329,67 @@ var ( big9 = big.NewInt(9) big10 = big.NewInt(10) bigMinus99 = big.NewInt(-99) - big2999999 = big.NewInt(2999999) - big4999999 = big.NewInt(4999999) ) -// calcDifficultyConstantinople is the difficulty adjustment algorithm. It returns -// the difficulty that a new block should have when created at time given the -// parent block's time and difficulty. The calculation uses the Constantinople rules. -func calcDifficultyConstantinople(time uint64, parent *types.Header) *big.Int { - // https://github.com/ethereum/EIPs/issues/100. - // algorithm: - // diff = (parent_diff + - // (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) - // ) + 2^(periodCount - 2) +// makeDifficultyCalculator creates a difficultyCalculator with the given bomb-delay. +// the difficulty is calculated with Byzantium rules, which differs from Homestead in +// how uncles affect the calculation +func makeDifficultyCalculator(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int { + return func(time uint64, parent *types.Header) *big.Int { + // https://github.com/ethereum/EIPs/issues/100. + // algorithm: + // diff = (parent_diff + + // (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) + // ) + 2^(periodCount - 2) - bigTime := new(big.Int).SetUint64(time) - bigParentTime := new(big.Int).Set(parent.Time) + bigTime := new(big.Int).SetUint64(time) + bigParentTime := new(big.Int).Set(parent.Time) - // holds intermediate values to make the algo easier to read & audit - x := new(big.Int) - y := new(big.Int) + // holds intermediate values to make the algo easier to read & audit + x := new(big.Int) + y := new(big.Int) - // (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9 - x.Sub(bigTime, bigParentTime) - x.Div(x, big9) - if parent.UncleHash == types.EmptyUncleHash { - x.Sub(big1, x) - } else { - x.Sub(big2, x) + // (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9 + x.Sub(bigTime, bigParentTime) + x.Div(x, big9) + if parent.UncleHash == types.EmptyUncleHash { + x.Sub(big1, x) + } else { + x.Sub(big2, x) + } + // max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99) + if x.Cmp(bigMinus99) < 0 { + x.Set(bigMinus99) + } + // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) + y.Div(parent.Difficulty, params.DifficultyBoundDivisor) + x.Mul(y, x) + x.Add(parent.Difficulty, x) + + // minimum difficulty can ever be (before exponential factor) + if x.Cmp(params.MinimumDifficulty) < 0 { + x.Set(params.MinimumDifficulty) + } + // calculate a fake block number for the ice-age delay: + // https://github.com/ethereum/EIPs/pull/669 + // fake_block_number = max(0, block.number - 3_000_000) + fakeBlockNumber := new(big.Int) + if parent.Number.Cmp(bombDelay) >= 0 { + fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, bombDelay) // Note, parent is 1 less than the actual block number + } + // for the exponential factor + periodCount := fakeBlockNumber + periodCount.Div(periodCount, expDiffPeriod) + + // the exponential factor, commonly referred to as "the bomb" + // diff = diff + 2^(periodCount - 2) + if periodCount.Cmp(big1) > 0 { + y.Sub(periodCount, big2) + y.Exp(big2, y, nil) + x.Add(x, y) + } + return x } - // max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99) - if x.Cmp(bigMinus99) < 0 { - x.Set(bigMinus99) - } - // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) - y.Div(parent.Difficulty, params.DifficultyBoundDivisor) - x.Mul(y, x) - x.Add(parent.Difficulty, x) - - // minimum difficulty can ever be (before exponential factor) - if x.Cmp(params.MinimumDifficulty) < 0 { - x.Set(params.MinimumDifficulty) - } - // calculate a fake block number for the ice-age delay: - // https://github.com/ethereum/EIPs/pull/1234 - // fake_block_number = max(0, block.number - 5_000_000) - fakeBlockNumber := new(big.Int) - if parent.Number.Cmp(big4999999) >= 0 { - fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, big4999999) // Note, parent is 1 less than the actual block number - } - // for the exponential factor - periodCount := fakeBlockNumber - periodCount.Div(periodCount, expDiffPeriod) - - // the exponential factor, commonly referred to as "the bomb" - // diff = diff + 2^(periodCount - 2) - if periodCount.Cmp(big1) > 0 { - y.Sub(periodCount, big2) - y.Exp(big2, y, nil) - x.Add(x, y) - } - return x -} - -// calcDifficultyByzantium is the difficulty adjustment algorithm. It returns -// the difficulty that a new block should have when created at time given the -// parent block's time and difficulty. The calculation uses the Byzantium rules. -func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int { - // https://github.com/ethereum/EIPs/issues/100. - // algorithm: - // diff = (parent_diff + - // (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) - // ) + 2^(periodCount - 2) - - bigTime := new(big.Int).SetUint64(time) - bigParentTime := new(big.Int).Set(parent.Time) - - // holds intermediate values to make the algo easier to read & audit - x := new(big.Int) - y := new(big.Int) - - // (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9 - x.Sub(bigTime, bigParentTime) - x.Div(x, big9) - if parent.UncleHash == types.EmptyUncleHash { - x.Sub(big1, x) - } else { - x.Sub(big2, x) - } - // max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99) - if x.Cmp(bigMinus99) < 0 { - x.Set(bigMinus99) - } - // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) - y.Div(parent.Difficulty, params.DifficultyBoundDivisor) - x.Mul(y, x) - x.Add(parent.Difficulty, x) - - // minimum difficulty can ever be (before exponential factor) - if x.Cmp(params.MinimumDifficulty) < 0 { - x.Set(params.MinimumDifficulty) - } - // calculate a fake block number for the ice-age delay: - // https://github.com/ethereum/EIPs/pull/669 - // fake_block_number = max(0, block.number - 3_000_000) - fakeBlockNumber := new(big.Int) - if parent.Number.Cmp(big2999999) >= 0 { - fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, big2999999) // Note, parent is 1 less than the actual block number - } - // for the exponential factor - periodCount := fakeBlockNumber - periodCount.Div(periodCount, expDiffPeriod) - - // the exponential factor, commonly referred to as "the bomb" - // diff = diff + 2^(periodCount - 2) - if periodCount.Cmp(big1) > 0 { - y.Sub(periodCount, big2) - y.Exp(big2, y, nil) - x.Add(x, y) - } - return x } // calcDifficultyHomestead is the difficulty adjustment algorithm. It returns From 7efb12d29b9c8134ab8ec8c3753b7aa5e503a157 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 17 Sep 2018 11:49:39 +0200 Subject: [PATCH 121/126] ethash: documentation + cleanup --- consensus/ethash/consensus.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 0394c37bb7..548c57cd9b 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -46,13 +46,16 @@ var ( // calcDifficultyConstantinople is the difficulty adjustment algorithm for Constantinople. // It returns the difficulty that a new block should have when created at time given the - // parent block's time and difficulty. The calculation uses the Byzantium rules, but with bomb offset 5M. - calcDifficultyConstantinople = makeDifficultyCalculator(big.NewInt(4999999)) + // parent block's time and difficulty. The calculation uses the Byzantium rules, but with + // bomb offset 5M. + // Specification EIP-1234: https://eips.ethereum.org/EIPS/eip-1234 + calcDifficultyConstantinople = makeDifficultyCalculator(big.NewInt(5000000)) // calcDifficultyByzantium is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time given the // parent block's time and difficulty. The calculation uses the Byzantium rules. - calcDifficultyByzantium = makeDifficultyCalculator(big.NewInt(2999999)) + // Specification EIP-649: https://eips.ethereum.org/EIPS/eip-649 + calcDifficultyByzantium = makeDifficultyCalculator(big.NewInt(3000000)) ) // Various error messages to mark blocks invalid. These should be private to @@ -335,6 +338,9 @@ var ( // the difficulty is calculated with Byzantium rules, which differs from Homestead in // how uncles affect the calculation func makeDifficultyCalculator(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int { + // Note, the calculations below looks at the parent number, which is 1 below + // the block number. Thus we remove one from the delay given + bombDelayFromParent := new(big.Int).Sub(bombDelay, big1) return func(time uint64, parent *types.Header) *big.Int { // https://github.com/ethereum/EIPs/issues/100. // algorithm: @@ -370,12 +376,11 @@ func makeDifficultyCalculator(bombDelay *big.Int) func(time uint64, parent *type if x.Cmp(params.MinimumDifficulty) < 0 { x.Set(params.MinimumDifficulty) } - // calculate a fake block number for the ice-age delay: - // https://github.com/ethereum/EIPs/pull/669 - // fake_block_number = max(0, block.number - 3_000_000) + // calculate a fake block number for the ice-age delay + // Specification: https://eips.ethereum.org/EIPS/eip-1234 fakeBlockNumber := new(big.Int) - if parent.Number.Cmp(bombDelay) >= 0 { - fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, bombDelay) // Note, parent is 1 less than the actual block number + if parent.Number.Cmp(bombDelayFromParent) >= 0 { + fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, bombDelayFromParent) } // for the exponential factor periodCount := fakeBlockNumber From c1345b074212b9dd774498f47377ca733ee8a9f0 Mon Sep 17 00:00:00 2001 From: chenyufeng Date: Mon, 17 Sep 2018 20:09:09 +0800 Subject: [PATCH 122/126] cmd/puppeth: fix comment typo (#17684) * ethdb: unified code comment style. * puppeth: it is unnecessary to alloc pre-funded to 256 addresses * Revert "puppeth: it is unnecessary to alloc pre-funded to 256 addresses" This reverts commit 5e04fbccf0b8aca85030af1779bb7a949033d9d8. * puppeth: fix comment typo * Revert "ethdb: unified code comment style." This reverts commit a581efb3f06a96fc7aec0bfae03c7b6d5a0c1a77. --- cmd/puppeth/wizard_network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/puppeth/wizard_network.go b/cmd/puppeth/wizard_network.go index c0ddcc2a3c..0d2f976d53 100644 --- a/cmd/puppeth/wizard_network.go +++ b/cmd/puppeth/wizard_network.go @@ -87,7 +87,7 @@ func (w *wizard) makeServer() string { return input } -// selectServer lists the user all the currnetly known servers to choose from, +// selectServer lists the user all the currently known servers to choose from, // also granting the option to add a new one. func (w *wizard) selectServer() string { // List the available server to the user and wait for a choice From 5d1d1a808d92440a7c0efad77fa5527e4255e596 Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 17 Sep 2018 20:32:34 +0800 Subject: [PATCH 123/126] consensus, ethdb, metrics: implement forced-meter (#17667) --- consensus/ethash/ethash.go | 4 ++-- ethdb/database.go | 15 ++++++------- metrics/ewma.go | 3 --- metrics/meter.go | 43 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index b4819ca38b..d124cb1e20 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -485,7 +485,7 @@ func New(config Config, notify []string, noverify bool) *Ethash { caches: newlru("cache", config.CachesInMem, newCache), datasets: newlru("dataset", config.DatasetsInMem, newDataset), update: make(chan struct{}), - hashrate: metrics.NewMeter(), + hashrate: metrics.NewMeterForced(), workCh: make(chan *sealTask), fetchWorkCh: make(chan *sealWork), submitWorkCh: make(chan *mineResult), @@ -505,7 +505,7 @@ func NewTester(notify []string, noverify bool) *Ethash { caches: newlru("cache", 1, newCache), datasets: newlru("dataset", 1, newDataset), update: make(chan struct{}), - hashrate: metrics.NewMeter(), + hashrate: metrics.NewMeterForced(), workCh: make(chan *sealTask), fetchWorkCh: make(chan *sealWork), submitWorkCh: make(chan *mineResult), diff --git a/ethdb/database.go b/ethdb/database.go index 1b262b73cb..99abd09b99 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -155,15 +155,12 @@ func (db *LDBDatabase) LDB() *leveldb.DB { // Meter configures the database metrics collectors and func (db *LDBDatabase) Meter(prefix string) { - if metrics.Enabled { - // Initialize all the metrics collector at the requested prefix - db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) - db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) - db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) - db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) - db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) - } - // Initialize write delay metrics no matter we are in metric mode or not. + // Initialize all the metrics collector at the requested prefix + db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) + db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) + db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) + db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) + db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil) db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil) diff --git a/metrics/ewma.go b/metrics/ewma.go index 3aecd4fa35..57c949e7d4 100644 --- a/metrics/ewma.go +++ b/metrics/ewma.go @@ -17,9 +17,6 @@ type EWMA interface { // NewEWMA constructs a new EWMA with the given alpha. func NewEWMA(alpha float64) EWMA { - if !Enabled { - return NilEWMA{} - } return &StandardEWMA{alpha: alpha} } diff --git a/metrics/meter.go b/metrics/meter.go index 82b2141a62..58d170fae0 100644 --- a/metrics/meter.go +++ b/metrics/meter.go @@ -29,6 +29,17 @@ func GetOrRegisterMeter(name string, r Registry) Meter { return r.GetOrRegister(name, NewMeter).(Meter) } +// GetOrRegisterMeterForced returns an existing Meter or constructs and registers a +// new StandardMeter no matter the global switch is enabled or not. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterMeterForced(name string, r Registry) Meter { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewMeterForced).(Meter) +} + // NewMeter constructs a new StandardMeter and launches a goroutine. // Be sure to call Stop() once the meter is of no use to allow for garbage collection. func NewMeter() Meter { @@ -46,8 +57,23 @@ func NewMeter() Meter { return m } -// NewMeter constructs and registers a new StandardMeter and launches a -// goroutine. +// NewMeterForced constructs a new StandardMeter and launches a goroutine no matter +// the global switch is enabled or not. +// Be sure to call Stop() once the meter is of no use to allow for garbage collection. +func NewMeterForced() Meter { + m := newStandardMeter() + arbiter.Lock() + defer arbiter.Unlock() + arbiter.meters[m] = struct{}{} + if !arbiter.started { + arbiter.started = true + go arbiter.tick() + } + return m +} + +// NewRegisteredMeter constructs and registers a new StandardMeter +// and launches a goroutine. // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func NewRegisteredMeter(name string, r Registry) Meter { @@ -59,6 +85,19 @@ func NewRegisteredMeter(name string, r Registry) Meter { return c } +// NewRegisteredMeterForced constructs and registers a new StandardMeter +// and launches a goroutine no matter the global switch is enabled or not. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredMeterForced(name string, r Registry) Meter { + c := NewMeterForced() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + // MeterSnapshot is a read-only copy of another Meter. type MeterSnapshot struct { count int64 From bd58098f2d88aabe5c555f5aa3d54d4786b7895b Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 18 Sep 2018 10:17:13 +0200 Subject: [PATCH 124/126] swarm: Chunk refactor improvements (#17683) * swarm/network: Protocol bump (for chunk refactor) * swarm/network: Increase discovery and stream protocol version too * swarm/network: Increase priority queue cap --- swarm/network/protocol.go | 4 ++-- swarm/network/protocol_test.go | 2 +- swarm/network/stream/stream.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index ef0956d5f8..d509d157bb 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -44,7 +44,7 @@ const ( // BzzSpec is the spec of the generic swarm handshake var BzzSpec = &protocols.Spec{ Name: "bzz", - Version: 6, + Version: 7, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ HandshakeMsg{}, @@ -54,7 +54,7 @@ var BzzSpec = &protocols.Spec{ // DiscoverySpec is the spec for the bzz discovery subprotocols var DiscoverySpec = &protocols.Spec{ Name: "hive", - Version: 5, + Version: 6, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ peersMsg{}, diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index c052c536a4..0fc8583711 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -31,7 +31,7 @@ import ( ) const ( - TestProtocolVersion = 6 + TestProtocolVersion = 7 TestProtocolNetworkID = 3 ) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 1f1f34b7bd..319fc62c9f 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -41,8 +41,8 @@ const ( Mid High Top - PriorityQueue = 4 // number of priority queues - Low, Mid, High, Top - PriorityQueueCap = 128 // queue capacity + PriorityQueue = 4 // number of priority queues - Low, Mid, High, Top + PriorityQueueCap = 4096 // queue capacity HashSize = 32 ) @@ -639,7 +639,7 @@ func (c *clientParams) clientCreated() { // Spec is the spec of the streamer protocol var Spec = &protocols.Spec{ Name: "stream", - Version: 5, + Version: 6, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ UnsubscribeMsg{}, From b8aa5980cf28313f36af9090c1491eb3fd0e1507 Mon Sep 17 00:00:00 2001 From: chenyufeng Date: Tue, 18 Sep 2018 16:30:39 +0800 Subject: [PATCH 125/126] cmd/puppeth: fix comment typo (#17690) * ethdb: unified code comment style. * puppeth: it is unnecessary to alloc pre-funded to 256 addresses * Revert "puppeth: it is unnecessary to alloc pre-funded to 256 addresses" This reverts commit 5e04fbccf0b8aca85030af1779bb7a949033d9d8. * puppeth: fix comment typo * Revert "ethdb: unified code comment style." This reverts commit a581efb3f06a96fc7aec0bfae03c7b6d5a0c1a77. * cmd/puppeth: fix comment typo --- cmd/puppeth/wizard_network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/puppeth/wizard_network.go b/cmd/puppeth/wizard_network.go index 0d2f976d53..83b10cf375 100644 --- a/cmd/puppeth/wizard_network.go +++ b/cmd/puppeth/wizard_network.go @@ -115,7 +115,7 @@ func (w *wizard) selectServer() string { // manageComponents displays a list of network components the user can tear down // and an option func (w *wizard) manageComponents() { - // List all the componens we can tear down, along with an entry to deploy a new one + // List all the components we can tear down, along with an entry to deploy a new one fmt.Println() var serviceHosts, serviceNames []string From 1f45ba9bb1c19489a6c8bf9caf100e56dcb79788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 18 Sep 2018 14:54:52 +0200 Subject: [PATCH 126/126] swarm/network: downgrade fetcher unable to request log message severity (#17692) --- swarm/network/fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/network/fetcher.go b/swarm/network/fetcher.go index 35e2f01328..413b40cb5b 100644 --- a/swarm/network/fetcher.go +++ b/swarm/network/fetcher.go @@ -215,7 +215,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { var err error sources, err = f.doRequest(ctx, gone, peers, sources) if err != nil { - log.Warn("unable to request", "request addr", f.addr, "err", err) + log.Info("unable to request", "request addr", f.addr, "err", err) } }