From 02aa86e6591d4a70f152d8dc579df25e9aa0cd12 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 14 Aug 2017 13:48:24 +0800 Subject: [PATCH 01/45] eth/downloader: exit loop when there is no more available task --- eth/downloader/downloader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 6ac58140a3..13f3b11c9a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -1093,6 +1093,10 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv throttled = true break } + // Short circuit if there is no more available task. + if pending() == 0 { + break + } // Reserve a chunk of fetches for a peer. A nil can mean either that // no more headers are available, or that the peer is known not to // have them. From dc92779c0a1e01de9561087a3a534d9d7cfdef3f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 4 Sep 2017 09:24:52 +0200 Subject: [PATCH 02/45] p2p: change ping ticker to timer (#15071) Using a Timer over Ticker seems to be a lot better, though I cannot fully account for why that it behaves so (since Ticker should be more bursty, but not necessarily more active over time, but that may depend on how long window it uses to decide on when to tick next) --- p2p/peer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/peer.go b/p2p/peer.go index a9c20189a4..fb4b39e950 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -190,7 +190,7 @@ loop: } func (p *Peer) pingLoop() { - ping := time.NewTicker(pingInterval) + ping := time.NewTimer(pingInterval) defer p.wg.Done() defer ping.Stop() for { @@ -200,6 +200,7 @@ func (p *Peer) pingLoop() { p.protoErr <- err return } + ping.Reset(pingInterval) case <-p.closed: return } From 23b51a68cbdd637824be906c9354a0b59c57491b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 4 Sep 2017 10:56:45 +0200 Subject: [PATCH 03/45] core/vm: avoid state lookup during gas calc for call (#15061) --- core/vm/gas_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 3b0698ce9e..13712295c9 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -324,7 +324,7 @@ func gasCall(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem eip158 = evm.ChainConfig().IsEIP158(evm.BlockNumber) ) if eip158 { - if evm.StateDB.Empty(address) && transfersValue { + if transfersValue && evm.StateDB.Empty(address) { gas += params.CallNewAccountGas } } else if !evm.StateDB.Exist(address) { From 1901521ed0423a5feaadd3635b10783c5a998151 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 4 Sep 2017 11:48:36 +0200 Subject: [PATCH 04/45] core: Fix flaw where underpriced locals were removed (#15081) * core: Fix flaw where underpriced locals were removed * core: minor code cleanups for tx pool tests --- core/tx_list.go | 1 + core/tx_pool_test.go | 62 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/core/tx_list.go b/core/tx_list.go index 0d87c20bc2..1087970fac 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -435,6 +435,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact } // Stop the discards if we've reached the threshold if tx.GasPrice().Cmp(threshold) >= 0 { + save = append(save, tx) break } // Non stale transaction found, discard unless local diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index ee1ddd4bbc..c1bcb1b2d6 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -976,7 +976,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { } } -// Tests that if transactions start being capped, transasctions are also removed from 'all' +// Tests that if transactions start being capped, transactions are also removed from 'all' func TestTransactionCapClearsFromAll(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() @@ -1141,6 +1141,66 @@ func TestTransactionPoolRepricing(t *testing.T) { } } +// Tests that setting the transaction pool gas price to a higher value does not +// remove local transactions. +func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { + // Create the pool to test the pricing enforcement with + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + + pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) + defer pool.Stop() + + // Create a number of test accounts and fund them + state, _ := pool.blockChain.State() + + keys := make([]*ecdsa.PrivateKey, 3) + for i := 0; i < len(keys); i++ { + keys[i], _ = crypto.GenerateKey() + state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000*1000000)) + } + // Create transaction (both pending and queued) with a linearly growing gasprice + for i := uint64(0); i < 500; i++ { + // Add pending + p_tx := pricedTransaction(i, big.NewInt(100000), big.NewInt(int64(i)), keys[2]) + if err := pool.AddLocal(p_tx); err != nil { + t.Fatal(err) + } + // Add queued + q_tx := pricedTransaction(i+501, big.NewInt(100000), big.NewInt(int64(i)), keys[2]) + if err := pool.AddLocal(q_tx); err != nil { + t.Fatal(err) + } + } + pending, queued := pool.Stats() + expPending, expQueued := 500, 500 + validate := func() { + pending, queued = pool.Stats() + if pending != expPending { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, expPending) + } + if queued != expQueued { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, expQueued) + } + + if err := validateTxPoolInternals(pool); err != nil { + t.Fatalf("pool internal state corrupted: %v", err) + } + } + validate() + + // Reprice the pool and check that nothing is dropped + pool.SetGasPrice(big.NewInt(2)) + validate() + + pool.SetGasPrice(big.NewInt(2)) + pool.SetGasPrice(big.NewInt(4)) + pool.SetGasPrice(big.NewInt(8)) + pool.SetGasPrice(big.NewInt(100)) + validate() +} + // Tests that when the pool reaches its global transaction limit, underpriced // transactions are gradually shifted out for more expensive ones and any gapped // pending transactions are moved into te queue. From e7408b5552002df7c3ba6a2351f14c533dfc5a36 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 4 Sep 2017 11:53:25 +0200 Subject: [PATCH 05/45] core/vm: Make MaxCodesize non-retroactive (#15072) * core/vm: Make max_codesize only applicable post Spurious Dragon/158/155/161/170 * tests: Remove expected failure --- core/vm/evm.go | 2 +- tests/state_test.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index caf8b4507d..495d9beea1 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -336,7 +336,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I ret, err = run(evm, snapshot, contract, nil) // check whether the max code size has been exceeded - maxCodeSizeExceeded := len(ret) > params.MaxCodeSize + maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize // if the contract creation ran successfully and no errors were returned // calculate the gas required to store the code. If the code could not // be stored due to not enough gas set an error and let it be handled diff --git a/tests/state_test.go b/tests/state_test.go index 3d7b29012e..4b6ba8b31b 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -35,8 +35,6 @@ func TestState(t *testing.T) { st.skipLoad(`^stTransactionTest/OverflowGasRequire\.json`) // gasLimit > 256 bits st.skipLoad(`^stTransactionTest/zeroSigTransa[^/]*\.json`) // EIP-86 is not supported yet // Expected failures: - st.fails(`^stCodeSizeLimit/codesizeOOGInvalidSize\.json/(Frontier|Homestead|EIP150)`, - "code size limit implementation is not conditional on fork") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") From 504278e8398ef787d4f6f73178afa342a31a88ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 11:16:46 +0300 Subject: [PATCH 06/45] build: bump PPA builders to Go 1.9 (#15083) --- build/ci-notes.md | 6 +++--- build/deb.control | 2 +- build/deb.rules | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/ci-notes.md b/build/ci-notes.md index 7574bfffa2..78e9575c01 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -21,18 +21,18 @@ variable which Travis CI makes available to certain builds. We want to build go-ethereum with the most recent version of Go, irrespective of the Go version that is available in the main Ubuntu repository. In order to make this possible, our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on -golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies +golang-1.9, which is co-installable alongside the regular golang package. PPA dependencies can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies ## Building Packages Locally (for testing) You need to run Ubuntu to do test packaging. -Add the gophers PPA and install Go 1.8 and Debian packaging tools: +Add the gophers PPA and install Go 1.9 and Debian packaging tools: $ sudo apt-add-repository ppa:gophers/ubuntu/archive $ sudo apt-get update - $ sudo apt-get install build-essential golang-1.8 devscripts debhelper + $ sudo apt-get install build-essential golang-1.9 devscripts debhelper Create the source packages: diff --git a/build/deb.control b/build/deb.control index 7394754ef0..5c9ce6705c 100644 --- a/build/deb.control +++ b/build/deb.control @@ -2,7 +2,7 @@ Source: {{.Name}} Section: science Priority: extra Maintainer: {{.Author}} -Build-Depends: debhelper (>= 8.0.0), golang-1.8 +Build-Depends: debhelper (>= 8.0.0), golang-1.9 Standards-Version: 3.9.5 Homepage: https://ethereum.org Vcs-Git: git://github.com/ethereum/go-ethereum.git diff --git a/build/deb.rules b/build/deb.rules index b3fe5267f3..7a78525139 100644 --- a/build/deb.rules +++ b/build/deb.rules @@ -5,7 +5,7 @@ #export DH_VERBOSE=1 override_dh_auto_build: - build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + build/env.sh /usr/lib/go-1.9/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} override_dh_auto_test: From 8f567dc8a2cf1e963a972a0d986703b41988f50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 12:16:59 +0300 Subject: [PATCH 07/45] Dockerfile: multi-stage builds, Go 1.9 --- Dockerfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 947f045e59..17fa40951e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,15 @@ -FROM alpine:3.6 +# Build Geth in a stock Go builder container +FROM golang:1.9-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers ADD . /go-ethereum -RUN \ - apk add --no-cache git go make gcc musl-dev linux-headers && \ - (cd go-ethereum && make geth) && \ - cp go-ethereum/build/bin/geth /usr/local/bin/ && \ - apk del git go make gcc musl-dev linux-headers && \ - rm -rf /go-ethereum +RUN cd /go-ethereum && make geth -EXPOSE 8545 30303 30303/udp +# Pull Geth into a second stage deploy alpine container +FROM alpine:latest +COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ + +EXPOSE 8545 8546 30303 30303/udp ENTRYPOINT ["geth"] From 8cab3ab4350fe06f5dd1e9bb9ce5f559c08bc4cf Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Sep 2017 11:24:26 +0200 Subject: [PATCH 08/45] cmd/evm: adds ability to run individual state test file (#14998) * cmd/evm: adds ability to run individual state test file * cmd/evm: Fix statetest runner to be more json friendly * cmd/evm, tests: minor polishes, dump state on fail --- cmd/evm/main.go | 1 + cmd/evm/staterunner.go | 119 +++++++++++++++++++++++++++++++++++++++ tests/state_test.go | 3 +- tests/state_test_util.go | 12 ++-- 4 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 cmd/evm/staterunner.go diff --git a/cmd/evm/main.go b/cmd/evm/main.go index a2e3b048e7..6c39cf8b88 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -143,6 +143,7 @@ func init() { compileCommand, disasmCommand, runCommand, + stateTestCommand, } } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go new file mode 100644 index 0000000000..3a4cc51c03 --- /dev/null +++ b/cmd/evm/staterunner.go @@ -0,0 +1,119 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/tests" + + cli "gopkg.in/urfave/cli.v1" +) + +var stateTestCommand = cli.Command{ + Action: stateTestCmd, + Name: "statetest", + Usage: "executes the given state tests", + ArgsUsage: "", +} + +type StatetestResult struct { + Name string `json:"name"` + Pass bool `json:"pass"` + Fork string `json:"fork"` + Error string `json:"error,omitempty"` + State *state.Dump `json:"state,omitempty"` +} + +func stateTestCmd(ctx *cli.Context) error { + if len(ctx.Args().First()) == 0 { + return errors.New("path-to-test argument required") + } + // Configure the go-ethereum logger + glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) + glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) + log.Root().SetHandler(glogger) + + // Configure the EVM logger + config := &vm.LogConfig{ + DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name), + DisableStack: ctx.GlobalBool(DisableStackFlag.Name), + } + var ( + tracer vm.Tracer + debugger *vm.StructLogger + ) + switch { + case ctx.GlobalBool(MachineFlag.Name): + tracer = NewJSONLogger(config, os.Stderr) + + case ctx.GlobalBool(DebugFlag.Name): + debugger = vm.NewStructLogger(config) + tracer = debugger + + default: + debugger = vm.NewStructLogger(config) + } + // Load the test content from the input file + src, err := ioutil.ReadFile(ctx.Args().First()) + if err != nil { + return err + } + var tests map[string]tests.StateTest + if err = json.Unmarshal(src, &tests); err != nil { + return err + } + // Iterate over all the tests, run them and aggregate the results + cfg := vm.Config{ + Tracer: tracer, + Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), + } + results := make([]StatetestResult, 0, len(tests)) + for key, test := range tests { + for _, st := range test.Subtests() { + // Run the test and aggregate the result + result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} + if state, err := test.Run(st, cfg); err != nil { + // Test failed, mark as so and dump any state to aid debugging + result.Pass, result.Error = false, err.Error() + if ctx.GlobalBool(DumpFlag.Name) && state != nil { + dump := state.RawDump() + result.State = &dump + } + } + results = append(results, *result) + + // Print any structured logs collected + if ctx.GlobalBool(DebugFlag.Name) { + if debugger != nil { + fmt.Fprintln(os.Stderr, "#### TRACE ####") + vm.WriteTrace(os.Stderr, debugger.StructLogs()) + } + } + } + } + out, _ := json.MarshalIndent(results, "", " ") + fmt.Println(string(out)) + return nil +} diff --git a/tests/state_test.go b/tests/state_test.go index 4b6ba8b31b..1fb7f5908f 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -50,7 +50,8 @@ func TestState(t *testing.T) { t.Skip("constantinople not supported yet") } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - return st.checkFailure(t, name, test.Run(subtest, vmconfig)) + _, err := test.Run(subtest, vmconfig) + return st.checkFailure(t, name, err) }) }) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index ecaa6c6684..64bf09cb42 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -120,10 +120,10 @@ func (t *StateTest) Subtests() []StateSubtest { } // Run executes a specific subtest. -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) error { +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateDB, error) { config, ok := Forks[subtest.Fork] if !ok { - return UnsupportedForkError{subtest.Fork} + return nil, UnsupportedForkError{subtest.Fork} } block, _ := t.genesis(config).ToBlock() db, _ := ethdb.NewMemDatabase() @@ -132,7 +132,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) error { post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) if err != nil { - return err + return nil, err } context := core.NewEVMContext(msg, block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash @@ -145,13 +145,13 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) error { statedb.RevertToSnapshot(snapshot) } if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { - return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) + return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } root, _ := statedb.CommitTo(db, config.IsEIP158(block.Number())) if root != common.Hash(post.Root) { - return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) + return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } - return nil + return statedb, nil } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { From da7d57e07c04dcbb7cc20b35f6606ef3f4c400e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 4 Sep 2017 22:35:00 +0300 Subject: [PATCH 09/45] core: make txpool operate on immutable state --- core/blockchain.go | 12 -- core/error.go | 4 + core/state_transition.go | 8 +- core/tx_list.go | 1 + core/tx_pool.go | 203 +++++++++++++++------------ core/tx_pool_test.go | 281 +++++++++++++------------------------ eth/api_backend.go | 8 -- internal/ethapi/api.go | 1 - internal/ethapi/backend.go | 1 - miner/worker.go | 21 ++- 10 files changed, 233 insertions(+), 307 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f3ca4e08cf..0bb12fc190 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -81,7 +81,6 @@ type BlockChain struct { hc *HeaderChain chainDb ethdb.Database - rmTxFeed event.Feed rmLogsFeed event.Feed chainFeed event.Feed chainSideFeed event.Feed @@ -1194,15 +1193,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { for _, tx := range diff { DeleteTxLookupEntry(bc.chainDb, tx.Hash()) } - // Must be posted in a goroutine because of the transaction pool trying - // to acquire the chain manager lock - if len(diff) > 0 { - go bc.rmTxFeed.Send(RemovedTransactionEvent{diff}) - } if len(deletedLogs) > 0 { go bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) } - if len(oldChain) > 0 { go func() { for _, block := range oldChain { @@ -1401,11 +1394,6 @@ func (bc *BlockChain) Config() *params.ChainConfig { return bc.config } // Engine retrieves the blockchain's consensus engine. func (bc *BlockChain) Engine() consensus.Engine { return bc.engine } -// SubscribeRemovedTxEvent registers a subscription of RemovedTransactionEvent. -func (bc *BlockChain) SubscribeRemovedTxEvent(ch chan<- RemovedTransactionEvent) event.Subscription { - return bc.scope.Track(bc.rmTxFeed.Subscribe(ch)) -} - // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) diff --git a/core/error.go b/core/error.go index 9ac4fff514..410eca1e1e 100644 --- a/core/error.go +++ b/core/error.go @@ -28,4 +28,8 @@ var ( // ErrBlacklistedHash is returned if a block to import is on the blacklist. ErrBlacklistedHash = errors.New("blacklisted hash") + + // ErrNonceTooHigh is returned if the nonce of a transaction is higher than the + // next one expected based on the local chain. + ErrNonceTooHigh = errors.New("nonce too high") ) diff --git a/core/state_transition.go b/core/state_transition.go index bab4540be8..e7a0685893 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -18,7 +18,6 @@ package core import ( "errors" - "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -197,8 +196,11 @@ func (st *StateTransition) preCheck() error { // Make sure this transaction's nonce is correct if msg.CheckNonce() { - if n := st.state.GetNonce(sender.Address()); n != msg.Nonce() { - return fmt.Errorf("invalid nonce: have %d, expected %d", msg.Nonce(), n) + nonce := st.state.GetNonce(sender.Address()) + if nonce < msg.Nonce() { + return ErrNonceTooHigh + } else if nonce > msg.Nonce() { + return ErrNonceTooLow } } return st.buyGas() diff --git a/core/tx_list.go b/core/tx_list.go index 1087970fac..2935929d72 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -298,6 +298,7 @@ func (l *txList) Filter(costLimit, gasLimit *big.Int) (types.Transactions, types // If the list was strict, filter anything above the lowest nonce var invalids types.Transactions + if l.strict && len(removed) > 0 { lowest := uint64(math.MaxUint64) for _, tx := range removed { diff --git a/core/tx_pool.go b/core/tx_pool.go index d835c94d1a..f41fbe069a 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -105,10 +105,11 @@ var ( // blockChain provides the state of blockchain and current gas limit to do // some pre checks in tx pool and event subscribers. type blockChain interface { - State() (*state.StateDB, error) - GasLimit() *big.Int + CurrentHeader() *types.Header SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription - SubscribeRemovedTxEvent(ch chan<- RemovedTransactionEvent) event.Subscription + + GetBlock(hash common.Hash, number uint64) *types.Block + StateAt(root common.Hash) (*state.StateDB, error) } // TxPoolConfig are the configuration parameters of the transaction pool. @@ -174,18 +175,19 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { type TxPool struct { config TxPoolConfig chainconfig *params.ChainConfig - blockChain blockChain - pendingState *state.ManagedState + chain blockChain gasPrice *big.Int txFeed event.Feed scope event.SubscriptionScope chainHeadCh chan ChainHeadEvent chainHeadSub event.Subscription - rmTxCh chan RemovedTransactionEvent - rmTxSub event.Subscription signer types.Signer mu sync.RWMutex + currentState *state.StateDB // Current state in the blockchain head + pendingState *state.ManagedState // Pending state tracking virtual nonces + currentMaxGas *big.Int // Current gas limit for transaction caps + locals *accountSet // Set of local transaction to exepmt from evicion rules journal *txJournal // Journal of local transaction to back up to disk @@ -202,28 +204,26 @@ type TxPool struct { // NewTxPool creates a new transaction pool to gather, sort and filter inbound // trnsactions from the network. -func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, blockChain blockChain) *TxPool { +func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain blockChain) *TxPool { // Sanitize the input to ensure no vulnerable gas prices are set config = (&config).sanitize() // Create the transaction pool with its initial settings pool := &TxPool{ - config: config, - chainconfig: chainconfig, - blockChain: blockChain, - signer: types.NewEIP155Signer(chainconfig.ChainId), - pending: make(map[common.Address]*txList), - queue: make(map[common.Address]*txList), - beats: make(map[common.Address]time.Time), - all: make(map[common.Hash]*types.Transaction), - chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), - rmTxCh: make(chan RemovedTransactionEvent, rmTxChanSize), - gasPrice: new(big.Int).SetUint64(config.PriceLimit), - pendingState: nil, + config: config, + chainconfig: chainconfig, + chain: chain, + signer: types.NewEIP155Signer(chainconfig.ChainId), + pending: make(map[common.Address]*txList), + queue: make(map[common.Address]*txList), + beats: make(map[common.Address]time.Time), + all: make(map[common.Hash]*types.Transaction), + chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), + gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) pool.priced = newTxPricedList(&pool.all) - pool.reset() + pool.reset(nil, chain.CurrentHeader()) // If local transactions and journaling is enabled, load from disk if !config.NoLocals && config.Journal != "" { @@ -237,8 +237,8 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, blockChain } } // Subscribe events from blockchain - pool.chainHeadSub = pool.blockChain.SubscribeChainHeadEvent(pool.chainHeadCh) - pool.rmTxSub = pool.blockChain.SubscribeRemovedTxEvent(pool.rmTxCh) + pool.chainHeadSub = pool.chain.SubscribeChainHeadEvent(pool.chainHeadCh) + // Start the event loop and return pool.wg.Add(1) go pool.loop() @@ -264,31 +264,28 @@ func (pool *TxPool) loop() { journal := time.NewTicker(pool.config.Rejournal) defer journal.Stop() + // Track the previous head headers for transaction reorgs + head := pool.chain.CurrentHeader() + // Keep waiting for and reacting to the various events for { select { // Handle ChainHeadEvent case ev := <-pool.chainHeadCh: - pool.mu.Lock() if ev.Block != nil { + pool.mu.Lock() if pool.chainconfig.IsHomestead(ev.Block.Number()) { pool.homestead = true } + pool.reset(head, ev.Block.Header()) + head = ev.Block.Header() + pool.mu.Unlock() } - pool.reset() - pool.mu.Unlock() // Be unsubscribed due to system stopped case <-pool.chainHeadSub.Err(): return - // Handle RemovedTransactionEvent - case ev := <-pool.rmTxCh: - pool.addTxs(ev.Txs, false) - // Be unsubscribed due to system stopped - case <-pool.rmTxSub.Err(): - return - // Handle stats reporting ticks case <-report.C: pool.mu.RLock() @@ -333,28 +330,76 @@ func (pool *TxPool) loop() { // lockedReset is a wrapper around reset to allow calling it in a thread safe // manner. This method is only ever used in the tester! -func (pool *TxPool) lockedReset() { +func (pool *TxPool) lockedReset(oldHead, newHead *types.Header) { pool.mu.Lock() defer pool.mu.Unlock() - pool.reset() + pool.reset(oldHead, newHead) } // reset retrieves the current state of the blockchain and ensures the content // of the transaction pool is valid with regard to the chain state. -func (pool *TxPool) reset() { - currentState, err := pool.blockChain.State() +func (pool *TxPool) reset(oldHead, newHead *types.Header) { + // If we're reorging an old state, reinject all dropped transactions + var reinject types.Transactions + + if oldHead != nil && oldHead.Hash() != newHead.ParentHash { + var discarded, included types.Transactions + + var ( + rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) + add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) + ) + for rem.NumberU64() > add.NumberU64() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + } + for add.NumberU64() > rem.NumberU64() { + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + for rem.Hash() != add.Hash() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + reinject = types.TxDifference(discarded, included) + } + // Initialize the internal state to the current head + if newHead == nil { + newHead = pool.chain.CurrentHeader() // Special case during testing + } + statedb, err := pool.chain.StateAt(newHead.Root) if err != nil { - log.Error("Failed reset txpool state", "err", err) + log.Error("Failed to reset txpool state", "err", err) return } - pool.pendingState = state.ManageState(currentState) + pool.currentState = statedb + pool.pendingState = state.ManageState(statedb) + pool.currentMaxGas = newHead.GasLimit + + // Inject any transactions discarded due to reorgs + log.Debug("Reinjecting stale transactions", "count", len(reinject)) + pool.addTxsLocked(reinject, false) // validate the pool of pending transactions, this will remove // any transactions that have been included in the block or // have been invalidated because of another transaction (e.g. // higher gas price) - pool.demoteUnexecutables(currentState) + pool.demoteUnexecutables() // Update all accounts to the latest known pending nonce for addr, list := range pool.pending { @@ -363,16 +408,16 @@ func (pool *TxPool) reset() { } // Check the queue and move transactions over to the pending if possible // or remove those that have become invalid - pool.promoteExecutables(currentState, nil) + pool.promoteExecutables(nil) } // Stop terminates the transaction pool. func (pool *TxPool) Stop() { // Unsubscribe all subscriptions registered from txpool pool.scope.Close() + // Unsubscribe subscriptions registered from blockchain pool.chainHeadSub.Unsubscribe() - pool.rmTxSub.Unsubscribe() pool.wg.Wait() if pool.journal != nil { @@ -442,8 +487,8 @@ func (pool *TxPool) stats() (int, int) { // Content retrieves the data content of the transaction pool, returning all the // pending as well as queued transactions, grouped by account and sorted by nonce. func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { - pool.mu.RLock() - defer pool.mu.RUnlock() + pool.mu.Lock() + defer pool.mu.Unlock() pending := make(map[common.Address]types.Transactions) for addr, list := range pool.pending { @@ -499,7 +544,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { return ErrNegativeValue } // Ensure the transaction doesn't exceed the current block limit gas. - if pool.blockChain.GasLimit().Cmp(tx.Gas()) < 0 { + if pool.currentMaxGas.Cmp(tx.Gas()) < 0 { return ErrGasLimit } // Make sure the transaction is signed properly @@ -513,16 +558,12 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { return ErrUnderpriced } // Ensure the transaction adheres to nonce ordering - currentState, err := pool.blockChain.State() - if err != nil { - return err - } - if currentState.GetNonce(from) > tx.Nonce() { + if pool.currentState.GetNonce(from) > tx.Nonce() { return ErrNonceTooLow } // Transactor should have enough funds to cover the costs // cost == V + GP * GL - if currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { + if pool.currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { return ErrInsufficientFunds } intrGas := IntrinsicGas(tx.Data(), tx.To() == nil, pool.homestead) @@ -721,12 +762,8 @@ func (pool *TxPool) addTx(tx *types.Transaction, local bool) error { } // If we added a new transaction, run promotion checks and return if !replace { - state, err := pool.blockChain.State() - if err != nil { - return err - } from, _ := types.Sender(pool.signer, tx) // already validated - pool.promoteExecutables(state, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) } return nil } @@ -736,6 +773,12 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error { pool.mu.Lock() defer pool.mu.Unlock() + return pool.addTxsLocked(txs, local) +} + +// addTxsLocked attempts to queue a batch of transactions if they are valid, +// whilst assuming the transaction pool lock is already held. +func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) error { // Add the batch of transaction, tracking the accepted ones dirty := make(map[common.Address]struct{}) for _, tx := range txs { @@ -748,15 +791,11 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error { } // Only reprocess the internal state if something was actually added if len(dirty) > 0 { - state, err := pool.blockChain.State() - if err != nil { - return err - } addrs := make([]common.Address, 0, len(dirty)) for addr, _ := range dirty { addrs = append(addrs, addr) } - pool.promoteExecutables(state, addrs) + pool.promoteExecutables(addrs) } return nil } @@ -770,24 +809,6 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction { return pool.all[hash] } -// Remove removes the transaction with the given hash from the pool. -func (pool *TxPool) Remove(hash common.Hash) { - pool.mu.Lock() - defer pool.mu.Unlock() - - pool.removeTx(hash) -} - -// RemoveBatch removes all given transactions from the pool. -func (pool *TxPool) RemoveBatch(txs types.Transactions) { - pool.mu.Lock() - defer pool.mu.Unlock() - - for _, tx := range txs { - pool.removeTx(tx.Hash()) - } -} - // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash) { @@ -834,9 +855,7 @@ func (pool *TxPool) removeTx(hash common.Hash) { // promoteExecutables moves transactions that have become processable from the // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. -func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.Address) { - gaslimit := pool.blockChain.GasLimit() - +func (pool *TxPool) promoteExecutables(accounts []common.Address) { // Gather all the accounts potentially needing updates if accounts == nil { accounts = make([]common.Address, 0, len(pool.queue)) @@ -851,14 +870,14 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A continue // Just in case someone calls with a non existing account } // Drop all transactions that are deemed too old (low nonce) - for _, tx := range list.Forward(state.GetNonce(addr)) { + for _, tx := range list.Forward(pool.currentState.GetNonce(addr)) { hash := tx.Hash() log.Trace("Removed old queued transaction", "hash", hash) delete(pool.all, hash) pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(state.GetBalance(addr), gaslimit) + drops, _ := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable queued transaction", "hash", hash) @@ -1003,12 +1022,10 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A // demoteUnexecutables removes invalid and processed transactions from the pools // executable/pending queue and any subsequent transactions that become unexecutable // are moved back into the future queue. -func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { - gaslimit := pool.blockChain.GasLimit() - +func (pool *TxPool) demoteUnexecutables() { // Iterate over all accounts and demote any non-executable transactions for addr, list := range pool.pending { - nonce := state.GetNonce(addr) + nonce := pool.currentState.GetNonce(addr) // Drop all transactions that are deemed too old (low nonce) for _, tx := range list.Forward(nonce) { @@ -1018,7 +1035,7 @@ func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later - drops, invalids := list.Filter(state.GetBalance(addr), gaslimit) + drops, invalids := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable pending transaction", "hash", hash) @@ -1031,6 +1048,14 @@ func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { log.Trace("Demoting pending transaction", "hash", hash) pool.enqueueTx(hash, tx) } + // If there's a gap in front, warn (should never happen) and postpone all transactions + if list.Len() > 0 && list.txs.Get(nonce) == nil { + for _, tx := range list.Cap(0) { + hash := tx.Hash() + log.Error("Demoting invalidated transaction", "hash", hash) + pool.enqueueTx(hash, tx) + } + } // Delete the entire queue entry if it became empty. if list.Empty() { delete(pool.pending, addr) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index c1bcb1b2d6..cdd45b4b1d 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -48,23 +48,24 @@ type testBlockChain struct { statedb *state.StateDB gasLimit *big.Int chainHeadFeed *event.Feed - rmTxFeed *event.Feed } -func (bc *testBlockChain) State() (*state.StateDB, error) { - return bc.statedb, nil -} - -func (bc *testBlockChain) GasLimit() *big.Int { - return new(big.Int).Set(bc.gasLimit) +func (bc *testBlockChain) CurrentHeader() *types.Header { + return &types.Header{ + GasLimit: bc.gasLimit, + } } func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription { return bc.chainHeadFeed.Subscribe(ch) } -func (bc *testBlockChain) SubscribeRemovedTxEvent(ch chan<- RemovedTransactionEvent) event.Subscription { - return bc.rmTxFeed.Subscribe(ch) +func (bc *testBlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { + return types.NewBlock(bc.CurrentHeader(), nil, nil, nil) +} + +func (bc *testBlockChain) StateAt(common.Hash) (*state.StateDB, error) { + return bc.statedb, nil } func transaction(nonce uint64, gaslimit *big.Int, key *ecdsa.PrivateKey) *types.Transaction { @@ -79,7 +80,7 @@ func pricedTransaction(nonce uint64, gaslimit, gasprice *big.Int, key *ecdsa.Pri func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} key, _ := crypto.GenerateKey() pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -159,7 +160,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { // setup pool with 2 transaction in it statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) - blockchain := &testChain{&testBlockChain{statedb, big.NewInt(1000000000), new(event.Feed), new(event.Feed)}, address, &trigger} + blockchain := &testChain{&testBlockChain{statedb, big.NewInt(1000000000), new(event.Feed)}, address, &trigger} tx0 := transaction(0, big.NewInt(100000), key) tx1 := transaction(1, big.NewInt(100000), key) @@ -182,7 +183,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { // trigger state change in the background trigger = true - pool.lockedReset() + pool.lockedReset(nil, nil) pendingTx, err := pool.Pending() if err != nil { @@ -205,20 +206,20 @@ func TestInvalidTransactions(t *testing.T) { tx := transaction(0, big.NewInt(100), key) from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1)) + + pool.currentState.AddBalance(from, big.NewInt(1)) if err := pool.AddRemote(tx); err != ErrInsufficientFunds { t.Error("expected", ErrInsufficientFunds) } balance := new(big.Int).Add(tx.Value(), new(big.Int).Mul(tx.Gas(), tx.GasPrice())) - currentState.AddBalance(from, balance) + pool.currentState.AddBalance(from, balance) if err := pool.AddRemote(tx); err != ErrIntrinsicGas { t.Error("expected", ErrIntrinsicGas, "got", err) } - currentState.SetNonce(from, 1) - currentState.AddBalance(from, big.NewInt(0xffffffffffffff)) + pool.currentState.SetNonce(from, 1) + pool.currentState.AddBalance(from, big.NewInt(0xffffffffffffff)) tx = transaction(0, big.NewInt(100000), key) if err := pool.AddRemote(tx); err != ErrNonceTooLow { t.Error("expected", ErrNonceTooLow) @@ -240,21 +241,20 @@ func TestTransactionQueue(t *testing.T) { tx := transaction(0, big.NewInt(100), key) from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1000)) - pool.lockedReset() + pool.currentState.AddBalance(from, big.NewInt(1000)) + pool.lockedReset(nil, nil) pool.enqueueTx(tx.Hash(), tx) - pool.promoteExecutables(currentState, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) if len(pool.pending) != 1 { t.Error("expected valid txs to be 1 is", len(pool.pending)) } tx = transaction(1, big.NewInt(100), key) from, _ = deriveSender(tx) - currentState.SetNonce(from, 2) + pool.currentState.SetNonce(from, 2) pool.enqueueTx(tx.Hash(), tx) - pool.promoteExecutables(currentState, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { t.Error("expected transaction to be in tx pool") } @@ -270,15 +270,14 @@ func TestTransactionQueue(t *testing.T) { tx2 := transaction(10, big.NewInt(100), key) tx3 := transaction(11, big.NewInt(100), key) from, _ = deriveSender(tx1) - currentState, _ = pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1000)) - pool.lockedReset() + pool.currentState.AddBalance(from, big.NewInt(1000)) + pool.lockedReset(nil, nil) pool.enqueueTx(tx1.Hash(), tx1) pool.enqueueTx(tx2.Hash(), tx2) pool.enqueueTx(tx3.Hash(), tx3) - pool.promoteExecutables(currentState, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) if len(pool.pending) != 1 { t.Error("expected tx pool to be 1, got", len(pool.pending)) @@ -288,45 +287,13 @@ func TestTransactionQueue(t *testing.T) { } } -func TestRemoveTx(t *testing.T) { - pool, key := setupTxPool() - defer pool.Stop() - - addr := crypto.PubkeyToAddress(key.PublicKey) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(1)) - - tx1 := transaction(0, big.NewInt(100), key) - tx2 := transaction(2, big.NewInt(100), key) - - pool.promoteTx(addr, tx1.Hash(), tx1) - pool.enqueueTx(tx2.Hash(), tx2) - - if len(pool.queue) != 1 { - t.Error("expected queue to be 1, got", len(pool.queue)) - } - if len(pool.pending) != 1 { - t.Error("expected pending to be 1, got", len(pool.pending)) - } - pool.Remove(tx1.Hash()) - pool.Remove(tx2.Hash()) - - if len(pool.queue) > 0 { - t.Error("expected queue to be 0, got", len(pool.queue)) - } - if len(pool.pending) > 0 { - t.Error("expected pending to be 0, got", len(pool.pending)) - } -} - func TestNegativeValue(t *testing.T) { pool, key := setupTxPool() defer pool.Stop() tx, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(-1), big.NewInt(100), big.NewInt(1), nil), types.HomesteadSigner{}, key) from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1)) + pool.currentState.AddBalance(from, big.NewInt(1)) if err := pool.AddRemote(tx); err != ErrNegativeValue { t.Error("expected", ErrNegativeValue, "got", err) } @@ -340,10 +307,10 @@ func TestTransactionChainFork(t *testing.T) { resetState := func() { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool.blockChain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(100000000000000)) - pool.lockedReset() + statedb.AddBalance(addr, big.NewInt(100000000000000)) + + pool.chain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} + pool.lockedReset(nil, nil) } resetState() @@ -351,7 +318,7 @@ func TestTransactionChainFork(t *testing.T) { if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) } - pool.RemoveBatch([]*types.Transaction{tx}) + pool.removeTx(tx.Hash()) // reset the pool's internal state resetState() @@ -368,10 +335,10 @@ func TestTransactionDoubleNonce(t *testing.T) { resetState := func() { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool.blockChain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(100000000000000)) - pool.lockedReset() + statedb.AddBalance(addr, big.NewInt(100000000000000)) + + pool.chain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} + pool.lockedReset(nil, nil) } resetState() @@ -387,8 +354,7 @@ func TestTransactionDoubleNonce(t *testing.T) { if replace, err := pool.add(tx2, false); err != nil || !replace { t.Errorf("second transaction insert failed (%v) or not reported replacement (%v)", err, replace) } - state, _ := pool.blockChain.State() - pool.promoteExecutables(state, []common.Address{addr}) + pool.promoteExecutables([]common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } @@ -397,7 +363,7 @@ func TestTransactionDoubleNonce(t *testing.T) { } // Add the third transaction and ensure it's not saved (smaller price) pool.add(tx3, false) - pool.promoteExecutables(state, []common.Address{addr}) + pool.promoteExecutables([]common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } @@ -415,8 +381,7 @@ func TestMissingNonce(t *testing.T) { defer pool.Stop() addr := crypto.PubkeyToAddress(key.PublicKey) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(100000000000000)) + pool.currentState.AddBalance(addr, big.NewInt(100000000000000)) tx := transaction(1, big.NewInt(100000), key) if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) @@ -432,47 +397,25 @@ func TestMissingNonce(t *testing.T) { } } -func TestNonceRecovery(t *testing.T) { +func TestTransactionNonceRecovery(t *testing.T) { const n = 10 pool, key := setupTxPool() defer pool.Stop() addr := crypto.PubkeyToAddress(key.PublicKey) - currentState, _ := pool.blockChain.State() - currentState.SetNonce(addr, n) - currentState.AddBalance(addr, big.NewInt(100000000000000)) - pool.lockedReset() + pool.currentState.SetNonce(addr, n) + pool.currentState.AddBalance(addr, big.NewInt(100000000000000)) + pool.lockedReset(nil, nil) + tx := transaction(n, big.NewInt(100000), key) if err := pool.AddRemote(tx); err != nil { t.Error(err) } // simulate some weird re-order of transactions and missing nonce(s) - currentState.SetNonce(addr, n-1) - pool.lockedReset() - if fn := pool.pendingState.GetNonce(addr); fn != n+1 { - t.Errorf("expected nonce to be %d, got %d", n+1, fn) - } -} - -func TestRemovedTxEvent(t *testing.T) { - pool, key := setupTxPool() - defer pool.Stop() - - tx := transaction(0, big.NewInt(1000000), key) - from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1000000000000)) - pool.lockedReset() - blockChain, _ := pool.blockChain.(*testBlockChain) - blockChain.rmTxFeed.Send(RemovedTransactionEvent{types.Transactions{tx}}) - blockChain.chainHeadFeed.Send(ChainHeadEvent{nil}) - // wait for handling events - <-time.After(500 * time.Millisecond) - if pool.pending[from].Len() != 1 { - t.Error("expected 1 pending tx, got", pool.pending[from].Len()) - } - if len(pool.all) != 1 { - t.Error("expected 1 total transactions, got", len(pool.all)) + pool.currentState.SetNonce(addr, n-1) + pool.lockedReset(nil, nil) + if fn := pool.pendingState.GetNonce(addr); fn != n-1 { + t.Errorf("expected nonce to be %d, got %d", n-1, fn) } } @@ -484,9 +427,7 @@ func TestTransactionDropping(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000)) + pool.currentState.AddBalance(account, big.NewInt(1000)) // Add some pending and some queued transactions var ( @@ -514,7 +455,7 @@ func TestTransactionDropping(t *testing.T) { if len(pool.all) != 6 { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) } - pool.lockedReset() + pool.lockedReset(nil, nil) if pool.pending[account].Len() != 3 { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } @@ -525,8 +466,8 @@ func TestTransactionDropping(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) } // Reduce the balance of the account, and check that invalidated transactions are dropped - state.AddBalance(account, big.NewInt(-650)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(-650)) + pool.lockedReset(nil, nil) if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) @@ -550,8 +491,8 @@ func TestTransactionDropping(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) } // Reduce the block gas limit, check that invalidated transactions are dropped - pool.blockChain.(*testBlockChain).gasLimit = big.NewInt(100) - pool.lockedReset() + pool.chain.(*testBlockChain).gasLimit = big.NewInt(100) + pool.lockedReset(nil, nil) if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) @@ -579,9 +520,7 @@ func TestTransactionPostponing(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000)) + pool.currentState.AddBalance(account, big.NewInt(1000)) // Add a batch consecutive pending transactions for validation txns := []*types.Transaction{} @@ -605,7 +544,7 @@ func TestTransactionPostponing(t *testing.T) { if len(pool.all) != len(txns) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)) } - pool.lockedReset() + pool.lockedReset(nil, nil) if pool.pending[account].Len() != len(txns) { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), len(txns)) } @@ -616,8 +555,8 @@ func TestTransactionPostponing(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)) } // Reduce the balance of the account, and check that transactions are reorganised - state.AddBalance(account, big.NewInt(-750)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(-750)) + pool.lockedReset(nil, nil) if _, ok := pool.pending[account].txs.items[txns[0].Nonce()]; !ok { t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txns[0]) @@ -655,10 +594,7 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep queuing up transactions and make sure all above a limit are dropped for i := uint64(1); i <= testTxPoolConfig.AccountQueue+5; i++ { @@ -699,7 +635,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.NoLocals = nolocals @@ -709,12 +645,10 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { defer pool.Stop() // Create a number of test accounts and fund them (last one will be the local) - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } local := keys[len(keys)-1] @@ -790,7 +724,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { // Create the pool to test the non-expiration enforcement db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.Lifetime = time.Second @@ -803,9 +737,8 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { local, _ := crypto.GenerateKey() remote, _ := crypto.GenerateKey() - state, _ := pool.blockChain.State() - state.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) - state.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) // Add the two transactions and ensure they both are queued up if err := pool.AddLocal(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), local)); err != nil { @@ -854,10 +787,7 @@ func TestTransactionPendingLimiting(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep queuing up transactions and make sure all above a limit are dropped for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { @@ -887,8 +817,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { defer pool1.Stop() account1, _ := deriveSender(transaction(0, big.NewInt(0), key1)) - state1, _ := pool1.blockChain.State() - state1.AddBalance(account1, big.NewInt(1000000)) + pool1.currentState.AddBalance(account1, big.NewInt(1000000)) for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { if err := pool1.AddRemote(transaction(origin+i, big.NewInt(100000), key1)); err != nil { @@ -900,8 +829,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { defer pool2.Stop() account2, _ := deriveSender(transaction(0, big.NewInt(0), key2)) - state2, _ := pool2.blockChain.State() - state2.AddBalance(account2, big.NewInt(1000000)) + pool2.currentState.AddBalance(account2, big.NewInt(1000000)) txns := []*types.Transaction{} for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { @@ -934,7 +862,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = config.AccountSlots * 10 @@ -943,12 +871,10 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) @@ -981,7 +907,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.AccountSlots = 2 @@ -992,11 +918,9 @@ func TestTransactionCapClearsFromAll(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - state.AddBalance(addr, big.NewInt(1000000)) + pool.currentState.AddBalance(addr, big.NewInt(1000000)) txs := types.Transactions{} for j := 0; j < int(config.GlobalSlots)*2; j++ { @@ -1016,7 +940,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 0 @@ -1025,12 +949,10 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) @@ -1065,18 +987,16 @@ func TestTransactionPoolRepricing(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 3) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} @@ -1147,18 +1067,16 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 3) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000*1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000*1000000)) } // Create transaction (both pending and queued) with a linearly growing gasprice for i := uint64(0); i < 500; i++ { @@ -1189,11 +1107,11 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { } } validate() - + // Reprice the pool and check that nothing is dropped pool.SetGasPrice(big.NewInt(2)) validate() - + pool.SetGasPrice(big.NewInt(2)) pool.SetGasPrice(big.NewInt(4)) pool.SetGasPrice(big.NewInt(8)) @@ -1210,7 +1128,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 2 @@ -1220,12 +1138,10 @@ func TestTransactionPoolUnderpricing(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 3) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} @@ -1298,16 +1214,14 @@ func TestTransactionReplacement(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Create a test account to add transactions with key, _ := crypto.GenerateKey() - - state, _ := pool.blockChain.State() - state.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000)) // Add pending transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too) price := int64(100) @@ -1378,7 +1292,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { // Create the original pool to inject transaction into the journal db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.NoLocals = nolocals @@ -1391,9 +1305,8 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { local, _ := crypto.GenerateKey() remote, _ := crypto.GenerateKey() - statedb, _ = pool.blockChain.State() - statedb.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) - statedb.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) // Add three local and a remote transactions and ensure they are queued up if err := pool.AddLocal(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), local)); err != nil { @@ -1421,7 +1334,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { // Terminate the old pool, bump the local nonce, create a new pool and ensure relevant transaction survive pool.Stop() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool = NewTxPool(config, params.TestChainConfig, blockchain) pending, queued = pool.Stats() @@ -1442,11 +1355,11 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { } // Bump the nonce temporarily and ensure the newly invalidated transaction is removed statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 2) - pool.lockedReset() + pool.lockedReset(nil, nil) time.Sleep(2 * config.Rejournal) pool.Stop() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool = NewTxPool(config, params.TestChainConfig, blockchain) pending, queued = pool.Stats() @@ -1480,8 +1393,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) for i := 0; i < size; i++ { tx := transaction(uint64(i), big.NewInt(100000), key) @@ -1490,7 +1402,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) { // Benchmark the speed of pool validation b.ResetTimer() for i := 0; i < b.N; i++ { - pool.demoteUnexecutables(state) + pool.demoteUnexecutables() } } @@ -1506,8 +1418,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) for i := 0; i < size; i++ { tx := transaction(uint64(1+i), big.NewInt(100000), key) @@ -1516,7 +1427,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { // Benchmark the speed of pool validation b.ResetTimer() for i := 0; i < b.N; i++ { - pool.promoteExecutables(state, nil) + pool.promoteExecutables(nil) } } @@ -1527,8 +1438,7 @@ func BenchmarkPoolInsert(b *testing.B) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) txs := make(types.Transactions, b.N) for i := 0; i < b.N; i++ { @@ -1552,8 +1462,7 @@ func benchmarkPoolBatchInsert(b *testing.B, size int) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) batches := make([]types.Transactions, b.N) for i := 0; i < b.N; i++ { diff --git a/eth/api_backend.go b/eth/api_backend.go index abf52326b6..19ef79f234 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -115,10 +115,6 @@ func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *sta return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil } -func (b *EthApiBackend) SubscribeRemovedTxEvent(ch chan<- core.RemovedTransactionEvent) event.Subscription { - return b.eth.BlockChain().SubscribeRemovedTxEvent(ch) -} - func (b *EthApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } @@ -143,10 +139,6 @@ func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) return b.eth.txPool.AddLocal(signedTx) } -func (b *EthApiBackend) RemoveTx(txHash common.Hash) { - b.eth.txPool.Remove(txHash) -} - func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { pending, err := b.eth.txPool.Pending() if err != nil { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 30710aaabd..0775749e70 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1265,7 +1265,6 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr if err != nil { return common.Hash{}, err } - s.b.RemoveTx(p.Hash()) if err = s.b.SendTx(ctx, signedTx); err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index be17ffeae4..368fa48726 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -59,7 +59,6 @@ type Backend interface { // TxPool API SendTx(ctx context.Context, signedTx *types.Transaction) error - RemoveTx(txHash common.Hash) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) diff --git a/miner/worker.go b/miner/worker.go index 24e03be60a..5bac5d6e85 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -71,7 +71,6 @@ type Work struct { family *set.Set // family set (used for checking uncle invalidity) uncles *set.Set // uncle set tcount int // tx count in cycle - failedTxs types.Transactions Block *types.Block // the new block @@ -477,8 +476,6 @@ func (self *worker) commitNewWork() { txs := types.NewTransactionsByPriceAndNonce(pending) work.commitTransactions(self.mux, txs, self.chain, self.coinbase) - self.eth.TxPool().RemoveBatch(work.failedTxs) - // compute uncles for the new block. var ( uncles []*types.Header @@ -563,6 +560,16 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB log.Trace("Gas limit exceeded for current block", "sender", from) txs.Pop() + case core.ErrNonceTooLow: + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + txs.Shift() + + case core.ErrNonceTooHigh: + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) + txs.Pop() + case nil: // Everything ok, collect the logs and shift in the next transaction from the same account coalescedLogs = append(coalescedLogs, logs...) @@ -570,10 +577,10 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB txs.Shift() default: - // Pop the current failed transaction without shifting in the next from the account - log.Trace("Transaction failed, will be removed", "hash", tx.Hash(), "err", err) - env.failedTxs = append(env.failedTxs, tx) - txs.Pop() + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + txs.Shift() } } From 2bacf36d8095ac7936f69552e2727ac6f276479f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Tue, 5 Sep 2017 12:38:36 +0200 Subject: [PATCH 10/45] bmt: Binary Merkle Tree Hash (#14334) bmt is a new package that provides hashers for binary merkle tree hashes on size-limited chunks. the main motivation is that using BMT hash as the chunk hash of the swarm hash offers logsize inclusion proofs for arbitrary files on a 32-byte resolution completely viable to use in challenges on the blockchain. --- bmt/bmt.go | 562 +++++++++++++++++++++++++++++++++++++++ bmt/bmt_r.go | 85 ++++++ bmt/bmt_test.go | 481 +++++++++++++++++++++++++++++++++ swarm/storage/chunker.go | 3 +- swarm/storage/types.go | 1 + 5 files changed, 1131 insertions(+), 1 deletion(-) create mode 100644 bmt/bmt.go create mode 100644 bmt/bmt_r.go create mode 100644 bmt/bmt_test.go diff --git a/bmt/bmt.go b/bmt/bmt.go new file mode 100644 index 0000000000..d62365bb1f --- /dev/null +++ b/bmt/bmt.go @@ -0,0 +1,562 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package bmt provides a binary merkle tree implementation +package bmt + +import ( + "fmt" + "hash" + "io" + "strings" + "sync" + "sync/atomic" +) + +/* +Binary Merkle Tree Hash is a hash function over arbitrary datachunks of limited size +It is defined as the root hash of the binary merkle tree built over fixed size segments +of the underlying chunk using any base hash function (e.g keccak 256 SHA3) + +It is used as the chunk hash function in swarm which in turn is the basis for the +128 branching swarm hash http://swarm-guide.readthedocs.io/en/latest/architecture.html#swarm-hash + +The BMT is optimal for providing compact inclusion proofs, i.e. prove that a +segment is a substring of a chunk starting at a particular offset +The size of the underlying segments is fixed at 32 bytes (called the resolution +of the BMT hash), the EVM word size to optimize for on-chain BMT verification +as well as the hash size optimal for inclusion proofs in the merkle tree of the swarm hash. + +Two implementations are provided: + +* RefHasher is optimized for code simplicity and meant as a reference implementation +* Hasher is optimized for speed taking advantage of concurrency with minimalistic + control structure to coordinate the concurrent routines + It implements the ChunkHash interface as well as the go standard hash.Hash interface + +*/ + +const ( + // DefaultSegmentCount is the maximum number of segments of the underlying chunk + DefaultSegmentCount = 128 // Should be equal to storage.DefaultBranches + // DefaultPoolSize is the maximum number of bmt trees used by the hashers, i.e, + // the maximum number of concurrent BMT hashing operations performed by the same hasher + DefaultPoolSize = 8 +) + +// BaseHasher is a hash.Hash constructor function used for the base hash of the BMT. +type BaseHasher func() hash.Hash + +// Hasher a reusable hasher for fixed maximum size chunks representing a BMT +// implements the hash.Hash interface +// reuse pool of Tree-s for amortised memory allocation and resource control +// supports order-agnostic concurrent segment writes +// as well as sequential read and write +// can not be called concurrently on more than one chunk +// can be further appended after Sum +// Reset gives back the Tree to the pool and guaranteed to leave +// the tree and itself in a state reusable for hashing a new chunk +type Hasher struct { + pool *TreePool // BMT resource pool + bmt *Tree // prebuilt BMT resource for flowcontrol and proofs + blocksize int // segment size (size of hash) also for hash.Hash + count int // segment count + size int // for hash.Hash same as hashsize + cur int // cursor position for righmost currently open chunk + segment []byte // the rightmost open segment (not complete) + depth int // index of last level + result chan []byte // result channel + hash []byte // to record the result + max int32 // max segments for SegmentWriter interface + blockLength []byte // The block length that needes to be added in Sum +} + +// New creates a reusable Hasher +// implements the hash.Hash interface +// pulls a new Tree from a resource pool for hashing each chunk +func New(p *TreePool) *Hasher { + return &Hasher{ + pool: p, + depth: depth(p.SegmentCount), + size: p.SegmentSize, + blocksize: p.SegmentSize, + count: p.SegmentCount, + result: make(chan []byte), + } +} + +// Node is a reuseable segment hasher representing a node in a BMT +// it allows for continued writes after a Sum +// and is left in completely reusable state after Reset +type Node struct { + level, index int // position of node for information/logging only + initial bool // first and last node + root bool // whether the node is root to a smaller BMT + isLeft bool // whether it is left side of the parent double segment + unbalanced bool // indicates if a node has only the left segment + parent *Node // BMT connections + state int32 // atomic increment impl concurrent boolean toggle + left, right []byte +} + +// NewNode constructor for segment hasher nodes in the BMT +func NewNode(level, index int, parent *Node) *Node { + return &Node{ + parent: parent, + level: level, + index: index, + initial: index == 0, + isLeft: index%2 == 0, + } +} + +// TreePool provides a pool of Trees used as resources by Hasher +// a Tree popped from the pool is guaranteed to have clean state +// for hashing a new chunk +// Hasher Reset releases the Tree to the pool +type TreePool struct { + lock sync.Mutex + c chan *Tree + hasher BaseHasher + SegmentSize int + SegmentCount int + Capacity int + count int +} + +// NewTreePool creates a Tree pool with hasher, segment size, segment count and capacity +// on GetTree it reuses free Trees or creates a new one if size is not reached +func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool { + return &TreePool{ + c: make(chan *Tree, capacity), + hasher: hasher, + SegmentSize: hasher().Size(), + SegmentCount: segmentCount, + Capacity: capacity, + } +} + +// Drain drains the pool uptil it has no more than n resources +func (self *TreePool) Drain(n int) { + self.lock.Lock() + defer self.lock.Unlock() + for len(self.c) > n { + <-self.c + self.count-- + } +} + +// Reserve is blocking until it returns an available Tree +// it reuses free Trees or creates a new one if size is not reached +func (self *TreePool) Reserve() *Tree { + self.lock.Lock() + defer self.lock.Unlock() + var t *Tree + if self.count == self.Capacity { + return <-self.c + } + select { + case t = <-self.c: + default: + t = NewTree(self.hasher, self.SegmentSize, self.SegmentCount) + self.count++ + } + return t +} + +// Release gives back a Tree to the pool. +// This Tree is guaranteed to be in reusable state +// does not need locking +func (self *TreePool) Release(t *Tree) { + self.c <- t // can never fail but... +} + +// Tree is a reusable control structure representing a BMT +// organised in a binary tree +// Hasher uses a TreePool to pick one for each chunk hash +// the Tree is 'locked' while not in the pool +type Tree struct { + leaves []*Node +} + +// Draw draws the BMT (badly) +func (self *Tree) Draw(hash []byte, d int) string { + var left, right []string + var anc []*Node + for i, n := range self.leaves { + left = append(left, fmt.Sprintf("%v", hashstr(n.left))) + if i%2 == 0 { + anc = append(anc, n.parent) + } + right = append(right, fmt.Sprintf("%v", hashstr(n.right))) + } + anc = self.leaves + var hashes [][]string + for l := 0; len(anc) > 0; l++ { + var nodes []*Node + hash := []string{""} + for i, n := range anc { + hash = append(hash, fmt.Sprintf("%v|%v", hashstr(n.left), hashstr(n.right))) + if i%2 == 0 && n.parent != nil { + nodes = append(nodes, n.parent) + } + } + hash = append(hash, "") + hashes = append(hashes, hash) + anc = nodes + } + hashes = append(hashes, []string{"", fmt.Sprintf("%v", hashstr(hash)), ""}) + total := 60 + del := " " + var rows []string + for i := len(hashes) - 1; i >= 0; i-- { + var textlen int + hash := hashes[i] + for _, s := range hash { + textlen += len(s) + } + if total < textlen { + total = textlen + len(hash) + } + delsize := (total - textlen) / (len(hash) - 1) + if delsize > len(del) { + delsize = len(del) + } + row := fmt.Sprintf("%v: %v", len(hashes)-i-1, strings.Join(hash, del[:delsize])) + rows = append(rows, row) + + } + rows = append(rows, strings.Join(left, " ")) + rows = append(rows, strings.Join(right, " ")) + return strings.Join(rows, "\n") + "\n" +} + +// NewTree initialises the Tree by building up the nodes of a BMT +// segment size is stipulated to be the size of the hash +// segmentCount needs to be positive integer and does not need to be +// a power of two and can even be an odd number +// segmentSize * segmentCount determines the maximum chunk size +// hashed using the tree +func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree { + n := NewNode(0, 0, nil) + n.root = true + prevlevel := []*Node{n} + // iterate over levels and creates 2^level nodes + level := 1 + count := 2 + for d := 1; d <= depth(segmentCount); d++ { + nodes := make([]*Node, count) + for i := 0; i < len(nodes); i++ { + var parent *Node + parent = prevlevel[i/2] + t := NewNode(level, i, parent) + nodes[i] = t + } + prevlevel = nodes + level++ + count *= 2 + } + // the datanode level is the nodes on the last level where + return &Tree{ + leaves: prevlevel, + } +} + +// methods needed by hash.Hash + +// Size returns the size +func (self *Hasher) Size() int { + return self.size +} + +// BlockSize returns the block size +func (self *Hasher) BlockSize() int { + return self.blocksize +} + +// Sum returns the hash of the buffer +// hash.Hash interface Sum method appends the byte slice to the underlying +// data before it calculates and returns the hash of the chunk +func (self *Hasher) Sum(b []byte) (r []byte) { + t := self.bmt + i := self.cur + n := t.leaves[i] + j := i + // must run strictly before all nodes calculate + // datanodes are guaranteed to have a parent + if len(self.segment) > self.size && i > 0 && n.parent != nil { + n = n.parent + } else { + i *= 2 + } + d := self.finalise(n, i) + self.writeSegment(j, self.segment, d) + c := <-self.result + self.releaseTree() + + // sha3(length + BMT(pure_chunk)) + if self.blockLength == nil { + return c + } + res := self.pool.hasher() + res.Reset() + res.Write(self.blockLength) + res.Write(c) + return res.Sum(nil) +} + +// Hasher implements the SwarmHash interface + +// Hash waits for the hasher result and returns it +// caller must call this on a BMT Hasher being written to +func (self *Hasher) Hash() []byte { + return <-self.result +} + +// Hasher implements the io.Writer interface + +// Write fills the buffer to hash +// with every full segment complete launches a hasher go routine +// that shoots up the BMT +func (self *Hasher) Write(b []byte) (int, error) { + l := len(b) + if l <= 0 { + return 0, nil + } + s := self.segment + i := self.cur + count := (self.count + 1) / 2 + need := self.count*self.size - self.cur*2*self.size + size := self.size + if need > size { + size *= 2 + } + if l < need { + need = l + } + // calculate missing bit to complete current open segment + rest := size - len(s) + if need < rest { + rest = need + } + s = append(s, b[:rest]...) + need -= rest + // read full segments and the last possibly partial segment + for need > 0 && i < count-1 { + // push all finished chunks we read + self.writeSegment(i, s, self.depth) + need -= size + if need < 0 { + size += need + } + s = b[rest : rest+size] + rest += size + i++ + } + self.segment = s + self.cur = i + // otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full + return l, nil +} + +// Hasher implements the io.ReaderFrom interface + +// ReadFrom reads from io.Reader and appends to the data to hash using Write +// it reads so that chunk to hash is maximum length or reader reaches EOF +// caller must Reset the hasher prior to call +func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { + bufsize := self.size*self.count - self.size*self.cur - len(self.segment) + buf := make([]byte, bufsize) + var read int + for { + var n int + n, err = r.Read(buf) + read += n + if err == io.EOF || read == len(buf) { + hash := self.Sum(buf[:n]) + if read == len(buf) { + err = NewEOC(hash) + } + break + } + if err != nil { + break + } + n, err = self.Write(buf[:n]) + if err != nil { + break + } + } + return int64(read), err +} + +// Reset needs to be called before writing to the hasher +func (self *Hasher) Reset() { + self.getTree() + self.blockLength = nil +} + +// Hasher implements the SwarmHash interface + +// ResetWithLength needs to be called before writing to the hasher +// the argument is supposed to be the byte slice binary representation of +// the legth of the data subsumed under the hash +func (self *Hasher) ResetWithLength(l []byte) { + self.Reset() + self.blockLength = l + +} + +// Release gives back the Tree to the pool whereby it unlocks +// it resets tree, segment and index +func (self *Hasher) releaseTree() { + if self.bmt != nil { + n := self.bmt.leaves[self.cur] + for ; n != nil; n = n.parent { + n.unbalanced = false + if n.parent != nil { + n.root = false + } + } + self.pool.Release(self.bmt) + self.bmt = nil + + } + self.cur = 0 + self.segment = nil +} + +func (self *Hasher) writeSegment(i int, s []byte, d int) { + h := self.pool.hasher() + n := self.bmt.leaves[i] + + if len(s) > self.size && n.parent != nil { + go func() { + h.Reset() + h.Write(s) + s = h.Sum(nil) + + if n.root { + self.result <- s + return + } + self.run(n.parent, h, d, n.index, s) + }() + return + } + go self.run(n, h, d, i*2, s) +} + +func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { + isLeft := i%2 == 0 + for { + if isLeft { + n.left = s + } else { + n.right = s + } + if !n.unbalanced && n.toggle() { + return + } + if !n.unbalanced || !isLeft || i == 0 && d == 0 { + h.Reset() + h.Write(n.left) + h.Write(n.right) + s = h.Sum(nil) + + } else { + s = append(n.left, n.right...) + } + + self.hash = s + if n.root { + self.result <- s + return + } + + isLeft = n.isLeft + n = n.parent + i++ + } +} + +// getTree obtains a BMT resource by reserving one from the pool +func (self *Hasher) getTree() *Tree { + if self.bmt != nil { + return self.bmt + } + t := self.pool.Reserve() + self.bmt = t + return t +} + +// atomic bool toggle implementing a concurrent reusable 2-state object +// atomic addint with %2 implements atomic bool toggle +// it returns true if the toggler just put it in the active/waiting state +func (self *Node) toggle() bool { + return atomic.AddInt32(&self.state, 1)%2 == 1 +} + +func hashstr(b []byte) string { + end := len(b) + if end > 4 { + end = 4 + } + return fmt.Sprintf("%x", b[:end]) +} + +func depth(n int) (d int) { + for l := (n - 1) / 2; l > 0; l /= 2 { + d++ + } + return d +} + +// finalise is following the zigzags on the tree belonging +// to the final datasegment +func (self *Hasher) finalise(n *Node, i int) (d int) { + isLeft := i%2 == 0 + for { + // when the final segment's path is going via left segments + // the incoming data is pushed to the parent upon pulling the left + // we do not need toogle the state since this condition is + // detectable + n.unbalanced = isLeft + n.right = nil + if n.initial { + n.root = true + return d + } + isLeft = n.isLeft + n = n.parent + d++ + } +} + +// EOC (end of chunk) implements the error interface +type EOC struct { + Hash []byte // read the hash of the chunk off the error +} + +// Error returns the error string +func (self *EOC) Error() string { + return fmt.Sprintf("hasher limit reached, chunk hash: %x", self.Hash) +} + +// NewEOC creates new end of chunk error with the hash +func NewEOC(hash []byte) *EOC { + return &EOC{hash} +} diff --git a/bmt/bmt_r.go b/bmt/bmt_r.go new file mode 100644 index 0000000000..649093ee3a --- /dev/null +++ b/bmt/bmt_r.go @@ -0,0 +1,85 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// simple nonconcurrent reference implementation for hashsize segment based +// Binary Merkle tree hash on arbitrary but fixed maximum chunksize +// +// This implementation does not take advantage of any paralellisms and uses +// far more memory than necessary, but it is easy to see that it is correct. +// It can be used for generating test cases for optimized implementations. +// see testBMTHasherCorrectness function in bmt_test.go +package bmt + +import ( + "hash" +) + +// RefHasher is the non-optimized easy to read reference implementation of BMT +type RefHasher struct { + span int + section int + cap int + h hash.Hash +} + +// NewRefHasher returns a new RefHasher +func NewRefHasher(hasher BaseHasher, count int) *RefHasher { + h := hasher() + hashsize := h.Size() + maxsize := hashsize * count + c := 2 + for ; c < count; c *= 2 { + } + if c > 2 { + c /= 2 + } + return &RefHasher{ + section: 2 * hashsize, + span: c * hashsize, + cap: maxsize, + h: h, + } +} + +// Hash returns the BMT hash of the byte slice +// implements the SwarmHash interface +func (rh *RefHasher) Hash(d []byte) []byte { + if len(d) > rh.cap { + d = d[:rh.cap] + } + + return rh.hash(d, rh.span) +} + +func (rh *RefHasher) hash(d []byte, s int) []byte { + l := len(d) + left := d + var right []byte + if l > rh.section { + for ; s >= l; s /= 2 { + } + left = rh.hash(d[:s], s) + right = d[s:] + if l-s > rh.section/2 { + right = rh.hash(right, s) + } + } + defer rh.h.Reset() + rh.h.Write(left) + rh.h.Write(right) + h := rh.h.Sum(nil) + return h +} diff --git a/bmt/bmt_test.go b/bmt/bmt_test.go new file mode 100644 index 0000000000..57df83060a --- /dev/null +++ b/bmt/bmt_test.go @@ -0,0 +1,481 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bmt + +import ( + "bytes" + crand "crypto/rand" + "fmt" + "hash" + "io" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto/sha3" +) + +const ( + maxproccnt = 8 +) + +// TestRefHasher tests that the RefHasher computes the expected BMT hash for +// all data lengths between 0 and 256 bytes +func TestRefHasher(t *testing.T) { + hashFunc := sha3.NewKeccak256 + + sha3 := func(data ...[]byte) []byte { + h := hashFunc() + for _, v := range data { + h.Write(v) + } + return h.Sum(nil) + } + + // the test struct is used to specify the expected BMT hash for data + // lengths between "from" and "to" + type test struct { + from int64 + to int64 + expected func([]byte) []byte + } + + var tests []*test + + // all lengths in [0,64] should be: + // + // sha3(data) + // + tests = append(tests, &test{ + from: 0, + to: 64, + expected: func(data []byte) []byte { + return sha3(data) + }, + }) + + // all lengths in [65,96] should be: + // + // sha3( + // sha3(data[:64]) + // data[64:] + // ) + // + tests = append(tests, &test{ + from: 65, + to: 96, + expected: func(data []byte) []byte { + return sha3(sha3(data[:64]), data[64:]) + }, + }) + + // all lengths in [97,128] should be: + // + // sha3( + // sha3(data[:64]) + // sha3(data[64:]) + // ) + // + tests = append(tests, &test{ + from: 97, + to: 128, + expected: func(data []byte) []byte { + return sha3(sha3(data[:64]), sha3(data[64:])) + }, + }) + + // all lengths in [129,160] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // data[128:] + // ) + // + tests = append(tests, &test{ + from: 129, + to: 160, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), data[128:]) + }, + }) + + // all lengths in [161,192] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // sha3(data[128:]) + // ) + // + tests = append(tests, &test{ + from: 161, + to: 192, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(data[128:])) + }, + }) + + // all lengths in [193,224] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // sha3( + // sha3(data[128:192]) + // data[192:] + // ) + // ) + // + tests = append(tests, &test{ + from: 193, + to: 224, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), data[192:])) + }, + }) + + // all lengths in [225,256] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // sha3( + // sha3(data[128:192]) + // sha3(data[192:]) + // ) + // ) + // + tests = append(tests, &test{ + from: 225, + to: 256, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), sha3(data[192:]))) + }, + }) + + // run the tests + for _, x := range tests { + for length := x.from; length <= x.to; length++ { + t.Run(fmt.Sprintf("%d_bytes", length), func(t *testing.T) { + data := make([]byte, length) + if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF { + t.Fatal(err) + } + expected := x.expected(data) + actual := NewRefHasher(hashFunc, 128).Hash(data) + if !bytes.Equal(actual, expected) { + t.Fatalf("expected %x, got %x", expected, actual) + } + }) + } + } +} + +func testDataReader(l int) (r io.Reader) { + return io.LimitReader(crand.Reader, int64(l)) +} + +func TestHasherCorrectness(t *testing.T) { + err := testHasher(testBaseHasher) + if err != nil { + t.Fatal(err) + } +} + +func testHasher(f func(BaseHasher, []byte, int, int) error) error { + tdata := testDataReader(4128) + data := make([]byte, 4128) + tdata.Read(data) + hasher := sha3.NewKeccak256 + size := hasher().Size() + counts := []int{1, 2, 3, 4, 5, 8, 16, 32, 64, 128} + + var err error + for _, count := range counts { + max := count * size + incr := 1 + for n := 0; n <= max+incr; n += incr { + err = f(hasher, data, n, count) + if err != nil { + return err + } + } + } + return nil +} + +func TestHasherReuseWithoutRelease(t *testing.T) { + testHasherReuse(1, t) +} + +func TestHasherReuseWithRelease(t *testing.T) { + testHasherReuse(maxproccnt, t) +} + +func testHasherReuse(i int, t *testing.T) { + hasher := sha3.NewKeccak256 + pool := NewTreePool(hasher, 128, i) + defer pool.Drain(0) + bmt := New(pool) + + for i := 0; i < 500; i++ { + n := rand.Intn(4096) + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + err := testHasherCorrectness(bmt, hasher, data, n, 128) + if err != nil { + t.Fatal(err) + } + } +} + +func TestHasherConcurrency(t *testing.T) { + hasher := sha3.NewKeccak256 + pool := NewTreePool(hasher, 128, maxproccnt) + defer pool.Drain(0) + wg := sync.WaitGroup{} + cycles := 100 + wg.Add(maxproccnt * cycles) + errc := make(chan error) + + for p := 0; p < maxproccnt; p++ { + for i := 0; i < cycles; i++ { + go func() { + bmt := New(pool) + n := rand.Intn(4096) + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + err := testHasherCorrectness(bmt, hasher, data, n, 128) + wg.Done() + if err != nil { + errc <- err + } + }() + } + } + go func() { + wg.Wait() + close(errc) + }() + var err error + select { + case <-time.NewTimer(5 * time.Second).C: + err = fmt.Errorf("timed out") + case err = <-errc: + } + if err != nil { + t.Fatal(err) + } +} + +func testBaseHasher(hasher BaseHasher, d []byte, n, count int) error { + pool := NewTreePool(hasher, count, 1) + defer pool.Drain(0) + bmt := New(pool) + return testHasherCorrectness(bmt, hasher, d, n, count) +} + +func testHasherCorrectness(bmt hash.Hash, hasher BaseHasher, d []byte, n, count int) (err error) { + data := d[:n] + rbmt := NewRefHasher(hasher, count) + exp := rbmt.Hash(data) + timeout := time.NewTimer(time.Second) + c := make(chan error) + + go func() { + bmt.Reset() + bmt.Write(data) + got := bmt.Sum(nil) + if !bytes.Equal(got, exp) { + c <- fmt.Errorf("wrong hash: expected %x, got %x", exp, got) + } + close(c) + }() + select { + case <-timeout.C: + err = fmt.Errorf("BMT hash calculation timed out") + case err = <-c: + } + return err +} + +func BenchmarkSHA3_4k(t *testing.B) { benchmarkSHA3(4096, t) } +func BenchmarkSHA3_2k(t *testing.B) { benchmarkSHA3(4096/2, t) } +func BenchmarkSHA3_1k(t *testing.B) { benchmarkSHA3(4096/4, t) } +func BenchmarkSHA3_512b(t *testing.B) { benchmarkSHA3(4096/8, t) } +func BenchmarkSHA3_256b(t *testing.B) { benchmarkSHA3(4096/16, t) } +func BenchmarkSHA3_128b(t *testing.B) { benchmarkSHA3(4096/32, t) } + +func BenchmarkBMTBaseline_4k(t *testing.B) { benchmarkBMTBaseline(4096, t) } +func BenchmarkBMTBaseline_2k(t *testing.B) { benchmarkBMTBaseline(4096/2, t) } +func BenchmarkBMTBaseline_1k(t *testing.B) { benchmarkBMTBaseline(4096/4, t) } +func BenchmarkBMTBaseline_512b(t *testing.B) { benchmarkBMTBaseline(4096/8, t) } +func BenchmarkBMTBaseline_256b(t *testing.B) { benchmarkBMTBaseline(4096/16, t) } +func BenchmarkBMTBaseline_128b(t *testing.B) { benchmarkBMTBaseline(4096/32, t) } + +func BenchmarkRefHasher_4k(t *testing.B) { benchmarkRefHasher(4096, t) } +func BenchmarkRefHasher_2k(t *testing.B) { benchmarkRefHasher(4096/2, t) } +func BenchmarkRefHasher_1k(t *testing.B) { benchmarkRefHasher(4096/4, t) } +func BenchmarkRefHasher_512b(t *testing.B) { benchmarkRefHasher(4096/8, t) } +func BenchmarkRefHasher_256b(t *testing.B) { benchmarkRefHasher(4096/16, t) } +func BenchmarkRefHasher_128b(t *testing.B) { benchmarkRefHasher(4096/32, t) } + +func BenchmarkHasher_4k(t *testing.B) { benchmarkHasher(4096, t) } +func BenchmarkHasher_2k(t *testing.B) { benchmarkHasher(4096/2, t) } +func BenchmarkHasher_1k(t *testing.B) { benchmarkHasher(4096/4, t) } +func BenchmarkHasher_512b(t *testing.B) { benchmarkHasher(4096/8, t) } +func BenchmarkHasher_256b(t *testing.B) { benchmarkHasher(4096/16, t) } +func BenchmarkHasher_128b(t *testing.B) { benchmarkHasher(4096/32, t) } + +func BenchmarkHasherNoReuse_4k(t *testing.B) { benchmarkHasherReuse(1, 4096, t) } +func BenchmarkHasherNoReuse_2k(t *testing.B) { benchmarkHasherReuse(1, 4096/2, t) } +func BenchmarkHasherNoReuse_1k(t *testing.B) { benchmarkHasherReuse(1, 4096/4, t) } +func BenchmarkHasherNoReuse_512b(t *testing.B) { benchmarkHasherReuse(1, 4096/8, t) } +func BenchmarkHasherNoReuse_256b(t *testing.B) { benchmarkHasherReuse(1, 4096/16, t) } +func BenchmarkHasherNoReuse_128b(t *testing.B) { benchmarkHasherReuse(1, 4096/32, t) } + +func BenchmarkHasherReuse_4k(t *testing.B) { benchmarkHasherReuse(16, 4096, t) } +func BenchmarkHasherReuse_2k(t *testing.B) { benchmarkHasherReuse(16, 4096/2, t) } +func BenchmarkHasherReuse_1k(t *testing.B) { benchmarkHasherReuse(16, 4096/4, t) } +func BenchmarkHasherReuse_512b(t *testing.B) { benchmarkHasherReuse(16, 4096/8, t) } +func BenchmarkHasherReuse_256b(t *testing.B) { benchmarkHasherReuse(16, 4096/16, t) } +func BenchmarkHasherReuse_128b(t *testing.B) { benchmarkHasherReuse(16, 4096/32, t) } + +// benchmarks the minimum hashing time for a balanced (for simplicity) BMT +// by doing count/segmentsize parallel hashings of 2*segmentsize bytes +// doing it on n maxproccnt each reusing the base hasher +// the premise is that this is the minimum computation needed for a BMT +// therefore this serves as a theoretical optimum for concurrent implementations +func benchmarkBMTBaseline(n int, t *testing.B) { + tdata := testDataReader(64) + data := make([]byte, 64) + tdata.Read(data) + hasher := sha3.NewKeccak256 + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + count := int32((n-1)/hasher().Size() + 1) + wg := sync.WaitGroup{} + wg.Add(maxproccnt) + var i int32 + for j := 0; j < maxproccnt; j++ { + go func() { + defer wg.Done() + h := hasher() + for atomic.AddInt32(&i, 1) < count { + h.Reset() + h.Write(data) + h.Sum(nil) + } + }() + } + wg.Wait() + } +} + +func benchmarkHasher(n int, t *testing.B) { + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + size := 1 + hasher := sha3.NewKeccak256 + segmentCount := 128 + pool := NewTreePool(hasher, segmentCount, size) + bmt := New(pool) + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + bmt.Reset() + bmt.Write(data) + bmt.Sum(nil) + } +} + +func benchmarkHasherReuse(poolsize, n int, t *testing.B) { + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + hasher := sha3.NewKeccak256 + segmentCount := 128 + pool := NewTreePool(hasher, segmentCount, poolsize) + cycles := 200 + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + wg := sync.WaitGroup{} + wg.Add(cycles) + for j := 0; j < cycles; j++ { + bmt := New(pool) + go func() { + defer wg.Done() + bmt.Reset() + bmt.Write(data) + bmt.Sum(nil) + }() + } + wg.Wait() + } +} + +func benchmarkSHA3(n int, t *testing.B) { + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hasher := sha3.NewKeccak256 + h := hasher() + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + h.Reset() + h.Write(data) + h.Sum(nil) + } +} + +func benchmarkRefHasher(n int, t *testing.B) { + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hasher := sha3.NewKeccak256 + rbmt := NewRefHasher(hasher, 128) + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + rbmt.Hash(data) + } +} diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 563793e98f..ca85e43337 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -51,7 +51,8 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} */ const ( - defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash + defaultHash = "SHA3" + // defaultHash = "BMTSHA3" // http://golang.org/pkg/hash/#Hash // defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash defaultBranches int64 = 128 // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes diff --git a/swarm/storage/types.go b/swarm/storage/types.go index cc5ded931d..a9de23c93a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -24,6 +24,7 @@ import ( "io" "sync" + // "github.com/ethereum/go-ethereum/bmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" ) From cd6c861dc567478adb64b7efef327a4aceda97f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 16:04:32 +0300 Subject: [PATCH 11/45] vendor: pull in latest changes for goleveldb (#15090) --- .../github.com/syndtr/goleveldb/leveldb/db.go | 2 +- .../syndtr/goleveldb/leveldb/db_compaction.go | 4 +- .../syndtr/goleveldb/leveldb/db_state.go | 7 ++- .../syndtr/goleveldb/leveldb/db_write.go | 9 ++++ .../syndtr/goleveldb/leveldb/doc.go | 2 + .../goleveldb/leveldb/storage/file_storage.go | 24 +++++++-- .../syndtr/goleveldb/leveldb/table/reader.go | 1 + vendor/vendor.json | 54 +++++++++---------- 8 files changed, 68 insertions(+), 35 deletions(-) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index a02cb2c500..b0cdcb3d0a 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -844,7 +844,7 @@ func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { // Has returns true if the DB does contains the given key. // -// It is safe to modify the contents of the argument after Get returns. +// It is safe to modify the contents of the argument after Has returns. func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { err = db.ok() if err != nil { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go index 2d0ad0753c..b6563e87e4 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -289,7 +289,7 @@ func (db *DB) memCompaction() { close(resumeC) resumeC = nil case <-db.closeC: - return + db.compactionExitTransact() } var ( @@ -338,7 +338,7 @@ func (db *DB) memCompaction() { case <-resumeC: close(resumeC) case <-db.closeC: - return + db.compactionExitTransact() } } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go index 85b02d24bf..65e1c54bb4 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go @@ -7,6 +7,7 @@ package leveldb import ( + "errors" "sync/atomic" "time" @@ -15,6 +16,10 @@ import ( "github.com/syndtr/goleveldb/leveldb/storage" ) +var ( + errHasFrozenMem = errors.New("has frozen mem") +) + type memDB struct { db *DB *memdb.DB @@ -126,7 +131,7 @@ func (db *DB) newMem(n int) (mem *memDB, err error) { defer db.memMu.Unlock() if db.frozenMem != nil { - panic("still has frozen mem") + return nil, errHasFrozenMem } if db.journal == nil { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go index cc428b695c..5b6cb487dc 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go @@ -32,15 +32,24 @@ func (db *DB) writeJournal(batches []*Batch, seq uint64, sync bool) error { } func (db *DB) rotateMem(n int, wait bool) (mem *memDB, err error) { + retryLimit := 3 +retry: // Wait for pending memdb compaction. err = db.compTriggerWait(db.mcompCmdC) if err != nil { return } + retryLimit-- // Create new memdb and journal. mem, err = db.newMem(n) if err != nil { + if err == errHasFrozenMem { + if retryLimit <= 0 { + panic("BUG: still has frozen memdb") + } + goto retry + } return } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/doc.go b/vendor/github.com/syndtr/goleveldb/leveldb/doc.go index 53f13bb24c..be768e5739 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/doc.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/doc.go @@ -8,6 +8,8 @@ // // Create or open a database: // +// // The returned DB instance is safe for concurrent use. Which mean that all +// // DB's methods may be called concurrently from multiple goroutine. // db, err := leveldb.OpenFile("path/to/db", nil) // ... // defer db.Close() diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go index e53434cab7..1189decac6 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go @@ -234,14 +234,30 @@ func (fs *fileStorage) SetMeta(fd FileDesc) (err error) { return } _, err = fmt.Fprintln(w, fsGenName(fd)) - // Close the file first. - if cerr := w.Close(); cerr != nil { - fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, cerr)) + if err != nil { + fs.log(fmt.Sprintf("write CURRENT.%d: %v", fd.Num, err)) + return + } + if err = w.Sync(); err != nil { + fs.log(fmt.Sprintf("flush CURRENT.%d: %v", fd.Num, err)) + return + } + if err = w.Close(); err != nil { + fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, err)) + return } if err != nil { return } - return rename(path, filepath.Join(fs.path, "CURRENT")) + if err = rename(path, filepath.Join(fs.path, "CURRENT")); err != nil { + fs.log(fmt.Sprintf("rename CURRENT.%d: %v", fd.Num, err)) + return + } + // Sync root directory. + if err = syncDir(fs.path); err != nil { + fs.log(fmt.Sprintf("syncDir: %v", err)) + } + return } func (fs *fileStorage) GetMeta() (fd FileDesc, err error) { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go index c5be420b3f..16cfbaa006 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go @@ -581,6 +581,7 @@ func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, erro case blockTypeSnappyCompression: decLen, err := snappy.DecodedLen(data[:bh.length]) if err != nil { + r.bpool.Put(data) return nil, r.newErrCorruptedBH(bh, err.Error()) } decData := r.bpool.Get(decLen) diff --git a/vendor/vendor.json b/vendor/vendor.json index 56c95e341f..a9de0ec728 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -334,76 +334,76 @@ "revisionTime": "2016-06-18T19:32:21Z" }, { - "checksumSHA1": "QvOrAO5S37PL/6XZVWIVGyAbn60=", + "checksumSHA1": "yHbyLpI/Meh0DGrmi8x6FrDxxUY=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", "path": "github.com/syndtr/goleveldb/leveldb/cache", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=", "path": "github.com/syndtr/goleveldb/leveldb/comparer", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=", "path": "github.com/syndtr/goleveldb/leveldb/errors", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=", "path": "github.com/syndtr/goleveldb/leveldb/filter", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "8dXuAVIsbtaMiGGuHjzGR6Ny/5c=", "path": "github.com/syndtr/goleveldb/leveldb/iterator", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=", "path": "github.com/syndtr/goleveldb/leveldb/journal", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "j+uaQ6DwJ50dkIdfMQu1TXdlQcY=", "path": "github.com/syndtr/goleveldb/leveldb/memdb", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=", "path": "github.com/syndtr/goleveldb/leveldb/opt", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { - "checksumSHA1": "/Wvv9HeJTN9UUjdjwUlz7X4ioIo=", + "checksumSHA1": "tQ2AqXXAEy9icbZI9dLVdZGvWMw=", "path": "github.com/syndtr/goleveldb/leveldb/storage", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { - "checksumSHA1": "JTJA+u8zk7EXy1UUmpFPNGvtO2A=", + "checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=", "path": "github.com/syndtr/goleveldb/leveldb/table", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "4zil8Gwg8VPkDn1YzlgCvtukJFU=", "path": "github.com/syndtr/goleveldb/leveldb/util", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=", From 8e14bb1448a792072f780dae3db896bbfe366ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 16:06:04 +0300 Subject: [PATCH 12/45] Dockerfile: fix missing SSL certificates --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 17fa40951e..eae8924997 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ RUN cd /go-ethereum && make geth # Pull Geth into a second stage deploy alpine container FROM alpine:latest +RUN apk add --no-cache ca-certificates COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ EXPOSE 8545 8546 30303 30303/udp From f90a193f92e0013dc657ebe03a524a2e7286d5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 16:06:36 +0300 Subject: [PATCH 13/45] cmd/puppeth: switch node containers to main ones --- cmd/puppeth/module_node.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 9fe97c892d..8f912f9ebd 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -30,7 +30,7 @@ import ( // nodeDockerfile is the Dockerfile required to run an Ethereum node. var nodeDockerfile = ` -FROM ethereum/client-go:alpine-develop +FROM ethereum/client-go:latest ADD genesis.json /genesis.json {{if .Unlock}} @@ -38,9 +38,9 @@ ADD genesis.json /genesis.json ADD signer.pass /signer.pass {{end}} RUN \ - echo '/geth init /genesis.json' > geth.sh && \{{if .Unlock}} + echo 'geth init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh + echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -197,7 +197,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) // Container available, retrieve its node ID and its genesis json var out []byte - if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 /geth --exec admin.nodeInfo.id attach", network, kind)); err != nil { + if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id attach", network, kind)); err != nil { return nil, ErrServiceUnreachable } id := bytes.Trim(bytes.TrimSpace(out), "\"") From b0ca1b67ce6e297fe02281d01a486225bbf385f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 19:18:28 +0300 Subject: [PATCH 14/45] eth: use maxpeers from p2p layer instead of extra config --- cmd/utils/flags.go | 4 ---- eth/backend.go | 30 +++++++++++++++--------------- eth/config.go | 1 - eth/gen_config.go | 4 ---- eth/handler.go | 8 +++++--- eth/handler_test.go | 4 ++-- eth/helper_test.go | 4 ++-- params/bootnodes.go | 2 -- 8 files changed, 24 insertions(+), 33 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 37ccd06efb..5ab6047e83 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -949,10 +949,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) } - // Ethereum needs to know maxPeers to calculate the light server peer ratio. - // TODO(fjl): ensure Ethereum can get MaxPeers from node. - cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name) - if ctx.GlobalIsSet(CacheFlag.Name) { cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) } diff --git a/eth/backend.go b/eth/backend.go index 5f4f2097a7..5837c85641 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -57,15 +57,19 @@ type LesServer interface { // Ethereum implements the Ethereum full node service. type Ethereum struct { + config *Config chainConfig *params.ChainConfig + // Channel for shutting down the service shutdownChan chan bool // Channel for shutting down the ethereum stopDbUpgrade func() error // stop chain db sequential key upgrade + // Handlers txPool *core.TxPool blockchain *core.BlockChain protocolManager *ProtocolManager lesServer LesServer + // DB interfaces chainDb ethdb.Database // Block chain database @@ -98,7 +102,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if !config.SyncMode.IsValid() { return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode) } - chainDb, err := CreateDB(ctx, config, "chaindata") if err != nil { return nil, err @@ -111,6 +114,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { log.Info("Initialised chain configuration", "config", chainConfig) eth := &Ethereum{ + config: config, chainDb: chainDb, chainConfig: chainConfig, eventMux: ctx.EventMux, @@ -153,21 +157,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } eth.txPool = core.NewTxPool(config.TxPool, eth.chainConfig, eth.blockchain) - maxPeers := config.MaxPeers - if config.LightServ > 0 { - // if we are running a light server, limit the number of ETH peers so that we reserve some space for incoming LES connections - // temporary solution until the new peer connectivity API is finished - halfPeers := maxPeers / 2 - maxPeers -= config.LightPeers - if maxPeers < halfPeers { - maxPeers = halfPeers - } - } - - if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, maxPeers, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { + if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { return nil, err } - eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) eth.miner.SetExtra(makeExtraData(config.ExtraData)) @@ -376,7 +368,15 @@ func (s *Ethereum) Protocols() []p2p.Protocol { func (s *Ethereum) Start(srvr *p2p.Server) error { s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) - s.protocolManager.Start() + // Figure out a max peers count based on the server limits + maxPeers := srvr.MaxPeers + if s.config.LightServ > 0 { + maxPeers -= s.config.LightPeers + if maxPeers < srvr.MaxPeers/2 { + maxPeers = srvr.MaxPeers / 2 + } + } + s.protocolManager.Start(maxPeers) if s.lesServer != nil { s.lesServer.Start(srvr) } diff --git a/eth/config.go b/eth/config.go index 4109cff8b7..7bcfd403ea 100644 --- a/eth/config.go +++ b/eth/config.go @@ -79,7 +79,6 @@ type Config struct { // Light client options LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - MaxPeers int `toml:"-"` // Maximum number of global peers // Database options SkipBcVersionCheck bool `toml:"-"` diff --git a/eth/gen_config.go b/eth/gen_config.go index 4774794197..4a4cd7b9c5 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -47,7 +47,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SyncMode = c.SyncMode enc.LightServ = c.LightServ enc.LightPeers = c.LightPeers - enc.MaxPeers = c.MaxPeers enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -119,9 +118,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightPeers != nil { c.LightPeers = *dec.LightPeers } - if dec.MaxPeers != nil { - c.MaxPeers = *dec.MaxPeers - } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } diff --git a/eth/handler.go b/eth/handler.go index 9d230a4ad6..28ae208c06 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -99,7 +99,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, maxPeers int, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { +func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ networkId: networkId, @@ -108,7 +108,6 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne blockchain: blockchain, chaindb: chaindb, chainconfig: config, - maxPeers: maxPeers, peers: newPeerSet(), newPeerCh: make(chan *peer), noMorePeers: make(chan struct{}), @@ -203,11 +202,14 @@ func (pm *ProtocolManager) removePeer(id string) { } } -func (pm *ProtocolManager) Start() { +func (pm *ProtocolManager) Start(maxPeers int) { + pm.maxPeers = maxPeers + // broadcast transactions pm.txCh = make(chan core.TxPreEvent, txChanSize) pm.txSub = pm.txpool.SubscribeTxPreEvent(pm.txCh) go pm.txBroadcastLoop() + // broadcast mined blocks pm.minedBlockSub = pm.eventMux.Subscribe(core.NewMinedBlockEvent{}) go pm.minedBroadcastLoop() diff --git a/eth/handler_test.go b/eth/handler_test.go index aba2774444..6752cd2a84 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -476,11 +476,11 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool genesis = gspec.MustCommit(db) blockchain, _ = core.NewBlockChain(db, config, pow, vm.Config{}) ) - pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, 1000, evmux, new(testTxPool), pow, blockchain, db) + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } - pm.Start() + pm.Start(1000) defer pm.Stop() // Connect a new peer and check that we receive the DAO challenge diff --git a/eth/helper_test.go b/eth/helper_test.go index f1dab95283..b665531352 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -66,11 +66,11 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func panic(err) } - pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, 1000, evmux, &testTxPool{added: newtx}, engine, blockchain, db) + pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db) if err != nil { return nil, err } - pm.Start() + pm.Start(1000) return pm, nil } diff --git a/params/bootnodes.go b/params/bootnodes.go index 6a145a7a8f..6c6c02f8cf 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -19,7 +19,6 @@ package params // MainnetBootnodes are the enode URLs of the P2P bootstrap nodes running on // the main Ethereum network. var MainnetBootnodes = []string{ - // Ethereum Foundation Go Bootnodes "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", // IE "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", // US-WEST @@ -29,7 +28,6 @@ var MainnetBootnodes = []string{ // Ethereum Foundation Cpp Bootnodes "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303", // DE - } // TestnetBootnodes are the enode URLs of the P2P bootstrap nodes running on the From cc313e78b7dfbf44dec64ac0d22e4134ee44cc74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Sep 2017 19:49:37 +0300 Subject: [PATCH 15/45] core: use blocks and avoid deep reorgs in txpool --- core/tx_pool.go | 86 ++++++++++++++++++++++++-------------------- core/tx_pool_test.go | 16 ++++----- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index f41fbe069a..0ad7651795 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -19,6 +19,7 @@ package core import ( "errors" "fmt" + "math" "math/big" "sort" "sync" @@ -105,11 +106,11 @@ var ( // blockChain provides the state of blockchain and current gas limit to do // some pre checks in tx pool and event subscribers. type blockChain interface { - CurrentHeader() *types.Header - SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription - + CurrentBlock() *types.Block GetBlock(hash common.Hash, number uint64) *types.Block StateAt(root common.Hash) (*state.StateDB, error) + + SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription } // TxPoolConfig are the configuration parameters of the transaction pool. @@ -223,7 +224,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block } pool.locals = newAccountSet(pool.signer) pool.priced = newTxPricedList(&pool.all) - pool.reset(nil, chain.CurrentHeader()) + pool.reset(nil, chain.CurrentBlock().Header()) // If local transactions and journaling is enabled, load from disk if !config.NoLocals && config.Journal != "" { @@ -265,7 +266,7 @@ func (pool *TxPool) loop() { defer journal.Stop() // Track the previous head headers for transaction reorgs - head := pool.chain.CurrentHeader() + head := pool.chain.CurrentBlock() // Keep waiting for and reacting to the various events for { @@ -277,8 +278,8 @@ func (pool *TxPool) loop() { if pool.chainconfig.IsHomestead(ev.Block.Number()) { pool.homestead = true } - pool.reset(head, ev.Block.Header()) - head = ev.Block.Header() + pool.reset(head.Header(), ev.Block.Header()) + head = ev.Block pool.mu.Unlock() } @@ -344,43 +345,52 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { var reinject types.Transactions if oldHead != nil && oldHead.Hash() != newHead.ParentHash { - var discarded, included types.Transactions + // If the reorg is too deep, avoid doing it (will happen during fast sync) + oldNum := oldHead.Number.Uint64() + newNum := newHead.Number.Uint64() - var ( - rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) - add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) - ) - for rem.NumberU64() > add.NumberU64() { - discarded = append(discarded, rem.Transactions()...) - if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { - log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) - return + if depth := uint64(math.Abs(float64(oldNum) - float64(newNum))); depth > 64 { + log.Warn("Skipping deep transaction reorg", "depth", depth) + } else { + // Reorg seems shallow enough to pull in all transactions into memory + var discarded, included types.Transactions + + var ( + rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) + add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) + ) + for rem.NumberU64() > add.NumberU64() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } } + for add.NumberU64() > rem.NumberU64() { + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + for rem.Hash() != add.Hash() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + reinject = types.TxDifference(discarded, included) } - for add.NumberU64() > rem.NumberU64() { - included = append(included, add.Transactions()...) - if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { - log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) - return - } - } - for rem.Hash() != add.Hash() { - discarded = append(discarded, rem.Transactions()...) - if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { - log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) - return - } - included = append(included, add.Transactions()...) - if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { - log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) - return - } - } - reinject = types.TxDifference(discarded, included) } // Initialize the internal state to the current head if newHead == nil { - newHead = pool.chain.CurrentHeader() // Special case during testing + newHead = pool.chain.CurrentBlock().Header() // Special case during testing } statedb, err := pool.chain.StateAt(newHead.Root) if err != nil { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index cdd45b4b1d..17d7368774 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -50,24 +50,24 @@ type testBlockChain struct { chainHeadFeed *event.Feed } -func (bc *testBlockChain) CurrentHeader() *types.Header { - return &types.Header{ +func (bc *testBlockChain) CurrentBlock() *types.Block { + return types.NewBlock(&types.Header{ GasLimit: bc.gasLimit, - } -} - -func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription { - return bc.chainHeadFeed.Subscribe(ch) + }, nil, nil, nil) } func (bc *testBlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { - return types.NewBlock(bc.CurrentHeader(), nil, nil, nil) + return bc.CurrentBlock() } func (bc *testBlockChain) StateAt(common.Hash) (*state.StateDB, error) { return bc.statedb, nil } +func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription { + return bc.chainHeadFeed.Subscribe(ch) +} + func transaction(nonce uint64, gaslimit *big.Int, key *ecdsa.PrivateKey) *types.Transaction { return pricedTransaction(nonce, gaslimit, big.NewInt(1), key) } From 4ea4d2dc3473afd9d2eda6ef6b359accce1f0946 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Fri, 18 Aug 2017 21:52:20 +0200 Subject: [PATCH 16/45] core, eth: add bloombit indexer, filter based on it --- core/blockchain.go | 15 - core/bloombits/fetcher_test.go | 101 ++++++ core/bloombits/matcher.go | 579 ++++++++++++++++++++++++++++++ core/bloombits/matcher_test.go | 196 ++++++++++ core/bloombits/utils.go | 63 ++++ core/chain_indexer.go | 76 +++- core/chain_indexer_test.go | 7 +- core/database_util.go | 69 ++-- core/database_util_test.go | 108 ------ core/types/bloom9.go | 14 + eth/api_backend.go | 27 ++ eth/backend.go | 10 +- eth/backend_test.go | 74 ---- eth/db_upgrade.go | 68 ++-- eth/filters/api.go | 48 ++- eth/filters/bench_test.go | 237 ++++++++++++ eth/filters/filter.go | 232 +++++++----- eth/filters/filter_system_test.go | 50 ++- eth/filters/filter_test.go | 70 +--- eth/handler.go | 5 + les/api_backend.go | 18 + les/backend.go | 2 +- miner/worker.go | 2 - 23 files changed, 1589 insertions(+), 482 deletions(-) create mode 100644 core/bloombits/fetcher_test.go create mode 100644 core/bloombits/matcher.go create mode 100644 core/bloombits/matcher_test.go create mode 100644 core/bloombits/utils.go delete mode 100644 eth/backend_test.go create mode 100644 eth/filters/bench_test.go diff --git a/core/blockchain.go b/core/blockchain.go index 0bb12fc190..d74b3520b1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -759,12 +759,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ log.Crit("Failed to write block receipts", "err", err) return } - if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { - errs[index] = fmt.Errorf("failed to write log blooms: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write log blooms", "err", err) - return - } if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { errs[index] = fmt.Errorf("failed to write lookup metadata: %v", err) atomic.AddInt32(&failed, 1) @@ -1017,10 +1011,6 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { return i, err } - // Write map map bloom filters - if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { - return i, err - } // Write hash preimages if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { return i, err @@ -1178,11 +1168,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { return err } - // Write map map bloom filters - receipts := GetBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64()) - if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { - return err - } addedTxs = append(addedTxs, block.Transactions()...) } diff --git a/core/bloombits/fetcher_test.go b/core/bloombits/fetcher_test.go new file mode 100644 index 0000000000..9c229cf8da --- /dev/null +++ b/core/bloombits/fetcher_test.go @@ -0,0 +1,101 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package bloombits + +import ( + "bytes" + "encoding/binary" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" +) + +const testFetcherReqCount = 5000 + +func fetcherTestVector(b uint, s uint64) []byte { + r := make([]byte, 10) + binary.BigEndian.PutUint16(r[0:2], uint16(b)) + binary.BigEndian.PutUint64(r[2:10], s) + return r +} + +func TestFetcher(t *testing.T) { + testFetcher(t, 1) +} + +func TestFetcherMultipleReaders(t *testing.T) { + testFetcher(t, 10) +} + +func testFetcher(t *testing.T, cnt int) { + f := &fetcher{ + requestMap: make(map[uint64]fetchRequest), + } + distCh := make(chan distRequest, channelCap) + stop := make(chan struct{}) + var reqCount uint32 + + for i := 0; i < 10; i++ { + go func() { + for { + req, ok := <-distCh + if !ok { + return + } + time.Sleep(time.Duration(rand.Intn(100000))) + atomic.AddUint32(&reqCount, 1) + f.deliver([]uint64{req.sectionIndex}, [][]byte{fetcherTestVector(req.bloomIndex, req.sectionIndex)}) + } + }() + } + + var wg, wg2 sync.WaitGroup + for cc := 0; cc < cnt; cc++ { + wg.Add(1) + in := make(chan uint64, channelCap) + out := f.fetch(in, distCh, stop, &wg2) + + time.Sleep(time.Millisecond * 10 * time.Duration(cc)) + go func() { + for i := uint64(0); i < testFetcherReqCount; i++ { + in <- i + } + }() + + go func() { + for i := uint64(0); i < testFetcherReqCount; i++ { + bv := <-out + if !bytes.Equal(bv, fetcherTestVector(0, i)) { + if len(bv) != 10 { + t.Errorf("Vector #%d length is %d, expected 10", i, len(bv)) + } else { + j := binary.BigEndian.Uint64(bv[2:10]) + t.Errorf("Expected vector #%d, fetched #%d", i, j) + } + } + } + wg.Done() + }() + } + + wg.Wait() + close(stop) + if reqCount != testFetcherReqCount { + t.Errorf("Request count mismatch: expected %v, got %v", testFetcherReqCount, reqCount) + } +} diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go new file mode 100644 index 0000000000..5a7df6b1c9 --- /dev/null +++ b/core/bloombits/matcher.go @@ -0,0 +1,579 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package bloombits + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/core/types" +) + +const channelCap = 100 + +// fetcher handles bit vector retrieval pipelines for a single bit index +type fetcher struct { + bloomIndex uint + requestMap map[uint64]fetchRequest + requestLock sync.RWMutex +} + +// fetchRequest represents the state of a bit vector requested from a fetcher. When a distRequest has been sent to the distributor but +// the data has not been delivered yet, queued is true. When delivered, it is stored in the data field and the delivered channel is closed. +type fetchRequest struct { + data []byte + queued bool + delivered chan struct{} +} + +// distRequest is sent by the fetcher to the distributor which groups and prioritizes these requests. +type distRequest struct { + bloomIndex uint + sectionIndex uint64 +} + +// fetch creates a retrieval pipeline, receiving section indexes from sectionCh and returning the results +// in the same order through the returned channel. Multiple fetch instances of the same fetcher are allowed +// to run in parallel, in case the same bit index appears multiple times in the filter structure. Each section +// is requested only once, requests are sent to the request distributor (part of Matcher) through distCh. +func (f *fetcher) fetch(sectionCh chan uint64, distCh chan distRequest, stop chan struct{}, wg *sync.WaitGroup) chan []byte { + dataCh := make(chan []byte, channelCap) + returnCh := make(chan uint64, channelCap) + wg.Add(2) + + go func() { + defer wg.Done() + defer close(returnCh) + + for { + select { + case <-stop: + return + case idx, ok := <-sectionCh: + if !ok { + return + } + + req := false + f.requestLock.Lock() + r := f.requestMap[idx] + if r.data == nil { + req = !r.queued + r.queued = true + if r.delivered == nil { + r.delivered = make(chan struct{}) + } + f.requestMap[idx] = r + } + f.requestLock.Unlock() + if req { + distCh <- distRequest{bloomIndex: f.bloomIndex, sectionIndex: idx} // success is guaranteed, distibuteRequests shuts down after fetch + } + select { + case <-stop: + return + case returnCh <- idx: + } + } + } + }() + + go func() { + defer wg.Done() + defer close(dataCh) + + for { + select { + case <-stop: + return + case idx, ok := <-returnCh: + if !ok { + return + } + + f.requestLock.RLock() + r := f.requestMap[idx] + f.requestLock.RUnlock() + + if r.data == nil { + select { + case <-stop: + return + case <-r.delivered: + f.requestLock.RLock() + r = f.requestMap[idx] + f.requestLock.RUnlock() + } + } + select { + case <-stop: + return + case dataCh <- r.data: + } + } + } + }() + + return dataCh +} + +// deliver is called by the request distributor when a reply to a request has +// arrived +func (f *fetcher) deliver(sectionIdxList []uint64, data [][]byte) { + f.requestLock.Lock() + defer f.requestLock.Unlock() + + for i, sectionIdx := range sectionIdxList { + r := f.requestMap[sectionIdx] + if r.data != nil { + panic("BloomBits section data delivered twice") + } + r.data = data[i] + close(r.delivered) + f.requestMap[sectionIdx] = r + } +} + +// Matcher is a pipelined structure of fetchers and logic matchers which perform +// binary AND/OR operations on the bitstreams, finally creating a stream of potential matches. +type Matcher struct { + addresses []types.BloomIndexList + topics [][]types.BloomIndexList + fetchers map[uint]*fetcher + sectionSize uint64 + + distCh chan distRequest + reqs map[uint][]uint64 + freeQueues map[uint]struct{} + allocQueue []chan uint + running bool + stop chan struct{} + lock sync.Mutex + wg, distWg sync.WaitGroup +} + +// NewMatcher creates a new Matcher instance +func NewMatcher(sectionSize uint64, addresses []common.Address, topics [][]common.Hash) *Matcher { + m := &Matcher{ + fetchers: make(map[uint]*fetcher), + reqs: make(map[uint][]uint64), + freeQueues: make(map[uint]struct{}), + distCh: make(chan distRequest, channelCap), + sectionSize: sectionSize, + } + m.setAddresses(addresses) + m.setTopics(topics) + return m +} + +// setAddresses matches only logs that are generated from addresses that are included +// in the given addresses. +func (m *Matcher) setAddresses(addresses []common.Address) { + m.addresses = make([]types.BloomIndexList, len(addresses)) + for i, address := range addresses { + m.addresses[i] = types.BloomIndexes(address.Bytes()) + } + + for _, bloomIndexList := range m.addresses { + for _, bloomIndex := range bloomIndexList { + m.newFetcher(bloomIndex) + } + } +} + +// setTopics matches only logs that have topics matching the given topics. +func (m *Matcher) setTopics(topics [][]common.Hash) { + m.topics = nil +loop: + for _, topicList := range topics { + t := make([]types.BloomIndexList, len(topicList)) + for i, topic := range topicList { + if (topic == common.Hash{}) { + continue loop + } + t[i] = types.BloomIndexes(topic.Bytes()) + } + m.topics = append(m.topics, t) + } + + for _, bloomIndexLists := range m.topics { + for _, bloomIndexList := range bloomIndexLists { + for _, bloomIndex := range bloomIndexList { + m.newFetcher(bloomIndex) + } + } + } +} + +// match creates a daisy-chain of sub-matchers, one for the address set and one for each topic set, each +// sub-matcher receiving a section only if the previous ones have all found a potential match in one of +// the blocks of the section, then binary AND-ing its own matches and forwaring the result to the next one +func (m *Matcher) match(processCh chan partialMatches) chan partialMatches { + indexLists := m.topics + if len(m.addresses) > 0 { + indexLists = append([][]types.BloomIndexList{m.addresses}, indexLists...) + } + m.distributeRequests() + + for _, subIndexList := range indexLists { + processCh = m.subMatch(processCh, subIndexList) + } + return processCh +} + +// partialMatches with a non-nil vector represents a section in which some sub-matchers have already +// found potential matches. Subsequent sub-matchers will binary AND their matches with this vector. +// If vector is nil, it represents a section to be processed by the first sub-matcher. +type partialMatches struct { + sectionIndex uint64 + vector []byte +} + +// newFetcher adds a fetcher for the given bit index if it has not existed before +func (m *Matcher) newFetcher(idx uint) { + if _, ok := m.fetchers[idx]; ok { + return + } + f := &fetcher{ + bloomIndex: idx, + requestMap: make(map[uint64]fetchRequest), + } + m.fetchers[idx] = f +} + +// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then +// binary AND-s the result to the daisy-chain input (processCh) and forwards it to the daisy-chain output. +// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to +// that address/topic, and binary AND-ing those vectors together. +func (m *Matcher) subMatch(processCh chan partialMatches, bloomIndexLists []types.BloomIndexList) chan partialMatches { + // set up fetchers + fetchIndexChannels := make([][3]chan uint64, len(bloomIndexLists)) + fetchDataChannels := make([][3]chan []byte, len(bloomIndexLists)) + for i, bloomIndexList := range bloomIndexLists { + for j, bloomIndex := range bloomIndexList { + fetchIndexChannels[i][j] = make(chan uint64, channelCap) + fetchDataChannels[i][j] = m.fetchers[bloomIndex].fetch(fetchIndexChannels[i][j], m.distCh, m.stop, &m.wg) + } + } + + fetchedCh := make(chan partialMatches, channelCap) // entries from processCh are forwarded here after fetches have been initiated + resultsCh := make(chan partialMatches, channelCap) + + m.wg.Add(2) + // goroutine for starting retrievals + go func() { + defer m.wg.Done() + + for { + select { + case <-m.stop: + return + case s, ok := <-processCh: + if !ok { + close(fetchedCh) + for _, fetchIndexChs := range fetchIndexChannels { + for _, fetchIndexCh := range fetchIndexChs { + close(fetchIndexCh) + } + } + return + } + + for _, fetchIndexChs := range fetchIndexChannels { + for _, fetchIndexCh := range fetchIndexChs { + select { + case <-m.stop: + return + case fetchIndexCh <- s.sectionIndex: + } + } + } + select { + case <-m.stop: + return + case fetchedCh <- s: + } + } + } + }() + + // goroutine for processing retrieved data + go func() { + defer m.wg.Done() + + for { + select { + case <-m.stop: + return + case s, ok := <-fetchedCh: + if !ok { + close(resultsCh) + return + } + + var orVector []byte + for _, fetchDataChs := range fetchDataChannels { + var andVector []byte + for _, fetchDataCh := range fetchDataChs { + var data []byte + select { + case <-m.stop: + return + case data = <-fetchDataCh: + } + if andVector == nil { + andVector = make([]byte, int(m.sectionSize/8)) + copy(andVector, data) + } else { + bitutil.ANDBytes(andVector, andVector, data) + } + } + if orVector == nil { + orVector = andVector + } else { + bitutil.ORBytes(orVector, orVector, andVector) + } + } + + if orVector == nil { + orVector = make([]byte, int(m.sectionSize/8)) + } + if s.vector != nil { + bitutil.ANDBytes(orVector, orVector, s.vector) + } + if bitutil.TestBytes(orVector) { + select { + case <-m.stop: + return + case resultsCh <- partialMatches{s.sectionIndex, orVector}: + } + } + } + } + }() + + return resultsCh +} + +// Start starts the matching process and returns a stream of bloom matches in +// a given range of blocks. +// It returns a results channel immediately and stops if Stop is called or there +// are no more matches in the range (in which case the results channel is closed). +// Start/Stop can be called multiple times for different ranges, in which case already +// delivered bit vectors are not requested again. +func (m *Matcher) Start(begin, end uint64) chan uint64 { + m.stop = make(chan struct{}) + processCh := make(chan partialMatches, channelCap) + resultsCh := make(chan uint64, channelCap) + + res := m.match(processCh) + + startSection := begin / m.sectionSize + endSection := end / m.sectionSize + + m.wg.Add(2) + go func() { + defer m.wg.Done() + defer close(processCh) + + for i := startSection; i <= endSection; i++ { + select { + case processCh <- partialMatches{i, nil}: + case <-m.stop: + return + } + } + }() + + go func() { + defer m.wg.Done() + defer close(resultsCh) + + for { + select { + case r, ok := <-res: + if !ok { + return + } + sectionStart := r.sectionIndex * m.sectionSize + s := sectionStart + if begin > s { + s = begin + } + e := sectionStart + m.sectionSize - 1 + if end < e { + e = end + } + for i := s; i <= e; i++ { + b := r.vector[(i-sectionStart)/8] + bit := 7 - i%8 + if b != 0 { + if b&(1< queue[i] { + i++ + } + queue = append(queue, 0) + copy(queue[i+1:], queue[i:len(queue)-1]) + queue[i] = r.sectionIndex + m.reqs[r.bloomIndex] = queue + if len(queue) == 1 { + m.freeQueue(r.bloomIndex) + } + m.lock.Unlock() + case <-stopDist: + m.lock.Lock() + for _, ch := range m.allocQueue { + close(ch) + } + m.allocQueue = nil + m.running = false + m.lock.Unlock() + m.distWg.Done() + return + } + } + }() +} + +// freeQueue marks a queue as free if there are no AllocSectionQueue functions +// waiting for allocation. If there is someone waiting, the queue is immediately +// allocated. +func (m *Matcher) freeQueue(bloomIndex uint) { + if len(m.allocQueue) > 0 { + m.allocQueue[0] <- bloomIndex + m.allocQueue = m.allocQueue[1:] + } else { + m.freeQueues[bloomIndex] = struct{}{} + } +} + +// AllocSectionQueue allocates a queue of requested section indexes belonging to the same +// bloom bit index for a client process that can either immediately fetch the contents +// of the queue or wait a little while for more section indexes to be requested. +func (m *Matcher) AllocSectionQueue() (uint, bool) { + m.lock.Lock() + if !m.running { + m.lock.Unlock() + return 0, false + } + + var allocCh chan uint + if len(m.freeQueues) > 0 { + var ( + found bool + bestSection uint64 + bestIndex uint + ) + for bloomIndex, _ := range m.freeQueues { + if !found || m.reqs[bloomIndex][0] < bestSection { + found = true + bestIndex = bloomIndex + bestSection = m.reqs[bloomIndex][0] + } + } + delete(m.freeQueues, bestIndex) + m.lock.Unlock() + return bestIndex, true + } else { + allocCh = make(chan uint) + m.allocQueue = append(m.allocQueue, allocCh) + } + m.lock.Unlock() + + bloomIndex, ok := <-allocCh + return bloomIndex, ok +} + +// SectionCount returns the length of the section index queue belonging to the given bloom bit index +func (m *Matcher) SectionCount(bloomIndex uint) int { + m.lock.Lock() + defer m.lock.Unlock() + + return len(m.reqs[bloomIndex]) +} + +// FetchSections fetches all or part of an already allocated queue and deallocates it +func (m *Matcher) FetchSections(bloomIndex uint, maxCount int) []uint64 { + m.lock.Lock() + defer m.lock.Unlock() + + queue := m.reqs[bloomIndex] + if maxCount < len(queue) { + // return only part of the existing queue, mark the rest as free + m.reqs[bloomIndex] = queue[maxCount:] + m.freeQueue(bloomIndex) + return queue[:maxCount] + } else { + // return the entire queue + delete(m.reqs, bloomIndex) + return queue + } +} + +// Deliver delivers a bit vector to the appropriate fetcher. +// It is possible to deliver data even after Stop has been called. Once a vector has been +// requested, the matcher will keep waiting for delivery. +func (m *Matcher) Deliver(bloomIndex uint, sectionIdxList []uint64, data [][]byte) { + m.fetchers[bloomIndex].deliver(sectionIdxList, data) +} diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go new file mode 100644 index 0000000000..bef1491b83 --- /dev/null +++ b/core/bloombits/matcher_test.go @@ -0,0 +1,196 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package bloombits + +import ( + "math/rand" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/core/types" +) + +const testSectionSize = 4096 + +func matcherTestVector(b uint, s uint64) []byte { + r := make([]byte, testSectionSize/8) + for i, _ := range r { + var bb byte + for bit := 0; bit < 8; bit++ { + blockIdx := s*testSectionSize + uint64(i*8+bit) + bb += bb + if (blockIdx % uint64(b)) == 0 { + bb++ + } + } + r[i] = bb + } + return r +} + +func expMatch1(idxs types.BloomIndexList, i uint64) bool { + for _, ii := range idxs { + if (i % uint64(ii)) != 0 { + return false + } + } + return true +} + +func expMatch2(idxs []types.BloomIndexList, i uint64) bool { + for _, ii := range idxs { + if expMatch1(ii, i) { + return true + } + } + return false +} + +func expMatch3(idxs [][]types.BloomIndexList, i uint64) bool { + for _, ii := range idxs { + if !expMatch2(ii, i) { + return false + } + } + return true +} + +func testServeMatcher(m *Matcher, stop chan struct{}, cnt *uint32, maxRequestLen int) { + // serve matcher with test vectors + for i := 0; i < 10; i++ { + go func() { + for { + select { + case <-stop: + return + default: + } + b, ok := m.AllocSectionQueue() + if !ok { + return + } + if m.SectionCount(b) < maxRequestLen { + time.Sleep(time.Microsecond * 100) + } + s := m.FetchSections(b, maxRequestLen) + res := make([][]byte, len(s)) + for i, ss := range s { + res[i] = matcherTestVector(b, ss) + atomic.AddUint32(cnt, 1) + } + m.Deliver(b, s, res) + } + }() + } +} + +func testMatcher(t *testing.T, idxs [][]types.BloomIndexList, cnt uint64, stopOnMatches bool, expCount uint32) uint32 { + count1 := testMatcherWithReqCount(t, idxs, cnt, stopOnMatches, expCount, 1) + count16 := testMatcherWithReqCount(t, idxs, cnt, stopOnMatches, expCount, 16) + if count1 != count16 { + t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: request count mismatch, %v with maxReqCount = 1 vs. %v with maxReqCount = 16", idxs, cnt, stopOnMatches, count1, count16) + } + return count1 +} + +func testMatcherWithReqCount(t *testing.T, idxs [][]types.BloomIndexList, cnt uint64, stopOnMatches bool, expCount uint32, maxReqCount int) uint32 { + m := NewMatcher(testSectionSize, nil, nil) + + for _, idxss := range idxs { + for _, idxs := range idxss { + for _, idx := range idxs { + m.newFetcher(idx) + } + } + } + + m.addresses = idxs[0] + m.topics = idxs[1:] + var reqCount uint32 + + stop := make(chan struct{}) + chn := m.Start(0, cnt-1) + testServeMatcher(m, stop, &reqCount, maxReqCount) + + for i := uint64(0); i < cnt; i++ { + if expMatch3(idxs, i) { + match, ok := <-chn + if !ok { + t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: expected #%v, results channel closed", idxs, cnt, stopOnMatches, i) + return 0 + } + if match != i { + t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: expected #%v, got #%v", idxs, cnt, stopOnMatches, i, match) + } + if stopOnMatches { + m.Stop() + close(stop) + stop = make(chan struct{}) + chn = m.Start(i+1, cnt-1) + testServeMatcher(m, stop, &reqCount, maxReqCount) + } + } + } + match, ok := <-chn + if ok { + t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: expected closed channel, got #%v", idxs, cnt, stopOnMatches, match) + } + m.Stop() + close(stop) + + if expCount != 0 && expCount != reqCount { + t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: request count mismatch, expected #%v, got #%v", idxs, cnt, stopOnMatches, expCount, reqCount) + } + + return reqCount +} + +func testRandomIdxs(l []int, max int) [][]types.BloomIndexList { + res := make([][]types.BloomIndexList, len(l)) + for i, ll := range l { + res[i] = make([]types.BloomIndexList, ll) + for j, _ := range res[i] { + for k, _ := range res[i][j] { + res[i][j][k] = uint(rand.Intn(max-1) + 2) + } + } + } + return res +} + +func TestMatcher(t *testing.T) { + testMatcher(t, [][]types.BloomIndexList{{{10, 20, 30}}}, 100000, false, 75) + testMatcher(t, [][]types.BloomIndexList{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81) + testMatcher(t, [][]types.BloomIndexList{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36) +} + +func TestMatcherStopOnMatches(t *testing.T) { + testMatcher(t, [][]types.BloomIndexList{{{10, 20, 30}}}, 100000, true, 75) + testMatcher(t, [][]types.BloomIndexList{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36) +} + +func TestMatcherRandom(t *testing.T) { + for i := 0; i < 20; i++ { + testMatcher(t, testRandomIdxs([]int{1}, 50), 100000, false, 0) + testMatcher(t, testRandomIdxs([]int{3}, 50), 100000, false, 0) + testMatcher(t, testRandomIdxs([]int{2, 2, 2}, 20), 100000, false, 0) + testMatcher(t, testRandomIdxs([]int{5, 5, 5}, 50), 100000, false, 0) + idxs := testRandomIdxs([]int{2, 2, 2}, 20) + reqCount := testMatcher(t, idxs, 10000, false, 0) + testMatcher(t, idxs, 10000, true, reqCount) + } +} diff --git a/core/bloombits/utils.go b/core/bloombits/utils.go new file mode 100644 index 0000000000..d0755cb65c --- /dev/null +++ b/core/bloombits/utils.go @@ -0,0 +1,63 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package bloombits + +import ( + "github.com/ethereum/go-ethereum/core/types" +) + +const BloomLength = 2048 + +// BloomBitsCreator takes SectionSize number of header bloom filters and calculates the bloomBits vectors of the section +type BloomBitsCreator struct { + blooms [BloomLength][]byte + sectionSize, bitIndex uint64 +} + +func NewBloomBitsCreator(sectionSize uint64) *BloomBitsCreator { + b := &BloomBitsCreator{sectionSize: sectionSize} + for i, _ := range b.blooms { + b.blooms[i] = make([]byte, sectionSize/8) + } + return b +} + +// AddHeaderBloom takes a single bloom filter and sets the corresponding bit column in memory accordingly +func (b *BloomBitsCreator) AddHeaderBloom(bloom types.Bloom) { + if b.bitIndex >= b.sectionSize { + panic("too many header blooms added") + } + + byteIdx := b.bitIndex / 8 + bitMask := byte(1) << byte(7-b.bitIndex%8) + for bloomBitIdx, _ := range b.blooms { + bloomByteIdx := BloomLength/8 - 1 - bloomBitIdx/8 + bloomBitMask := byte(1) << byte(bloomBitIdx%8) + if (bloom[bloomByteIdx] & bloomBitMask) != 0 { + b.blooms[bloomBitIdx][byteIdx] |= bitMask + } + } + b.bitIndex++ +} + +// GetBitVector returns the bit vector belonging to the given bit index after header blooms have been added +func (b *BloomBitsCreator) GetBitVector(idx uint) []byte { + if b.bitIndex != b.sectionSize { + panic("not enough header blooms added") + } + + return b.blooms[idx][:] +} diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 9a88a5b1b8..56360b59a3 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -36,7 +36,7 @@ import ( type ChainIndexerBackend interface { // Reset initiates the processing of a new chain segment, potentially terminating // any partially completed operations (in case of a reorg). - Reset(section uint64) + Reset(section uint64, lastSectionHead common.Hash) // Process crunches through the next header in the chain segment. The caller // will ensure a sequential order of headers. @@ -44,7 +44,7 @@ type ChainIndexerBackend interface { // Commit finalizes the section metadata and stores it into the database. This // interface will usually be a batch writer. - Commit(db ethdb.Database) error + Commit() error } // ChainIndexer does a post-processing job for equally sized sections of the @@ -101,10 +101,34 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken return c } +// AddKnownSectionHead marks a new section head as known/processed if it is newer +// than the already known best section head +func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) { + c.lock.Lock() + defer c.lock.Unlock() + + if section < c.storedSections { + return + } + c.setSectionHead(section, shead) + c.setValidSections(section + 1) +} + +// IndexerChain interface is used for connecting the indexer to a blockchain +type IndexerChain interface { + CurrentHeader() *types.Header + SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription +} + // Start creates a goroutine to feed chain head events into the indexer for -// cascading background processing. -func (c *ChainIndexer) Start(currentHeader *types.Header, eventMux *event.TypeMux) { - go c.eventLoop(currentHeader, eventMux) +// cascading background processing. Children do not need to be started, they +// are notified about new events by their parents. +func (c *ChainIndexer) Start(chain IndexerChain) { + ch := make(chan ChainEvent, 10) + sub := chain.SubscribeChainEvent(ch) + currentHeader := chain.CurrentHeader() + + go c.eventLoop(currentHeader, ch, sub) } // Close tears down all goroutines belonging to the indexer and returns any error @@ -125,6 +149,14 @@ func (c *ChainIndexer) Close() error { errs = append(errs, err) } } + + // Close all children + for _, child := range c.children { + if err := child.Close(); err != nil { + errs = append(errs, err) + } + } + // Return any failures switch { case len(errs) == 0: @@ -141,12 +173,10 @@ func (c *ChainIndexer) Close() error { // eventLoop is a secondary - optional - event loop of the indexer which is only // started for the outermost indexer to push chain head events into a processing // queue. -func (c *ChainIndexer) eventLoop(currentHeader *types.Header, eventMux *event.TypeMux) { +func (c *ChainIndexer) eventLoop(currentHeader *types.Header, ch chan ChainEvent, sub event.Subscription) { // Mark the chain indexer as active, requiring an additional teardown atomic.StoreUint32(&c.active, 1) - // Subscribe to chain head events - sub := eventMux.Subscribe(ChainEvent{}) defer sub.Unsubscribe() // Fire the initial new head event to start any outstanding processing @@ -163,14 +193,14 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, eventMux *event.Ty errc <- nil return - case ev, ok := <-sub.Chan(): + case ev, ok := <-ch: // Received a new event, ensure it's not nil (closing) and update if !ok { errc := <-c.quit errc <- nil return } - header := ev.Data.(ChainEvent).Block.Header() + header := ev.Block.Header() if header.ParentHash != prevHash { c.newHead(FindCommonAncestor(c.chainDb, prevHeader, header).Number.Uint64(), true) } @@ -226,7 +256,10 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { // updateLoop is the main event loop of the indexer which pushes chain segments // down into the processing backend. func (c *ChainIndexer) updateLoop() { - var updated time.Time + var ( + updated time.Time + updateMsg bool + ) for { select { @@ -242,6 +275,7 @@ func (c *ChainIndexer) updateLoop() { // Periodically print an upgrade log message to the user if time.Since(updated) > 8*time.Second { if c.knownSections > c.storedSections+1 { + updateMsg = true c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections) } updated = time.Now() @@ -250,17 +284,24 @@ func (c *ChainIndexer) updateLoop() { section := c.storedSections var oldHead common.Hash if section > 0 { - oldHead = c.sectionHead(section - 1) + oldHead = c.SectionHead(section - 1) } // Process the newly defined section in the background c.lock.Unlock() newHead, err := c.processSection(section, oldHead) + if err != nil { + c.log.Error("Section processing failed", "error", err) + } c.lock.Lock() // If processing succeeded and no reorgs occcurred, mark the section completed - if err == nil && oldHead == c.sectionHead(section-1) { + if err == nil && oldHead == c.SectionHead(section-1) { c.setSectionHead(section, newHead) c.setValidSections(section + 1) + if c.storedSections == c.knownSections && updateMsg { + updateMsg = false + c.log.Info("Finished upgrading chain index") + } c.cascadedHead = c.storedSections*c.sectionSize - 1 for _, child := range c.children { @@ -295,7 +336,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com c.log.Trace("Processing new chain section", "section", section) // Reset and partial processing - c.backend.Reset(section) + c.backend.Reset(section, lastHead) for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { hash := GetCanonicalHash(c.chainDb, number) @@ -311,7 +352,8 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com c.backend.Process(header) lastHead = header.Hash() } - if err := c.backend.Commit(c.chainDb); err != nil { + if err := c.backend.Commit(); err != nil { + c.log.Error("Section commit failed", "error", err) return common.Hash{}, err } return lastHead, nil @@ -324,7 +366,7 @@ func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) { c.lock.Lock() defer c.lock.Unlock() - return c.storedSections, c.storedSections*c.sectionSize - 1, c.sectionHead(c.storedSections - 1) + return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1) } // AddChildIndexer adds a child ChainIndexer that can use the output of this one @@ -366,7 +408,7 @@ func (c *ChainIndexer) setValidSections(sections uint64) { // sectionHead retrieves the last block hash of a processed section from the // index database. -func (c *ChainIndexer) sectionHead(section uint64) common.Hash { +func (c *ChainIndexer) SectionHead(section uint64) common.Hash { var data [8]byte binary.BigEndian.PutUint64(data[:], section) diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 780e46e433..247f52cf9e 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" ) @@ -58,7 +59,6 @@ func testChainIndexer(t *testing.T, count int) { ) backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)} backends[i].indexer = NewChainIndexer(db, ethdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i)) - defer backends[i].indexer.Close() if sections, _, _ := backends[i].indexer.Sections(); sections != 0 { t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0) @@ -67,6 +67,7 @@ func testChainIndexer(t *testing.T, count int) { backends[i-1].indexer.AddChildIndexer(backends[i].indexer) } } + defer backends[0].indexer.Close() // parent indexer shuts down children // notify pings the root indexer about a new head or reorg, then expect // processed blocks if a section is processable notify := func(headNum, failNum uint64, reorg bool) { @@ -208,7 +209,7 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { return b.stored * b.indexer.sectionSize } -func (b *testChainIndexBackend) Reset(section uint64) { +func (b *testChainIndexBackend) Reset(section uint64, lastSectionHead common.Hash) { b.section = section b.headerCnt = 0 } @@ -226,7 +227,7 @@ func (b *testChainIndexBackend) Process(header *types.Header) { } } -func (b *testChainIndexBackend) Commit(db ethdb.Database) error { +func (b *testChainIndexBackend) Commit() error { if b.headerCnt != b.indexer.sectionSize { b.t.Error("Not enough headers processed") } diff --git a/core/database_util.go b/core/database_util.go index 6971113944..179d6f1b2d 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "math/big" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -48,9 +47,6 @@ var ( lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage - mipmapPre = []byte("mipmap-log-bloom-") - MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000} - configPrefix = []byte("ethereum-config-") // config prefix for the db // used by old db, now only used for conversion @@ -59,10 +55,10 @@ var ( ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error - mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms - preimageCounter = metrics.NewCounter("db/preimage/total") preimageHitCounter = metrics.NewCounter("db/preimage/hits") + + bloomBitsPrefix = []byte("bloomBits-") ) // txLookupEntry is a positional metadata to help looking up the data content of @@ -497,48 +493,6 @@ func DeleteTxLookupEntry(db ethdb.Database, hash common.Hash) { db.Delete(append(lookupPrefix, hash.Bytes()...)) } -// returns a formatted MIP mapped key by adding prefix, canonical number and level -// -// ex. fn(98, 1000) = (prefix || 1000 || 0) -func mipmapKey(num, level uint64) []byte { - lkey := make([]byte, 8) - binary.BigEndian.PutUint64(lkey, level) - key := new(big.Int).SetUint64(num / level * level) - - return append(mipmapPre, append(lkey, key.Bytes()...)...) -} - -// WriteMipmapBloom writes each address included in the receipts' logs to the -// MIP bloom bin. -func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error { - mipmapBloomMu.Lock() - defer mipmapBloomMu.Unlock() - - batch := db.NewBatch() - for _, level := range MIPMapLevels { - key := mipmapKey(number, level) - bloomDat, _ := db.Get(key) - bloom := types.BytesToBloom(bloomDat) - for _, receipt := range receipts { - for _, log := range receipt.Logs { - bloom.Add(log.Address.Big()) - } - } - batch.Put(key, bloom.Bytes()) - } - if err := batch.Write(); err != nil { - return fmt.Errorf("mipmap write fail for: %d: %v", number, err) - } - return nil -} - -// GetMipmapBloom returns a bloom filter using the number and level as input -// parameters. For available levels see MIPMapLevels. -func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom { - bloomDat, _ := db.Get(mipmapKey(number, level)) - return types.BytesToBloom(bloomDat) -} - // PreimageTable returns a Database instance with the key prefix for preimage entries. func PreimageTable(db ethdb.Database) ethdb.Database { return ethdb.NewTable(db, preimagePrefix) @@ -637,3 +591,22 @@ func FindCommonAncestor(db ethdb.Database, a, b *types.Header) *types.Header { } return a } + +// GetBloomBits reads the compressed bloomBits vector belonging to the given section and bit index from the db +func GetBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, sectionHead common.Hash) ([]byte, error) { + var encKey [10]byte + binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx)) + binary.BigEndian.PutUint64(encKey[2:10], sectionIdx) + key := append(append(bloomBitsPrefix, encKey[:]...), sectionHead.Bytes()...) + bloomBits, err := db.Get(key) + return bloomBits, err +} + +// StoreBloomBits writes the compressed bloomBits vector belonging to the given section and bit index to the db +func StoreBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, sectionHead common.Hash, bloomBits []byte) { + var encKey [10]byte + binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx)) + binary.BigEndian.PutUint64(encKey[2:10], sectionIdx) + key := append(append(bloomBitsPrefix, encKey[:]...), sectionHead.Bytes()...) + db.Put(key, bloomBits) +} diff --git a/core/database_util_test.go b/core/database_util_test.go index e91f1b5930..940221a299 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -18,17 +18,13 @@ package core import ( "bytes" - "io/ioutil" "math/big" - "os" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -390,107 +386,3 @@ func TestBlockReceiptStorage(t *testing.T) { t.Fatalf("deleted receipts returned: %v", rs) } } - -func TestMipmapBloom(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - - receipt1 := new(types.Receipt) - receipt1.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test"))}, - {Address: common.BytesToAddress([]byte("address"))}, - } - receipt2 := new(types.Receipt) - receipt2.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test"))}, - {Address: common.BytesToAddress([]byte("address1"))}, - } - - WriteMipmapBloom(db, 1, types.Receipts{receipt1}) - WriteMipmapBloom(db, 2, types.Receipts{receipt2}) - - for _, level := range MIPMapLevels { - bloom := GetMipmapBloom(db, 2, level) - if !bloom.Test(new(big.Int).SetBytes([]byte("address1"))) { - t.Error("expected test to be included on level:", level) - } - } - - // reset - db, _ = ethdb.NewMemDatabase() - receipt := new(types.Receipt) - receipt.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test"))}, - } - WriteMipmapBloom(db, 999, types.Receipts{receipt1}) - - receipt = new(types.Receipt) - receipt.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test 1"))}, - } - WriteMipmapBloom(db, 1000, types.Receipts{receipt}) - - bloom := GetMipmapBloom(db, 1000, 1000) - if bloom.TestBytes([]byte("test")) { - t.Error("test should not have been included") - } -} - -func TestMipmapChain(t *testing.T) { - dir, err := ioutil.TempDir("", "mipmap") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - var ( - db, _ = ethdb.NewLDBDatabase(dir, 0, 0) - key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - addr = crypto.PubkeyToAddress(key1.PublicKey) - addr2 = common.BytesToAddress([]byte("jeff")) - - hash1 = common.BytesToHash([]byte("topic1")) - ) - defer db.Close() - - gspec := &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{addr: {Balance: big.NewInt(1000000)}}, - } - genesis := gspec.MustCommit(db) - chain, receipts := GenerateChain(params.TestChainConfig, genesis, db, 1010, func(i int, gen *BlockGen) { - var receipts types.Receipts - switch i { - case 1: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr, Topics: []common.Hash{hash1}}} - gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} - case 1000: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr2}} - gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} - - } - - // store the receipts - WriteMipmapBloom(db, uint64(i+1), receipts) - }) - for i, block := range chain { - WriteBlock(db, block) - if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } - } - - bloom := GetMipmapBloom(db, 0, 1000) - if bloom.TestBytes(addr2[:]) { - t.Error("address was included in bloom and should not have") - } -} diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 60aacc3016..bdc6e60e79 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -106,6 +106,20 @@ func LogsBloom(logs []*Log) *big.Int { return bin } +type BloomIndexList [3]uint + +// BloomIndexes returns the bloom filter bit indexes belonging to the given key +func BloomIndexes(b []byte) BloomIndexList { + b = crypto.Keccak256(b[:]) + + var r [3]uint + for i, _ := range r { + r[i] = (uint(b[i+i+1]) + (uint(b[i+i]) << 8)) & 2047 + } + + return r +} + func bloom9(b []byte) *big.Int { b = crypto.Keccak256(b[:]) diff --git a/eth/api_backend.go b/eth/api_backend.go index 19ef79f234..fa3cf3f805 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -194,3 +195,29 @@ func (b *EthApiBackend) EventMux() *event.TypeMux { func (b *EthApiBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } + +func (b *EthApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { + results := make([][]byte, len(sectionIdxList)) + var err error + for i, sectionIdx := range sectionIdxList { + sectionHead := core.GetCanonicalHash(b.eth.chainDb, (sectionIdx+1)*bloomBitsSection-1) + results[i], err = core.GetBloomBits(b.eth.chainDb, bitIdx, sectionIdx, sectionHead) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (b *EthApiBackend) BloomBitsSections() uint64 { + sections, _, _ := b.eth.bbIndexer.Sections() + return sections +} + +func (b *EthApiBackend) BloomBitsConfig() filters.BloomConfig { + return filters.BloomConfig{ + SectionSize: bloomBitsSection, + MaxRequestLen: 16, + MaxRequestWait: 0, + } +} diff --git a/eth/backend.go b/eth/backend.go index 5837c85641..efc0a2317f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -77,6 +77,8 @@ type Ethereum struct { engine consensus.Engine accountManager *accounts.Manager + bbIndexer *core.ChainIndexer + ApiBackend *EthApiBackend miner *miner.Miner @@ -125,11 +127,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { networkId: config.NetworkId, gasPrice: config.GasPrice, etherbase: config.Etherbase, + bbIndexer: NewBloomBitsProcessor(chainDb, bloomBitsSection), } - if err := addMipmapBloomBins(chainDb); err != nil { - return nil, err - } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) if !config.SkipBcVersionCheck { @@ -151,6 +151,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.blockchain.SetHead(compat.RewindTo) core.WriteChainConfig(chainDb, genesisHash, chainConfig) } + eth.bbIndexer.Start(eth.blockchain) if config.TxPool.Journal != "" { config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal) @@ -260,7 +261,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, false), + Service: filters.NewPublicFilterAPI(s.ApiBackend, false, bloomBitsSection), Public: true, }, { Namespace: "admin", @@ -389,6 +390,7 @@ func (s *Ethereum) Stop() error { if s.stopDbUpgrade != nil { s.stopDbUpgrade() } + s.bbIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() if s.lesServer != nil { diff --git a/eth/backend_test.go b/eth/backend_test.go deleted file mode 100644 index 1fd25e95aa..0000000000 --- a/eth/backend_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" -) - -func TestMipmapUpgrade(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - addr := common.BytesToAddress([]byte("jeff")) - genesis := new(core.Genesis).MustCommit(db) - - chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) { - switch i { - case 1: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr}} - gen.AddUncheckedReceipt(receipt) - case 2: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr}} - gen.AddUncheckedReceipt(receipt) - } - }) - for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } - } - - err := addMipmapBloomBins(db) - if err != nil { - t.Fatal(err) - } - - bloom := core.GetMipmapBloom(db, 1, core.MIPMapLevels[0]) - if (bloom == types.Bloom{}) { - t.Error("got empty bloom filter") - } - - data, _ := db.Get([]byte("setting-mipmap-version")) - if len(data) == 0 { - t.Error("setting-mipmap-version not written to database") - } -} diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go index 90111b2b38..ce8ce699ab 100644 --- a/eth/db_upgrade.go +++ b/eth/db_upgrade.go @@ -19,11 +19,13 @@ package eth import ( "bytes" - "fmt" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -135,45 +137,37 @@ func upgradeDeduplicateData(db ethdb.Database) func() error { } } -func addMipmapBloomBins(db ethdb.Database) (err error) { - const mipmapVersion uint = 2 +// BloomBitsIndex implements ChainIndex +type BloomBitsIndex struct { + db ethdb.Database + bc *bloombits.BloomBitsCreator + section, sectionSize uint64 + sectionHead common.Hash +} - // check if the version is set. We ignore data for now since there's - // only one version so we can easily ignore it for now - var data []byte - data, _ = db.Get([]byte("setting-mipmap-version")) - if len(data) > 0 { - var version uint - if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion { - return nil - } - } +// number of confirmation blocks before a section is considered probably final and its bloom bits are calculated +const bloomBitsConfirmations = 256 - defer func() { - if err == nil { - var val []byte - val, err = rlp.EncodeToBytes(mipmapVersion) - if err == nil { - err = db.Put([]byte("setting-mipmap-version"), val) - } - return - } - }() - latestHash := core.GetHeadBlockHash(db) - latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash)) - if latestBlock == nil { // clean database - return - } +// NewBloomBitsProcessor returns a chain processor that generates bloom bits data for the canonical chain +func NewBloomBitsProcessor(db ethdb.Database, sectionSize uint64) *core.ChainIndexer { + backend := &BloomBitsIndex{db: db, sectionSize: sectionSize} + return core.NewChainIndexer(db, ethdb.NewTable(db, "bbIndex-"), backend, sectionSize, bloomBitsConfirmations, time.Millisecond*100, "bloombits") +} - tstart := time.Now() - log.Warn("Upgrading db log bloom bins") - for i := uint64(0); i <= latestBlock.NumberU64(); i++ { - hash := core.GetCanonicalHash(db, i) - if (hash == common.Hash{}) { - return fmt.Errorf("chain db corrupted. Could not find block %d.", i) - } - core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i)) +func (b *BloomBitsIndex) Reset(section uint64, lastSectionHead common.Hash) { + b.bc = bloombits.NewBloomBitsCreator(b.sectionSize) + b.section = section +} + +func (b *BloomBitsIndex) Process(header *types.Header) { + b.bc.AddHeaderBloom(header.Bloom) + b.sectionHead = header.Hash() +} + +func (b *BloomBitsIndex) Commit() error { + for i := 0; i < bloombits.BloomLength; i++ { + compVector := bitutil.CompressBytes(b.bc.GetBitVector(uint(i))) + core.StoreBloomBits(b.db, uint64(i), b.section, b.sectionHead, compVector) } - log.Info("Bloom-bin upgrade completed", "elapsed", common.PrettyDuration(time.Since(tstart))) return nil } diff --git a/eth/filters/api.go b/eth/filters/api.go index fff58a268e..11767753e3 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -51,24 +51,25 @@ type filter struct { // PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. type PublicFilterAPI struct { - backend Backend - useMipMap bool - mux *event.TypeMux - chainDb ethdb.Database - events *EventSystem - filtersMu sync.Mutex - filters map[rpc.ID]*filter + backend Backend + bloomBitsSection uint64 + mux *event.TypeMux + quit chan struct{} + chainDb ethdb.Database + events *EventSystem + filtersMu sync.Mutex + filters map[rpc.ID]*filter } // NewPublicFilterAPI returns a new PublicFilterAPI instance. -func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI { +func NewPublicFilterAPI(backend Backend, lightMode bool, bloomBitsSection uint64) *PublicFilterAPI { api := &PublicFilterAPI{ - backend: backend, - useMipMap: !lightMode, - mux: backend.EventMux(), - chainDb: backend.ChainDb(), - events: NewEventSystem(backend.EventMux(), backend, lightMode), - filters: make(map[rpc.ID]*filter), + backend: backend, + bloomBitsSection: bloomBitsSection, + mux: backend.EventMux(), + chainDb: backend.ChainDb(), + events: NewEventSystem(backend.EventMux(), backend, lightMode), + filters: make(map[rpc.ID]*filter), } go api.timeoutLoop() @@ -332,11 +333,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) } - filter := New(api.backend, api.useMipMap) - filter.SetBeginBlock(crit.FromBlock.Int64()) - filter.SetEndBlock(crit.ToBlock.Int64()) - filter.SetAddresses(crit.Addresses) - filter.SetTopics(crit.Topics) + filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics) logs, err := filter.Find(ctx) return returnLogs(logs), err @@ -372,19 +369,18 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty return nil, fmt.Errorf("filter not found") } - filter := New(api.backend, api.useMipMap) + var begin, end int64 if f.crit.FromBlock != nil { - filter.SetBeginBlock(f.crit.FromBlock.Int64()) + begin = f.crit.FromBlock.Int64() } else { - filter.SetBeginBlock(rpc.LatestBlockNumber.Int64()) + begin = rpc.LatestBlockNumber.Int64() } if f.crit.ToBlock != nil { - filter.SetEndBlock(f.crit.ToBlock.Int64()) + end = f.crit.ToBlock.Int64() } else { - filter.SetEndBlock(rpc.LatestBlockNumber.Int64()) + end = rpc.LatestBlockNumber.Int64() } - filter.SetAddresses(f.crit.Addresses) - filter.SetTopics(f.crit.Topics) + filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) logs, err := filter.Find(ctx) if err != nil { diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go new file mode 100644 index 0000000000..2487bc0eb3 --- /dev/null +++ b/eth/filters/bench_test.go @@ -0,0 +1,237 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filters + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/node" + "github.com/golang/snappy" +) + +func BenchmarkBloomBits512(b *testing.B) { + benchmarkBloomBitsForSize(b, 512) +} + +func BenchmarkBloomBits1k(b *testing.B) { + benchmarkBloomBitsForSize(b, 1024) +} + +func BenchmarkBloomBits2k(b *testing.B) { + benchmarkBloomBitsForSize(b, 2048) +} + +func BenchmarkBloomBits4k(b *testing.B) { + benchmarkBloomBitsForSize(b, 4096) +} + +func BenchmarkBloomBits8k(b *testing.B) { + benchmarkBloomBitsForSize(b, 8192) +} + +func BenchmarkBloomBits16k(b *testing.B) { + benchmarkBloomBitsForSize(b, 16384) +} + +func BenchmarkBloomBits32k(b *testing.B) { + benchmarkBloomBitsForSize(b, 32768) +} + +func benchmarkBloomBitsForSize(b *testing.B, sectionSize uint64) { + benchmarkBloomBits(b, sectionSize, 0) + benchmarkBloomBits(b, sectionSize, 1) + benchmarkBloomBits(b, sectionSize, 2) +} + +const benchFilterCnt = 2000 + +func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { + benchDataDir := node.DefaultDataDir() + "/geth/chaindata" + fmt.Println("Running bloombits benchmark section size:", sectionSize, " compression method:", comp) + + var ( + compressFn func([]byte) []byte + decompressFn func([]byte, int) ([]byte, error) + ) + switch comp { + case 0: + // no compression + compressFn = func(data []byte) []byte { + return data + } + decompressFn = func(data []byte, target int) ([]byte, error) { + if len(data) != target { + panic(nil) + } + return data, nil + } + case 1: + // bitutil/compress.go + compressFn = bitutil.CompressBytes + decompressFn = bitutil.DecompressBytes + case 2: + // go snappy + compressFn = func(data []byte) []byte { + return snappy.Encode(nil, data) + } + decompressFn = func(data []byte, target int) ([]byte, error) { + decomp, err := snappy.Decode(nil, data) + if err != nil || len(decomp) != target { + panic(err) + } + return decomp, nil + } + } + + db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", benchDataDir, err) + } + head := core.GetHeadBlockHash(db) + if head == (common.Hash{}) { + b.Fatalf("chain data not found at %v", benchDataDir) + } + + clearBloomBits(db) + fmt.Println("Generating bloombits data...") + headNum := core.GetBlockNumber(db, head) + if headNum < sectionSize+512 { + b.Fatalf("not enough blocks for running a benchmark") + } + + start := time.Now() + cnt := (headNum - 512) / sectionSize + var dataSize, compSize uint64 + for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { + bc := bloombits.NewBloomBitsCreator(sectionSize) + var header *types.Header + for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { + hash := core.GetCanonicalHash(db, i) + header = core.GetHeader(db, hash, i) + if header == nil { + b.Fatalf("Error creating bloomBits data") + } + bc.AddHeaderBloom(header.Bloom) + } + sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) + for i := 0; i < bloombits.BloomLength; i++ { + data := bc.GetBitVector(uint(i)) + comp := compressFn(data) + dataSize += uint64(len(data)) + compSize += uint64(len(comp)) + core.StoreBloomBits(db, uint64(i), sectionIdx, sectionHead, comp) + } + //if sectionIdx%50 == 0 { + // fmt.Println(" section", sectionIdx, "/", cnt) + //} + } + + d := time.Since(start) + fmt.Println("Finished generating bloombits data") + fmt.Println(" ", d, "total ", d/time.Duration(cnt*sectionSize), "per block") + fmt.Println(" data size:", dataSize, " compressed size:", compSize, " compression ratio:", float64(compSize)/float64(dataSize)) + + fmt.Println("Running filter benchmarks...") + start = time.Now() + mux := new(event.TypeMux) + var backend *testBackend + + for i := 0; i < benchFilterCnt; i++ { + if i%20 == 0 { + db.Close() + db, _ = ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + backend = &testBackend{mux, db, cnt, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + } + var addr common.Address + addr[0] = byte(i) + addr[1] = byte(i / 256) + filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + filter.decompress = decompressFn + if _, err := filter.Find(context.Background()); err != nil { + b.Error("filter.Find error:", err) + } + } + d = time.Since(start) + fmt.Println("Finished running filter benchmarks") + fmt.Println(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") + db.Close() +} + +func forEachKey(db ethdb.Database, startPrefix, endPrefix []byte, fn func(key []byte)) { + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(startPrefix) + for it.Valid() { + key := it.Key() + cmpLen := len(key) + if len(endPrefix) < cmpLen { + cmpLen = len(endPrefix) + } + if bytes.Compare(key[:cmpLen], endPrefix) == 1 { + break + } + fn(common.CopyBytes(key)) + it.Next() + } + it.Release() +} + +var bloomBitsPrefix = []byte("bloomBits-") + +func clearBloomBits(db ethdb.Database) { + fmt.Println("Clearing bloombits data...") + forEachKey(db, bloomBitsPrefix, bloomBitsPrefix, func(key []byte) { + db.Delete(key) + }) +} + +func BenchmarkNoBloomBits(b *testing.B) { + benchDataDir := node.DefaultDataDir() + "/geth/chaindata" + fmt.Println("Running benchmark without bloombits") + db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", benchDataDir, err) + } + head := core.GetHeadBlockHash(db) + if head == (common.Hash{}) { + b.Fatalf("chain data not found at %v", benchDataDir) + } + headNum := core.GetBlockNumber(db, head) + + clearBloomBits(db) + + fmt.Println("Running filter benchmarks...") + start := time.Now() + mux := new(event.TypeMux) + backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + filter := New(backend, 0, int64(headNum), []common.Address{common.Address{}}, nil) + filter.Find(context.Background()) + d := time.Since(start) + fmt.Println("Finished running filter benchmarks") + fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks") + db.Close() +} diff --git a/eth/filters/filter.go b/eth/filters/filter.go index f848bc6af9..ea9ccf2f92 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -18,11 +18,14 @@ package filters import ( "context" - "math" "math/big" + "sync" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -34,58 +37,51 @@ type Backend interface { EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + BloomBitsSections() uint64 + BloomBitsConfig() BloomConfig SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) +} + +type BloomConfig struct { + SectionSize uint64 + MaxRequestLen int + MaxRequestWait time.Duration } // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend - useMipMap bool + backend Backend + bloomBitsConfig BloomConfig db ethdb.Database begin, end int64 addresses []common.Address topics [][]common.Hash + + decompress func([]byte, int) ([]byte, error) + matcher *bloombits.Matcher } // New creates a new filter which uses a bloom filter on blocks to figure out whether // a particular block is interesting or not. -// MipMaps allow past blocks to be searched much more efficiently, but are not available -// to light clients. -func New(backend Backend, useMipMap bool) *Filter { +func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { return &Filter{ - backend: backend, - useMipMap: useMipMap, - db: backend.ChainDb(), + backend: backend, + begin: begin, + end: end, + addresses: addresses, + topics: topics, + bloomBitsConfig: backend.BloomBitsConfig(), + db: backend.ChainDb(), + matcher: bloombits.NewMatcher(backend.BloomBitsConfig().SectionSize, addresses, topics), + decompress: bitutil.DecompressBytes, } } -// SetBeginBlock sets the earliest block for filtering. -// -1 = latest block (i.e., the current block) -// hash = particular hash from-to -func (f *Filter) SetBeginBlock(begin int64) { - f.begin = begin -} - -// SetEndBlock sets the latest block for filtering. -func (f *Filter) SetEndBlock(end int64) { - f.end = end -} - -// SetAddresses matches only logs that are generated from addresses that are included -// in the given addresses. -func (f *Filter) SetAddresses(addr []common.Address) { - f.addresses = addr -} - -// SetTopics matches only logs that have topics matching the given topics. -func (f *Filter) SetTopics(topics [][]common.Hash) { - f.topics = topics -} - // FindOnce searches the blockchain for matching log entries, returning // all matching entries from the first block that contains matches, // updating the start point of the filter accordingly. If no results are @@ -106,18 +102,9 @@ func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) { endBlockNo = headBlockNumber } - // if no addresses are present we can't make use of fast search which - // uses the mipmap bloom filters to check for fast inclusion and uses - // higher range probability in order to ensure at least a false positive - if !f.useMipMap || len(f.addresses) == 0 { - logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo) - f.begin = int64(blockNumber + 1) - return logs, err - } - - logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0) + logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo) f.begin = int64(blockNumber + 1) - return logs, nil + return logs, err } // Run filters logs with the current parameters set @@ -131,43 +118,134 @@ func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) { } } -func (f *Filter) mipFind(start, end uint64, depth int) (logs []*types.Log, blockNumber uint64) { - level := core.MIPMapLevels[depth] - // normalise numerator so we can work in level specific batches and - // work with the proper range checks - for num := start / level * level; num <= end; num += level { - // find addresses in bloom filters - bloom := core.GetMipmapBloom(f.db, num, level) - // Don't bother checking the first time through the loop - we're probably picking - // up where a previous run left off. - first := true - for _, addr := range f.addresses { - if first || bloom.TestBytes(addr[:]) { - first = false - // range check normalised values and make sure that - // we're resolving the correct range instead of the - // normalised values. - start := uint64(math.Max(float64(num), float64(start))) - end := uint64(math.Min(float64(num+level-1), float64(end))) - if depth+1 == len(core.MIPMapLevels) { - l, blockNumber, _ := f.getLogs(context.Background(), start, end) - if len(l) > 0 { - return l, blockNumber +// nextRequest returns the next request to retrieve for the bloombits matcher +func (f *Filter) nextRequest() (bloombits uint, sections []uint64) { + bloomIndex, ok := f.matcher.AllocSectionQueue() + if !ok { + return 0, nil + } + if f.bloomBitsConfig.MaxRequestWait > 0 && + (f.bloomBitsConfig.MaxRequestLen <= 1 || // SectionCount is always greater than zero after a successful alloc + f.matcher.SectionCount(bloomIndex) < f.bloomBitsConfig.MaxRequestLen) { + time.Sleep(f.bloomBitsConfig.MaxRequestWait) + } + return bloomIndex, f.matcher.FetchSections(bloomIndex, f.bloomBitsConfig.MaxRequestLen) +} + +// serveMatcher serves the bloombits matcher by fetching the requested vectors +// through the filter backend +func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}, wg *sync.WaitGroup) chan error { + errChn := make(chan error, 1) + wg.Add(10) + for i := 0; i < 10; i++ { + go func(i int) { + defer wg.Done() + + for { + b, s := f.nextRequest() + if s == nil { + return + } + data, err := f.backend.GetBloomBits(ctx, uint64(b), s) + if err != nil { + select { + case errChn <- err: + case <-stop: } - } else { - l, blockNumber := f.mipFind(start, end, depth+1) - if len(l) > 0 { - return l, blockNumber + return + } + decomp := make([][]byte, len(data)) + for i, d := range data { + var err error + if decomp[i], err = f.decompress(d, int(f.bloomBitsConfig.SectionSize/8)); err != nil { + select { + case errChn <- err: + case <-stop: + } + return } } + f.matcher.Deliver(b, s, decomp) } - } + }(i) } - return nil, end + return errChn +} + +// checkMatches checks if the receipts belonging to the given header contain any log events that +// match the filter criteria. This function is called when the bloom filter signals a potential match. +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { + // Get the logs of the block + receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + if err != nil { + return nil, err + } + var unfiltered []*types.Log + for _, receipt := range receipts { + unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...) + } + logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + if len(logs) > 0 { + return logs, nil + } + return nil, nil } func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) { + haveBloomBitsBefore := f.backend.BloomBitsSections() * f.bloomBitsConfig.SectionSize + if haveBloomBitsBefore > start { + e := end + if haveBloomBitsBefore <= e { + e = haveBloomBitsBefore - 1 + } + + stop := make(chan struct{}) + var wg sync.WaitGroup + matches := f.matcher.Start(start, e) + errChn := f.serveMatcher(ctx, stop, &wg) + + defer func() { + f.matcher.Stop() + close(stop) + wg.Wait() + }() + + loop: + for { + select { + case i, ok := <-matches: + if !ok { + break loop + } + + blockNumber := rpc.BlockNumber(i) + header, err := f.backend.HeaderByNumber(ctx, blockNumber) + if header == nil || err != nil { + return logs, end, err + } + + logs, err := f.checkMatches(ctx, header) + if err != nil { + return nil, end, err + } + if logs != nil { + return logs, i, nil + } + case err := <-errChn: + return logs, end, err + case <-ctx.Done(): + return nil, end, ctx.Err() + } + } + + if end < haveBloomBitsBefore { + return logs, end, nil + } + start = haveBloomBitsBefore + } + + // search the rest with regular block-by-block bloom filtering for i := start; i <= end; i++ { blockNumber := rpc.BlockNumber(i) header, err := f.backend.HeaderByNumber(ctx, blockNumber) @@ -178,18 +256,12 @@ func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types. // Use bloom filtering to see if this block is interesting given the // current parameters if f.bloomFilter(header.Bloom) { - // Get the logs of the block - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + logs, err := f.checkMatches(ctx, header) if err != nil { return nil, end, err } - var unfiltered []*types.Log - for _, receipt := range receipts { - unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) - if len(logs) > 0 { - return logs, uint64(blockNumber), nil + if logs != nil { + return logs, i, nil } } } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index fcc888b8cf..140fad555c 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -36,6 +36,7 @@ import ( type testBackend struct { mux *event.TypeMux db ethdb.Database + sections uint64 txFeed *event.Feed rmLogsFeed *event.Feed logsFeed *event.Feed @@ -84,6 +85,31 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } +func (b *testBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { + results := make([][]byte, len(sectionIdxList)) + var err error + for i, sectionIdx := range sectionIdxList { + sectionHead := core.GetCanonicalHash(b.db, (sectionIdx+1)*testBloomBitsSection-1) + results[i], err = core.GetBloomBits(b.db, bitIdx, sectionIdx, sectionHead) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (b *testBackend) BloomBitsSections() uint64 { + return b.sections +} + +func (b *testBackend) BloomBitsConfig() BloomConfig { + return BloomConfig{ + SectionSize: testBloomBitsSection, + MaxRequestLen: 16, + MaxRequestWait: 0, + } +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -99,8 +125,8 @@ func TestBlockSubscription(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false, 0) genesis = new(core.Genesis).MustCommit(db) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {}) chainEvents = []core.ChainEvent{} @@ -156,8 +182,8 @@ func TestPendingTxFilter(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false, 0) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), new(big.Int), new(big.Int), nil), @@ -219,8 +245,8 @@ func TestLogFilterCreation(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false, 0) testCases = []struct { crit FilterCriteria @@ -268,8 +294,8 @@ func TestInvalidLogFilterCreation(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false, 0) ) // different situations where log filter creation should fail. @@ -298,8 +324,8 @@ func TestLogFilter(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false, 0) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -415,8 +441,8 @@ func TestPendingLogsSubscription(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false, 0) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index cf508a218d..f1c6481d7e 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -32,6 +32,8 @@ import ( "github.com/ethereum/go-ethereum/params" ) +const testBloomBitsSection = 4096 + func makeReceipt(addr common.Address) *types.Receipt { receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -41,8 +43,8 @@ func makeReceipt(addr common.Address) *types.Receipt { return receipt } -func BenchmarkMipmaps(b *testing.B) { - dir, err := ioutil.TempDir("", "mipmap") +func BenchmarkFilters(b *testing.B) { + dir, err := ioutil.TempDir("", "filtertest") if err != nil { b.Fatal(err) } @@ -55,7 +57,7 @@ func BenchmarkMipmaps(b *testing.B) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) @@ -66,27 +68,21 @@ func BenchmarkMipmaps(b *testing.B) { genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 100010, func(i int, gen *core.BlockGen) { - var receipts types.Receipts switch i { case 2403: receipt := makeReceipt(addr1) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) case 1034: receipt := makeReceipt(addr2) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) case 34: receipt := makeReceipt(addr3) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) case 99999: receipt := makeReceipt(addr4) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) } - core.WriteMipmapBloom(db, uint64(i+1), receipts) }) for i, block := range chain { core.WriteBlock(db, block) @@ -102,10 +98,7 @@ func BenchmarkMipmaps(b *testing.B) { } b.ResetTimer() - filter := New(backend, true) - filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { logs, _ := filter.Find(context.Background()) @@ -116,7 +109,7 @@ func BenchmarkMipmaps(b *testing.B) { } func TestFilters(t *testing.T) { - dir, err := ioutil.TempDir("", "mipmap") + dir, err := ioutil.TempDir("", "filtertest") if err != nil { t.Fatal(err) } @@ -129,7 +122,7 @@ func TestFilters(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -142,7 +135,6 @@ func TestFilters(t *testing.T) { genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 1000, func(i int, gen *core.BlockGen) { - var receipts types.Receipts switch i { case 1: receipt := types.NewReceipt(nil, false, new(big.Int)) @@ -153,7 +145,6 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} case 2: receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -163,7 +154,6 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} case 998: receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -173,7 +163,6 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} case 999: receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -183,12 +172,7 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} } - // i is used as block number for the writes but since the i - // starts at 0 and block 0 (genesis) is already present increment - // by one - core.WriteMipmapBloom(db, uint64(i+1), receipts) }) for i, block := range chain { core.WriteBlock(db, block) @@ -203,22 +187,14 @@ func TestFilters(t *testing.T) { } } - filter := New(backend, true) - filter.SetAddresses([]common.Address{addr}) - filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) logs, _ := filter.Find(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = New(backend, true) - filter.SetAddresses([]common.Address{addr}) - filter.SetTopics([][]common.Hash{{hash3}}) - filter.SetBeginBlock(900) - filter.SetEndBlock(999) + filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Find(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -227,11 +203,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(backend, true) - filter.SetAddresses([]common.Address{addr}) - filter.SetTopics([][]common.Hash{{hash3}}) - filter.SetBeginBlock(990) - filter.SetEndBlock(-1) + filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Find(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -240,10 +212,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(backend, true) - filter.SetTopics([][]common.Hash{{hash1, hash2}}) - filter.SetBeginBlock(1) - filter.SetEndBlock(10) + filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) logs, _ = filter.Find(context.Background()) if len(logs) != 2 { @@ -251,10 +220,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = New(backend, true) - filter.SetTopics([][]common.Hash{{failHash}}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}}) logs, _ = filter.Find(context.Background()) if len(logs) != 0 { @@ -262,20 +228,14 @@ func TestFilters(t *testing.T) { } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = New(backend, true) - filter.SetAddresses([]common.Address{failAddr}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter = New(backend, 0, -1, []common.Address{failAddr}, nil) logs, _ = filter.Find(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } - filter = New(backend, true) - filter.SetTopics([][]common.Hash{{failHash}, {hash1}}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) logs, _ = filter.Find(context.Background()) if len(logs) != 0 { diff --git a/eth/handler.go b/eth/handler.go index 28ae208c06..aadd771df5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -49,6 +49,8 @@ const ( // txChanSize is the size of channel listening to TxPreEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 + + bloomBitsSection = 4096 ) var ( @@ -92,6 +94,8 @@ type ProtocolManager struct { quitSync chan struct{} noMorePeers chan struct{} + lesServer LesServer + // wait group is used for graceful shutdowns during downloading // and processing wg sync.WaitGroup @@ -114,6 +118,7 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } + // Figure out whether to allow fast sync or not if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 { log.Warn("Blockchain not empty, fast sync disabled") diff --git a/les/api_backend.go b/les/api_backend.go index 1323e88644..c2ba270283 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -19,6 +19,7 @@ package les import ( "context" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -171,3 +173,19 @@ func (b *LesApiBackend) EventMux() *event.TypeMux { func (b *LesApiBackend) AccountManager() *accounts.Manager { return b.eth.accountManager } + +func (b *LesApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { + return nil, nil // implemented in a subsequent PR +} + +func (b *LesApiBackend) BloomBitsSections() uint64 { + return 0 +} + +func (b *LesApiBackend) BloomBitsConfig() filters.BloomConfig { + return filters.BloomConfig{ + SectionSize: 32768, + MaxRequestLen: 16, + MaxRequestWait: time.Microsecond * 100, + } +} diff --git a/les/backend.go b/les/backend.go index 4c33417c03..a3670b5ac7 100644 --- a/les/backend.go +++ b/les/backend.go @@ -169,7 +169,7 @@ func (s *LightEthereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, true), + Service: filters.NewPublicFilterAPI(s.ApiBackend, true, 0), Public: true, }, { Namespace: "net", diff --git a/miner/worker.go b/miner/worker.go index 5bac5d6e85..e1154ac068 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -324,8 +324,6 @@ func (self *worker) wait() { if stat == core.CanonStatTy { // This puts transactions in a extra db for rpc core.WriteTxLookupEntries(self.chainDb, block) - // Write map map bloom filters - core.WriteMipmapBloom(self.chainDb, block.NumberU64(), work.receipts) // implicit by posting ChainHeadEvent mustCommitNewWork = false } From f585f9eee8cb18423c23fe8b517b5b4cbe3b3755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 29 Aug 2017 14:13:11 +0300 Subject: [PATCH 17/45] core, eth: clean up bloom filtering, add some tests --- core/bloombits/doc.go | 18 + core/bloombits/fetcher_test.go | 101 ---- core/bloombits/generator.go | 84 +++ core/bloombits/generator_test.go | 60 ++ core/bloombits/matcher.go | 936 ++++++++++++++++-------------- core/bloombits/matcher_test.go | 347 ++++++----- core/bloombits/scheduler.go | 181 ++++++ core/bloombits/scheduler_test.go | 105 ++++ core/bloombits/utils.go | 63 -- core/chain_indexer.go | 61 +- core/chain_indexer_test.go | 3 +- core/database_util.go | 158 ++--- core/types/bloom9.go | 28 +- eth/api_backend.go | 29 +- eth/backend.go | 21 +- eth/bloombits.go | 142 +++++ eth/db_upgrade.go | 38 -- eth/filters/api.go | 49 +- eth/filters/bench_test.go | 82 +-- eth/filters/filter.go | 276 ++++----- eth/filters/filter_system_test.go | 64 +- eth/filters/filter_test.go | 18 +- eth/handler.go | 5 - les/api_backend.go | 18 +- les/backend.go | 2 +- params/network_params.go | 26 + 26 files changed, 1650 insertions(+), 1265 deletions(-) create mode 100644 core/bloombits/doc.go delete mode 100644 core/bloombits/fetcher_test.go create mode 100644 core/bloombits/generator.go create mode 100644 core/bloombits/generator_test.go create mode 100644 core/bloombits/scheduler.go create mode 100644 core/bloombits/scheduler_test.go delete mode 100644 core/bloombits/utils.go create mode 100644 eth/bloombits.go create mode 100644 params/network_params.go diff --git a/core/bloombits/doc.go b/core/bloombits/doc.go new file mode 100644 index 0000000000..3d159e74f7 --- /dev/null +++ b/core/bloombits/doc.go @@ -0,0 +1,18 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package bloombits implements bloom filtering on batches of data. +package bloombits diff --git a/core/bloombits/fetcher_test.go b/core/bloombits/fetcher_test.go deleted file mode 100644 index 9c229cf8da..0000000000 --- a/core/bloombits/fetcher_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . -package bloombits - -import ( - "bytes" - "encoding/binary" - "math/rand" - "sync" - "sync/atomic" - "testing" - "time" -) - -const testFetcherReqCount = 5000 - -func fetcherTestVector(b uint, s uint64) []byte { - r := make([]byte, 10) - binary.BigEndian.PutUint16(r[0:2], uint16(b)) - binary.BigEndian.PutUint64(r[2:10], s) - return r -} - -func TestFetcher(t *testing.T) { - testFetcher(t, 1) -} - -func TestFetcherMultipleReaders(t *testing.T) { - testFetcher(t, 10) -} - -func testFetcher(t *testing.T, cnt int) { - f := &fetcher{ - requestMap: make(map[uint64]fetchRequest), - } - distCh := make(chan distRequest, channelCap) - stop := make(chan struct{}) - var reqCount uint32 - - for i := 0; i < 10; i++ { - go func() { - for { - req, ok := <-distCh - if !ok { - return - } - time.Sleep(time.Duration(rand.Intn(100000))) - atomic.AddUint32(&reqCount, 1) - f.deliver([]uint64{req.sectionIndex}, [][]byte{fetcherTestVector(req.bloomIndex, req.sectionIndex)}) - } - }() - } - - var wg, wg2 sync.WaitGroup - for cc := 0; cc < cnt; cc++ { - wg.Add(1) - in := make(chan uint64, channelCap) - out := f.fetch(in, distCh, stop, &wg2) - - time.Sleep(time.Millisecond * 10 * time.Duration(cc)) - go func() { - for i := uint64(0); i < testFetcherReqCount; i++ { - in <- i - } - }() - - go func() { - for i := uint64(0); i < testFetcherReqCount; i++ { - bv := <-out - if !bytes.Equal(bv, fetcherTestVector(0, i)) { - if len(bv) != 10 { - t.Errorf("Vector #%d length is %d, expected 10", i, len(bv)) - } else { - j := binary.BigEndian.Uint64(bv[2:10]) - t.Errorf("Expected vector #%d, fetched #%d", i, j) - } - } - } - wg.Done() - }() - } - - wg.Wait() - close(stop) - if reqCount != testFetcherReqCount { - t.Errorf("Request count mismatch: expected %v, got %v", testFetcherReqCount, reqCount) - } -} diff --git a/core/bloombits/generator.go b/core/bloombits/generator.go new file mode 100644 index 0000000000..04a7f5146c --- /dev/null +++ b/core/bloombits/generator.go @@ -0,0 +1,84 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "errors" + + "github.com/ethereum/go-ethereum/core/types" +) + +// errSectionOutOfBounds is returned if the user tried to add more bloom filters +// to the batch than available space, or if tries to retrieve above the capacity, +var errSectionOutOfBounds = errors.New("section out of bounds") + +// Generator takes a number of bloom filters and generates the rotated bloom bits +// to be used for batched filtering. +type Generator struct { + blooms [types.BloomBitLength][]byte // Rotated blooms for per-bit matching + sections uint // Number of sections to batch together + nextBit uint // Next bit to set when adding a bloom +} + +// NewGenerator creates a rotated bloom generator that can iteratively fill a +// batched bloom filter's bits. +func NewGenerator(sections uint) (*Generator, error) { + if sections%8 != 0 { + return nil, errors.New("section count not multiple of 8") + } + b := &Generator{sections: sections} + for i := 0; i < types.BloomBitLength; i++ { + b.blooms[i] = make([]byte, sections/8) + } + return b, nil +} + +// AddBloom takes a single bloom filter and sets the corresponding bit column +// in memory accordingly. +func (b *Generator) AddBloom(bloom types.Bloom) error { + // Make sure we're not adding more bloom filters than our capacity + if b.nextBit >= b.sections { + return errSectionOutOfBounds + } + // Rotate the bloom and insert into our collection + byteMask := b.nextBit / 8 + bitMask := byte(1) << byte(7-b.nextBit%8) + + for i := 0; i < types.BloomBitLength; i++ { + bloomByteMask := types.BloomByteLength - 1 - i/8 + bloomBitMask := byte(1) << byte(i%8) + + if (bloom[bloomByteMask] & bloomBitMask) != 0 { + b.blooms[i][byteMask] |= bitMask + } + } + b.nextBit++ + + return nil +} + +// Bitset returns the bit vector belonging to the given bit index after all +// blooms have been added. +func (b *Generator) Bitset(idx uint) ([]byte, error) { + if b.nextBit != b.sections { + return nil, errors.New("bloom not fully generated yet") + } + if idx >= b.sections { + return nil, errSectionOutOfBounds + } + return b.blooms[idx], nil +} diff --git a/core/bloombits/generator_test.go b/core/bloombits/generator_test.go new file mode 100644 index 0000000000..f4aa9551c2 --- /dev/null +++ b/core/bloombits/generator_test.go @@ -0,0 +1,60 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/core/types" +) + +// Tests that batched bloom bits are correctly rotated from the input bloom +// filters. +func TestGenerator(t *testing.T) { + // Generate the input and the rotated output + var input, output [types.BloomBitLength][types.BloomByteLength]byte + + for i := 0; i < types.BloomBitLength; i++ { + for j := 0; j < types.BloomBitLength; j++ { + bit := byte(rand.Int() % 2) + + input[i][j/8] |= bit << byte(7-j%8) + output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8) + } + } + // Crunch the input through the generator and verify the result + gen, err := NewGenerator(types.BloomBitLength) + if err != nil { + t.Fatalf("failed to create bloombit generator: %v", err) + } + for i, bloom := range input { + if err := gen.AddBloom(bloom); err != nil { + t.Fatalf("bloom %d: failed to add: %v", i, err) + } + } + for i, want := range output { + have, err := gen.Bitset(uint(i)) + if err != nil { + t.Fatalf("output %d: failed to retrieve bits: %v", i, err) + } + if !bytes.Equal(have, want[:]) { + t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want) + } + } +} diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index 5a7df6b1c9..e365fd6d04 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -13,327 +13,350 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . + package bloombits import ( + "errors" + "math" + "sort" "sync" + "sync/atomic" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" ) -const channelCap = 100 +// bloomIndexes represents the bit indexes inside the bloom filter that belong +// to some key. +type bloomIndexes [3]uint -// fetcher handles bit vector retrieval pipelines for a single bit index -type fetcher struct { - bloomIndex uint - requestMap map[uint64]fetchRequest - requestLock sync.RWMutex -} +// calcBloomIndexes returns the bloom filter bit indexes belonging to the given key. +func calcBloomIndexes(b []byte) bloomIndexes { + b = crypto.Keccak256(b) -// fetchRequest represents the state of a bit vector requested from a fetcher. When a distRequest has been sent to the distributor but -// the data has not been delivered yet, queued is true. When delivered, it is stored in the data field and the delivered channel is closed. -type fetchRequest struct { - data []byte - queued bool - delivered chan struct{} -} - -// distRequest is sent by the fetcher to the distributor which groups and prioritizes these requests. -type distRequest struct { - bloomIndex uint - sectionIndex uint64 -} - -// fetch creates a retrieval pipeline, receiving section indexes from sectionCh and returning the results -// in the same order through the returned channel. Multiple fetch instances of the same fetcher are allowed -// to run in parallel, in case the same bit index appears multiple times in the filter structure. Each section -// is requested only once, requests are sent to the request distributor (part of Matcher) through distCh. -func (f *fetcher) fetch(sectionCh chan uint64, distCh chan distRequest, stop chan struct{}, wg *sync.WaitGroup) chan []byte { - dataCh := make(chan []byte, channelCap) - returnCh := make(chan uint64, channelCap) - wg.Add(2) - - go func() { - defer wg.Done() - defer close(returnCh) - - for { - select { - case <-stop: - return - case idx, ok := <-sectionCh: - if !ok { - return - } - - req := false - f.requestLock.Lock() - r := f.requestMap[idx] - if r.data == nil { - req = !r.queued - r.queued = true - if r.delivered == nil { - r.delivered = make(chan struct{}) - } - f.requestMap[idx] = r - } - f.requestLock.Unlock() - if req { - distCh <- distRequest{bloomIndex: f.bloomIndex, sectionIndex: idx} // success is guaranteed, distibuteRequests shuts down after fetch - } - select { - case <-stop: - return - case returnCh <- idx: - } - } - } - }() - - go func() { - defer wg.Done() - defer close(dataCh) - - for { - select { - case <-stop: - return - case idx, ok := <-returnCh: - if !ok { - return - } - - f.requestLock.RLock() - r := f.requestMap[idx] - f.requestLock.RUnlock() - - if r.data == nil { - select { - case <-stop: - return - case <-r.delivered: - f.requestLock.RLock() - r = f.requestMap[idx] - f.requestLock.RUnlock() - } - } - select { - case <-stop: - return - case dataCh <- r.data: - } - } - } - }() - - return dataCh -} - -// deliver is called by the request distributor when a reply to a request has -// arrived -func (f *fetcher) deliver(sectionIdxList []uint64, data [][]byte) { - f.requestLock.Lock() - defer f.requestLock.Unlock() - - for i, sectionIdx := range sectionIdxList { - r := f.requestMap[sectionIdx] - if r.data != nil { - panic("BloomBits section data delivered twice") - } - r.data = data[i] - close(r.delivered) - f.requestMap[sectionIdx] = r + var idxs bloomIndexes + for i := 0; i < len(idxs); i++ { + idxs[i] = (uint(b[2*i])<<8)&2047 + uint(b[2*i+1]) } + return idxs } -// Matcher is a pipelined structure of fetchers and logic matchers which perform -// binary AND/OR operations on the bitstreams, finally creating a stream of potential matches. +// partialMatches with a non-nil vector represents a section in which some sub- +// matchers have already found potential matches. Subsequent sub-matchers will +// binary AND their matches with this vector. If vector is nil, it represents a +// section to be processed by the first sub-matcher. +type partialMatches struct { + section uint64 + bitset []byte +} + +// Retrieval represents a request for retrieval task assignments for a given +// bit with the given number of fetch elements, or a response for such a request. +// It can also have the actual results set to be used as a delivery data struct. +type Retrieval struct { + Bit uint + Sections []uint64 + Bitsets [][]byte +} + +// Matcher is a pipelined system of schedulers and logic matchers which perform +// binary AND/OR operations on the bit-streams, creating a stream of potential +// blocks to inspect for data content. type Matcher struct { - addresses []types.BloomIndexList - topics [][]types.BloomIndexList - fetchers map[uint]*fetcher - sectionSize uint64 + sectionSize uint64 // Size of the data batches to filter on - distCh chan distRequest - reqs map[uint][]uint64 - freeQueues map[uint]struct{} - allocQueue []chan uint - running bool - stop chan struct{} - lock sync.Mutex - wg, distWg sync.WaitGroup + addresses []bloomIndexes // Addresses the system is filtering for + topics [][]bloomIndexes // Topics the system is filtering for + schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits + + retrievers chan chan uint // Retriever processes waiting for bit allocations + counters chan chan uint // Retriever processes waiting for task count reports + retrievals chan chan *Retrieval // Retriever processes waiting for task allocations + deliveries chan *Retrieval // Retriever processes waiting for task response deliveries + + running uint32 // Atomic flag whether a session is live or not } -// NewMatcher creates a new Matcher instance +// NewMatcher creates a new pipeline for retrieving bloom bit streams and doing +// address and topic filtering on them. func NewMatcher(sectionSize uint64, addresses []common.Address, topics [][]common.Hash) *Matcher { m := &Matcher{ - fetchers: make(map[uint]*fetcher), - reqs: make(map[uint][]uint64), - freeQueues: make(map[uint]struct{}), - distCh: make(chan distRequest, channelCap), sectionSize: sectionSize, + schedulers: make(map[uint]*scheduler), + retrievers: make(chan chan uint), + counters: make(chan chan uint), + retrievals: make(chan chan *Retrieval), + deliveries: make(chan *Retrieval), } m.setAddresses(addresses) m.setTopics(topics) return m } -// setAddresses matches only logs that are generated from addresses that are included -// in the given addresses. +// setAddresses configures the matcher to only return logs that are generated +// from addresses that are included in the given list. func (m *Matcher) setAddresses(addresses []common.Address) { - m.addresses = make([]types.BloomIndexList, len(addresses)) + // Calculate the bloom bit indexes for the addresses we're interested in + m.addresses = make([]bloomIndexes, len(addresses)) for i, address := range addresses { - m.addresses[i] = types.BloomIndexes(address.Bytes()) + m.addresses[i] = calcBloomIndexes(address.Bytes()) } - + // For every bit, create a scheduler to load/download the bit vectors for _, bloomIndexList := range m.addresses { for _, bloomIndex := range bloomIndexList { - m.newFetcher(bloomIndex) + m.addScheduler(bloomIndex) } } } -// setTopics matches only logs that have topics matching the given topics. -func (m *Matcher) setTopics(topics [][]common.Hash) { +// setTopics configures the matcher to only return logs that have topics matching +// the given list. +func (m *Matcher) setTopics(topicsList [][]common.Hash) { + // Calculate the bloom bit indexes for the topics we're interested in m.topics = nil -loop: - for _, topicList := range topics { - t := make([]types.BloomIndexList, len(topicList)) - for i, topic := range topicList { - if (topic == common.Hash{}) { - continue loop - } - t[i] = types.BloomIndexes(topic.Bytes()) - } - m.topics = append(m.topics, t) - } + for _, topics := range topicsList { + bloomBits := make([]bloomIndexes, len(topics)) + for i, topic := range topics { + bloomBits[i] = calcBloomIndexes(topic.Bytes()) + } + m.topics = append(m.topics, bloomBits) + } + // For every bit, create a scheduler to load/download the bit vectors for _, bloomIndexLists := range m.topics { for _, bloomIndexList := range bloomIndexLists { for _, bloomIndex := range bloomIndexList { - m.newFetcher(bloomIndex) + m.addScheduler(bloomIndex) } } } } -// match creates a daisy-chain of sub-matchers, one for the address set and one for each topic set, each -// sub-matcher receiving a section only if the previous ones have all found a potential match in one of -// the blocks of the section, then binary AND-ing its own matches and forwaring the result to the next one -func (m *Matcher) match(processCh chan partialMatches) chan partialMatches { - indexLists := m.topics - if len(m.addresses) > 0 { - indexLists = append([][]types.BloomIndexList{m.addresses}, indexLists...) - } - m.distributeRequests() - - for _, subIndexList := range indexLists { - processCh = m.subMatch(processCh, subIndexList) - } - return processCh -} - -// partialMatches with a non-nil vector represents a section in which some sub-matchers have already -// found potential matches. Subsequent sub-matchers will binary AND their matches with this vector. -// If vector is nil, it represents a section to be processed by the first sub-matcher. -type partialMatches struct { - sectionIndex uint64 - vector []byte -} - -// newFetcher adds a fetcher for the given bit index if it has not existed before -func (m *Matcher) newFetcher(idx uint) { - if _, ok := m.fetchers[idx]; ok { +// addScheduler adds a bit stream retrieval scheduler for the given bit index if +// it has not existed before. If the bit is already selected for filtering, the +// existing scheduler can be used. +func (m *Matcher) addScheduler(idx uint) { + if _, ok := m.schedulers[idx]; ok { return } - f := &fetcher{ - bloomIndex: idx, - requestMap: make(map[uint64]fetchRequest), - } - m.fetchers[idx] = f + m.schedulers[idx] = newScheduler(idx) } -// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then -// binary AND-s the result to the daisy-chain input (processCh) and forwards it to the daisy-chain output. -// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to -// that address/topic, and binary AND-ing those vectors together. -func (m *Matcher) subMatch(processCh chan partialMatches, bloomIndexLists []types.BloomIndexList) chan partialMatches { - // set up fetchers - fetchIndexChannels := make([][3]chan uint64, len(bloomIndexLists)) - fetchDataChannels := make([][3]chan []byte, len(bloomIndexLists)) - for i, bloomIndexList := range bloomIndexLists { - for j, bloomIndex := range bloomIndexList { - fetchIndexChannels[i][j] = make(chan uint64, channelCap) - fetchDataChannels[i][j] = m.fetchers[bloomIndex].fetch(fetchIndexChannels[i][j], m.distCh, m.stop, &m.wg) - } +// Start starts the matching process and returns a stream of bloom matches in +// a given range of blocks. If there are no more matches in the range, the result +// channel is closed. +func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession, error) { + // Make sure we're not creating concurrent sessions + if atomic.SwapUint32(&m.running, 1) == 1 { + return nil, errors.New("matcher already running") } + defer atomic.StoreUint32(&m.running, 0) - fetchedCh := make(chan partialMatches, channelCap) // entries from processCh are forwarded here after fetches have been initiated - resultsCh := make(chan partialMatches, channelCap) + // Initiate a new matching round + session := &MatcherSession{ + matcher: m, + quit: make(chan struct{}), + kill: make(chan struct{}), + } + for _, scheduler := range m.schedulers { + scheduler.reset() + } + sink := m.run(begin, end, cap(results), session) - m.wg.Add(2) - // goroutine for starting retrievals + // Read the output from the result sink and deliver to the user + session.pend.Add(1) go func() { - defer m.wg.Done() + defer session.pend.Done() + defer close(results) for { select { - case <-m.stop: + case <-session.quit: return - case s, ok := <-processCh: - if !ok { - close(fetchedCh) - for _, fetchIndexChs := range fetchIndexChannels { - for _, fetchIndexCh := range fetchIndexChs { - close(fetchIndexCh) - } - } - return - } - for _, fetchIndexChs := range fetchIndexChannels { - for _, fetchIndexCh := range fetchIndexChs { + case res, ok := <-sink: + // New match result found + if !ok { + return + } + // Calculate the first and last blocks of the section + sectionStart := res.section * m.sectionSize + + first := sectionStart + if begin > first { + first = begin + } + last := sectionStart + m.sectionSize - 1 + if end < last { + last = end + } + // Iterate over all the blocks in the section and return the matching ones + for i := first; i <= last; i++ { + // If the bitset is nil, we're a special match-all cornercase + if res.bitset == nil { select { - case <-m.stop: + case <-session.quit: return - case fetchIndexCh <- s.sectionIndex: + case results <- i: + } + continue + } + // Skip the entire byte if no matches are found inside + next := res.bitset[(i-sectionStart)/8] + if next == 0 { + i += 7 + continue + } + // Some bit it set, do the actual submatching + if bit := 7 - i%8; next&(1< 0 { + blooms = append([][]bloomIndexes{m.addresses}, blooms...) + } + next := source + dist := make(chan *request, buffer) + + for _, bloom := range blooms { + next = m.subMatch(next, dist, bloom, session) + } + // Start the request distribution + session.pend.Add(1) + go m.distributor(dist, session) + + return next +} + +// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then +// binary AND-s the result to the daisy-chain input (source) and forwards it to the daisy-chain output. +// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to +// that address/topic, and binary AND-ing those vectors together. +func (m *Matcher) subMatch(source chan *partialMatches, dist chan *request, bloom []bloomIndexes, session *MatcherSession) chan *partialMatches { + // Start the concurrent schedulers for each bit required by the bloom filter + sectionSources := make([][3]chan uint64, len(bloom)) + sectionSinks := make([][3]chan []byte, len(bloom)) + for i, bits := range bloom { + for j, bit := range bits { + sectionSources[i][j] = make(chan uint64, cap(source)) + sectionSinks[i][j] = make(chan []byte, cap(source)) + + m.schedulers[bit].run(sectionSources[i][j], dist, sectionSinks[i][j], session.quit, &session.pend) + } + } + + process := make(chan *partialMatches, cap(source)) // entries from source are forwarded here after fetches have been initiated + results := make(chan *partialMatches, cap(source)) + + session.pend.Add(2) + go func() { + // Tear down the goroutine and terminate all source channels + defer session.pend.Done() + defer close(process) + + defer func() { + for _, bloomSources := range sectionSources { + for _, bitSource := range bloomSources { + close(bitSource) + } + } + }() + // Read sections from the source channel and multiplex into all bit-schedulers + for { + select { + case <-session.quit: + return + + case subres, ok := <-source: + // New subresult from previous link + if !ok { return - case fetchedCh <- s: + } + // Multiplex the section index to all bit-schedulers + for _, bloomSources := range sectionSources { + for _, bitSource := range bloomSources { + select { + case <-session.quit: + return + case bitSource <- subres.section: + } + } + } + // Notify the processor that this section will become available + select { + case <-session.quit: + return + case process <- subres: } } } }() - // goroutine for processing retrieved data go func() { - defer m.wg.Done() + // Tear down the goroutine and terminate the final sink channel + defer session.pend.Done() + defer close(results) + // Read the source notifications and collect the delivered results for { select { - case <-m.stop: + case <-session.quit: return - case s, ok := <-fetchedCh: + + case subres, ok := <-process: + // Notified of a section being retrieved if !ok { - close(resultsCh) return } - + // Gather all the sub-results and merge them together var orVector []byte - for _, fetchDataChs := range fetchDataChannels { + for _, bloomSinks := range sectionSinks { var andVector []byte - for _, fetchDataCh := range fetchDataChs { + for _, bitSink := range bloomSinks { var data []byte select { - case <-m.stop: + case <-session.quit: return - case data = <-fetchDataCh: + case data = <-bitSink: } if andVector == nil { andVector = make([]byte, int(m.sectionSize/8)) @@ -352,228 +375,277 @@ func (m *Matcher) subMatch(processCh chan partialMatches, bloomIndexLists []type if orVector == nil { orVector = make([]byte, int(m.sectionSize/8)) } - if s.vector != nil { - bitutil.ANDBytes(orVector, orVector, s.vector) + if subres.bitset != nil { + bitutil.ANDBytes(orVector, orVector, subres.bitset) } if bitutil.TestBytes(orVector) { select { - case <-m.stop: + case <-session.quit: return - case resultsCh <- partialMatches{s.sectionIndex, orVector}: + case results <- &partialMatches{subres.section, orVector}: } } } } }() - - return resultsCh + return results } -// Start starts the matching process and returns a stream of bloom matches in -// a given range of blocks. -// It returns a results channel immediately and stops if Stop is called or there -// are no more matches in the range (in which case the results channel is closed). -// Start/Stop can be called multiple times for different ranges, in which case already -// delivered bit vectors are not requested again. -func (m *Matcher) Start(begin, end uint64) chan uint64 { - m.stop = make(chan struct{}) - processCh := make(chan partialMatches, channelCap) - resultsCh := make(chan uint64, channelCap) +// distributor receives requests from the schedulers and queues them into a set +// of pending requests, which are assigned to retrievers wanting to fulfil them. +func (m *Matcher) distributor(dist chan *request, session *MatcherSession) { + defer session.pend.Done() - res := m.match(processCh) + var ( + requests = make(map[uint][]uint64) // Per-bit list of section requests, ordered by section number + unallocs = make(map[uint]struct{}) // Bits with pending requests but not allocated to any retriever + retrievers chan chan uint // Waiting retrievers (toggled to nil if unallocs is empty) + ) + var ( + allocs int // Number of active allocations to handle graceful shutdown requests + shutdown = session.quit // Shutdown request channel, will gracefully wait for pending requests + ) - startSection := begin / m.sectionSize - endSection := end / m.sectionSize + // assign is a helper method fo try to assign a pending bit an an actively + // listening servicer, or schedule it up for later when one arrives. + assign := func(bit uint) { + select { + case fetcher := <-m.retrievers: + allocs++ + fetcher <- bit + default: + // No retrievers active, start listening for new ones + retrievers = m.retrievers + unallocs[bit] = struct{}{} + } + } - m.wg.Add(2) - go func() { - defer m.wg.Done() - defer close(processCh) + for { + select { + case <-shutdown: + // Graceful shutdown requested, wait until all pending requests are honoured + if allocs == 0 { + return + } + shutdown = nil - for i := startSection; i <= endSection; i++ { - select { - case processCh <- partialMatches{i, nil}: - case <-m.stop: + case <-session.kill: + // Pending requests not honoured in time, hard terminate + return + + case req := <-dist: + // New retrieval request arrived to be distributed to some fetcher process + queue := requests[req.bit] + index := sort.Search(len(queue), func(i int) bool { return queue[i] >= req.section }) + requests[req.bit] = append(queue[:index], append([]uint64{req.section}, queue[index:]...)...) + + // If it's a new bit and we have waiting fetchers, allocate to them + if len(queue) == 0 { + assign(req.bit) + } + + case fetcher := <-retrievers: + // New retriever arrived, find the lowest section-ed bit to assign + bit, best := uint(0), uint64(math.MaxUint64) + for idx := range unallocs { + if requests[idx][0] < best { + bit, best = idx, requests[idx][0] + } + } + // Stop tracking this bit (and alloc notifications if no more work is available) + delete(unallocs, bit) + if len(unallocs) == 0 { + retrievers = nil + } + allocs++ + fetcher <- bit + + case fetcher := <-m.counters: + // New task count request arrives, return number of items + fetcher <- uint(len(requests[<-fetcher])) + + case fetcher := <-m.retrievals: + // New fetcher waiting for tasks to retrieve, assign + task := <-fetcher + if want := len(task.Sections); want >= len(requests[task.Bit]) { + task.Sections = requests[task.Bit] + delete(requests, task.Bit) + } else { + task.Sections = append(task.Sections[:0], requests[task.Bit][:want]...) + requests[task.Bit] = append(requests[task.Bit][:0], requests[task.Bit][want:]...) + } + fetcher <- task + + // If anything was left unallocated, try to assign to someone else + if len(requests[task.Bit]) > 0 { + assign(task.Bit) + } + + case result := <-m.deliveries: + // New retrieval task response from fetcher, split out missing sections and + // deliver complete ones + var ( + sections = make([]uint64, 0, len(result.Sections)) + bitsets = make([][]byte, 0, len(result.Bitsets)) + missing = make([]uint64, 0, len(result.Sections)) + ) + for i, bitset := range result.Bitsets { + if len(bitset) == 0 { + missing = append(missing, result.Sections[i]) + continue + } + sections = append(sections, result.Sections[i]) + bitsets = append(bitsets, bitset) + } + m.schedulers[result.Bit].deliver(sections, bitsets) + allocs-- + + // Reschedule missing sections and allocate bit if newly available + if len(missing) > 0 { + queue := requests[result.Bit] + for _, section := range missing { + index := sort.Search(len(queue), func(i int) bool { return queue[i] >= section }) + queue = append(queue[:index], append([]uint64{section}, queue[index:]...)...) + } + requests[result.Bit] = queue + + if len(queue) == len(missing) { + assign(result.Bit) + } + } + // If we're in the process of shutting down, terminate + if allocs == 0 && shutdown == nil { return } } - }() - - go func() { - defer m.wg.Done() - defer close(resultsCh) - - for { - select { - case r, ok := <-res: - if !ok { - return - } - sectionStart := r.sectionIndex * m.sectionSize - s := sectionStart - if begin > s { - s = begin - } - e := sectionStart + m.sectionSize - 1 - if end < e { - e = end - } - for i := s; i <= e; i++ { - b := r.vector[(i-sectionStart)/8] - bit := 7 - i%8 - if b != 0 { - if b&(1< queue[i] { - i++ - } - queue = append(queue, 0) - copy(queue[i+1:], queue[i:len(queue)-1]) - queue[i] = r.sectionIndex - m.reqs[r.bloomIndex] = queue - if len(queue) == 1 { - m.freeQueue(r.bloomIndex) - } - m.lock.Unlock() - case <-stopDist: - m.lock.Lock() - for _, ch := range m.allocQueue { - close(ch) - } - m.allocQueue = nil - m.running = false - m.lock.Unlock() - m.distWg.Done() - return - } - } - }() -} - -// freeQueue marks a queue as free if there are no AllocSectionQueue functions -// waiting for allocation. If there is someone waiting, the queue is immediately -// allocated. -func (m *Matcher) freeQueue(bloomIndex uint) { - if len(m.allocQueue) > 0 { - m.allocQueue[0] <- bloomIndex - m.allocQueue = m.allocQueue[1:] - } else { - m.freeQueues[bloomIndex] = struct{}{} } } -// AllocSectionQueue allocates a queue of requested section indexes belonging to the same -// bloom bit index for a client process that can either immediately fetch the contents -// of the queue or wait a little while for more section indexes to be requested. -func (m *Matcher) AllocSectionQueue() (uint, bool) { - m.lock.Lock() - if !m.running { - m.lock.Unlock() +// MatcherSession is returned by a started matcher to be used as a terminator +// for the actively running matching operation. +type MatcherSession struct { + matcher *Matcher + + quit chan struct{} // Quit channel to request pipeline termination + kill chan struct{} // Term channel to signal non-graceful forced shutdown + pend sync.WaitGroup +} + +// Close stops the matching process and waits for all subprocesses to terminate +// before returning. The timeout may be used for graceful shutdown, allowing the +// currently running retrievals to complete before this time. +func (s *MatcherSession) Close(timeout time.Duration) { + // Bail out if the matcher is not running + select { + case <-s.quit: + return + default: + } + // Signal termination and wait for all goroutines to tear down + close(s.quit) + time.AfterFunc(timeout, func() { close(s.kill) }) + s.pend.Wait() +} + +// AllocateRetrieval assigns a bloom bit index to a client process that can either +// immediately reuest and fetch the section contents assigned to this bit or wait +// a little while for more sections to be requested. +func (s *MatcherSession) AllocateRetrieval() (uint, bool) { + fetcher := make(chan uint) + + select { + case <-s.quit: return 0, false + case s.matcher.retrievers <- fetcher: + bit, ok := <-fetcher + return bit, ok } +} - var allocCh chan uint - if len(m.freeQueues) > 0 { - var ( - found bool - bestSection uint64 - bestIndex uint - ) - for bloomIndex, _ := range m.freeQueues { - if !found || m.reqs[bloomIndex][0] < bestSection { - found = true - bestIndex = bloomIndex - bestSection = m.reqs[bloomIndex][0] +// PendingSections returns the number of pending section retrievals belonging to +// the given bloom bit index. +func (s *MatcherSession) PendingSections(bit uint) int { + fetcher := make(chan uint) + + select { + case <-s.quit: + return 0 + case s.matcher.counters <- fetcher: + fetcher <- bit + return int(<-fetcher) + } +} + +// AllocateSections assigns all or part of an already allocated bit-task queue +// to the requesting process. +func (s *MatcherSession) AllocateSections(bit uint, count int) []uint64 { + fetcher := make(chan *Retrieval) + + select { + case <-s.quit: + return nil + case s.matcher.retrievals <- fetcher: + task := &Retrieval{ + Bit: bit, + Sections: make([]uint64, count), + } + fetcher <- task + return (<-fetcher).Sections + } +} + +// DeliverSections delivers a batch of section bit-vectors for a specific bloom +// bit index to be injected into the processing pipeline. +func (s *MatcherSession) DeliverSections(bit uint, sections []uint64, bitsets [][]byte) { + select { + case <-s.kill: + return + case s.matcher.deliveries <- &Retrieval{Bit: bit, Sections: sections, Bitsets: bitsets}: + } +} + +// Multiplex polls the matcher session for rerieval tasks and multiplexes it into +// the reuested retrieval queue to be serviced together with other sessions. +// +// This method will block for the lifetime of the session. Even after termination +// of the session, any request in-flight need to be responded to! Empty responses +// are fine though in that case. +func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan *Retrieval) { + for { + // Allocate a new bloom bit index to retrieve data for, stopping when done + bit, ok := s.AllocateRetrieval() + if !ok { + return + } + // Bit allocated, throttle a bit if we're below our batch limit + if s.PendingSections(bit) < batch { + select { + case <-s.quit: + // Session terminating, we can't meaningfully service, abort + s.AllocateSections(bit, 0) + s.DeliverSections(bit, []uint64{}, [][]byte{}) + return + + case <-time.After(wait): + // Throttling up, fetch whatever's available } } - delete(m.freeQueues, bestIndex) - m.lock.Unlock() - return bestIndex, true - } else { - allocCh = make(chan uint) - m.allocQueue = append(m.allocQueue, allocCh) - } - m.lock.Unlock() + // Allocate as much as we can handle and request servicing + sections := s.AllocateSections(bit, batch) + request := make(chan *Retrieval) - bloomIndex, ok := <-allocCh - return bloomIndex, ok -} + select { + case <-s.quit: + // Session terminating, we can't meaningfully service, abort + s.DeliverSections(bit, sections, make([][]byte, len(sections))) + return -// SectionCount returns the length of the section index queue belonging to the given bloom bit index -func (m *Matcher) SectionCount(bloomIndex uint) int { - m.lock.Lock() - defer m.lock.Unlock() + case mux <- request: + // Retrieval accepted, something must arrive before we're aborting + request <- &Retrieval{Bit: bit, Sections: sections} - return len(m.reqs[bloomIndex]) -} - -// FetchSections fetches all or part of an already allocated queue and deallocates it -func (m *Matcher) FetchSections(bloomIndex uint, maxCount int) []uint64 { - m.lock.Lock() - defer m.lock.Unlock() - - queue := m.reqs[bloomIndex] - if maxCount < len(queue) { - // return only part of the existing queue, mark the rest as free - m.reqs[bloomIndex] = queue[maxCount:] - m.freeQueue(bloomIndex) - return queue[:maxCount] - } else { - // return the entire queue - delete(m.reqs, bloomIndex) - return queue + result := <-request + s.DeliverSections(result.Bit, result.Sections, result.Bitsets) + } } } - -// Deliver delivers a bit vector to the appropriate fetcher. -// It is possible to deliver data even after Stop has been called. Once a vector has been -// requested, the matcher will keep waiting for delivery. -func (m *Matcher) Deliver(bloomIndex uint, sectionIdxList []uint64, data [][]byte) { - m.fetchers[bloomIndex].deliver(sectionIdxList, data) -} diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index bef1491b83..fc49b43b8f 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . + package bloombits import ( @@ -20,151 +21,45 @@ import ( "sync/atomic" "testing" "time" - - "github.com/ethereum/go-ethereum/core/types" ) const testSectionSize = 4096 -func matcherTestVector(b uint, s uint64) []byte { - r := make([]byte, testSectionSize/8) - for i, _ := range r { - var bb byte - for bit := 0; bit < 8; bit++ { - blockIdx := s*testSectionSize + uint64(i*8+bit) - bb += bb - if (blockIdx % uint64(b)) == 0 { - bb++ - } - } - r[i] = bb - } - return r +// Tests the matcher pipeline on a single continuous workflow without interrupts. +func TestMatcherContinuous(t *testing.T) { + testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75) + testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81) + testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36) } -func expMatch1(idxs types.BloomIndexList, i uint64) bool { - for _, ii := range idxs { - if (i % uint64(ii)) != 0 { - return false - } - } - return true +// Tests the matcher pipeline on a constantly interrupted and resumed work pattern +// with the aim of ensuring data items are requested only once. +func TestMatcherIntermittent(t *testing.T) { + testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, true, 75) + testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, true, 81) + testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36) } -func expMatch2(idxs []types.BloomIndexList, i uint64) bool { - for _, ii := range idxs { - if expMatch1(ii, i) { - return true - } - } - return false -} - -func expMatch3(idxs [][]types.BloomIndexList, i uint64) bool { - for _, ii := range idxs { - if !expMatch2(ii, i) { - return false - } - } - return true -} - -func testServeMatcher(m *Matcher, stop chan struct{}, cnt *uint32, maxRequestLen int) { - // serve matcher with test vectors +// Tests the matcher pipeline on random input to hopefully catch anomalies. +func TestMatcherRandom(t *testing.T) { for i := 0; i < 10; i++ { - go func() { - for { - select { - case <-stop: - return - default: - } - b, ok := m.AllocSectionQueue() - if !ok { - return - } - if m.SectionCount(b) < maxRequestLen { - time.Sleep(time.Microsecond * 100) - } - s := m.FetchSections(b, maxRequestLen) - res := make([][]byte, len(s)) - for i, ss := range s { - res[i] = matcherTestVector(b, ss) - atomic.AddUint32(cnt, 1) - } - m.Deliver(b, s, res) - } - }() + testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 10000, 0) } } -func testMatcher(t *testing.T, idxs [][]types.BloomIndexList, cnt uint64, stopOnMatches bool, expCount uint32) uint32 { - count1 := testMatcherWithReqCount(t, idxs, cnt, stopOnMatches, expCount, 1) - count16 := testMatcherWithReqCount(t, idxs, cnt, stopOnMatches, expCount, 16) - if count1 != count16 { - t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: request count mismatch, %v with maxReqCount = 1 vs. %v with maxReqCount = 16", idxs, cnt, stopOnMatches, count1, count16) - } - return count1 -} - -func testMatcherWithReqCount(t *testing.T, idxs [][]types.BloomIndexList, cnt uint64, stopOnMatches bool, expCount uint32, maxReqCount int) uint32 { - m := NewMatcher(testSectionSize, nil, nil) - - for _, idxss := range idxs { - for _, idxs := range idxss { - for _, idx := range idxs { - m.newFetcher(idx) - } - } - } - - m.addresses = idxs[0] - m.topics = idxs[1:] - var reqCount uint32 - - stop := make(chan struct{}) - chn := m.Start(0, cnt-1) - testServeMatcher(m, stop, &reqCount, maxReqCount) - - for i := uint64(0); i < cnt; i++ { - if expMatch3(idxs, i) { - match, ok := <-chn - if !ok { - t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: expected #%v, results channel closed", idxs, cnt, stopOnMatches, i) - return 0 - } - if match != i { - t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: expected #%v, got #%v", idxs, cnt, stopOnMatches, i, match) - } - if stopOnMatches { - m.Stop() - close(stop) - stop = make(chan struct{}) - chn = m.Start(i+1, cnt-1) - testServeMatcher(m, stop, &reqCount, maxReqCount) - } - } - } - match, ok := <-chn - if ok { - t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: expected closed channel, got #%v", idxs, cnt, stopOnMatches, match) - } - m.Stop() - close(stop) - - if expCount != 0 && expCount != reqCount { - t.Errorf("Error matching idxs = %v count = %v stopOnMatches = %v: request count mismatch, expected #%v, got #%v", idxs, cnt, stopOnMatches, expCount, reqCount) - } - - return reqCount -} - -func testRandomIdxs(l []int, max int) [][]types.BloomIndexList { - res := make([][]types.BloomIndexList, len(l)) - for i, ll := range l { - res[i] = make([]types.BloomIndexList, ll) - for j, _ := range res[i] { - for k, _ := range res[i][j] { +// makeRandomIndexes generates a random filter system, composed on multiple filter +// criteria, each having one bloom list component for the address and arbitrarilly +// many topic bloom list components. +func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { + res := make([][]bloomIndexes, len(lengths)) + for i, topics := range lengths { + res[i] = make([]bloomIndexes, topics) + for j := 0; j < topics; j++ { + for k := 0; k < len(res[i][j]); k++ { res[i][j][k] = uint(rand.Intn(max-1) + 2) } } @@ -172,25 +67,173 @@ func testRandomIdxs(l []int, max int) [][]types.BloomIndexList { return res } -func TestMatcher(t *testing.T) { - testMatcher(t, [][]types.BloomIndexList{{{10, 20, 30}}}, 100000, false, 75) - testMatcher(t, [][]types.BloomIndexList{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81) - testMatcher(t, [][]types.BloomIndexList{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36) -} +// testMatcherDiffBatches runs the given matches test in single-delivery and also +// in batches delivery mode, verifying that all kinds of deliveries are handled +// correctly withn. +func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32) { + singleton := testMatcher(t, filter, blocks, intermittent, retrievals, 1) + batched := testMatcher(t, filter, blocks, intermittent, retrievals, 16) -func TestMatcherStopOnMatches(t *testing.T) { - testMatcher(t, [][]types.BloomIndexList{{{10, 20, 30}}}, 100000, true, 75) - testMatcher(t, [][]types.BloomIndexList{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36) -} - -func TestMatcherRandom(t *testing.T) { - for i := 0; i < 20; i++ { - testMatcher(t, testRandomIdxs([]int{1}, 50), 100000, false, 0) - testMatcher(t, testRandomIdxs([]int{3}, 50), 100000, false, 0) - testMatcher(t, testRandomIdxs([]int{2, 2, 2}, 20), 100000, false, 0) - testMatcher(t, testRandomIdxs([]int{5, 5, 5}, 50), 100000, false, 0) - idxs := testRandomIdxs([]int{2, 2, 2}, 20) - reqCount := testMatcher(t, idxs, 10000, false, 0) - testMatcher(t, idxs, 10000, true, reqCount) + if singleton != batched { + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) } } + +// testMatcherBothModes runs the given matcher test in both continuous as well as +// in intermittent mode, verifying that the request counts match each other. +func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, retrievals uint32) { + continuous := testMatcher(t, filter, blocks, false, retrievals, 16) + intermittent := testMatcher(t, filter, blocks, true, retrievals, 16) + + if continuous != intermittent { + t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent) + } +} + +// testMatcher is a generic tester to run the given matcher test and return the +// number of requests made for cross validation between different modes. +func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { + // Create a new matcher an simulate our explicit random bitsets + matcher := NewMatcher(testSectionSize, nil, nil) + + matcher.addresses = filter[0] + matcher.topics = filter[1:] + + for _, rule := range filter { + for _, topic := range rule { + for _, bit := range topic { + matcher.addScheduler(bit) + } + } + } + // Track the number of retrieval requests made + var requested uint32 + + // Start the matching session for the filter and the retriver goroutines + quit := make(chan struct{}) + matches := make(chan uint64, 16) + + session, err := matcher.Start(0, blocks-1, matches) + if err != nil { + t.Fatalf("failed to stat matcher session: %v", err) + } + startRetrievers(session, quit, &requested, maxReqCount) + + // Iterate over all the blocks and verify that the pipeline produces the correct matches + for i := uint64(0); i < blocks; i++ { + if expMatch3(filter, i) { + match, ok := <-matches + if !ok { + t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i) + return 0 + } + if match != i { + t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match) + } + // If we're testing intermittent mode, abort and restart the pipeline + if intermittent { + session.Close(time.Second) + close(quit) + + quit = make(chan struct{}) + matches = make(chan uint64, 16) + + session, err = matcher.Start(i+1, blocks-1, matches) + if err != nil { + t.Fatalf("failed to stat matcher session: %v", err) + } + startRetrievers(session, quit, &requested, maxReqCount) + } + } + } + // Ensure the result channel is torn down after the last block + match, ok := <-matches + if ok { + t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match) + } + // Clean up the session and ensure we match the expected retrieval count + session.Close(time.Second) + close(quit) + + if retrievals != 0 && requested != retrievals { + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals) + } + return requested +} + +// startRetrievers starts a batch of goroutines listening for section requests +// and serving them. +func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) { + requests := make(chan chan *Retrieval) + + for i := 0; i < 10; i++ { + // Start a multiplexer to test multiple threaded execution + go session.Multiplex(batch, 100*time.Microsecond, requests) + + // Start a services to match the above multiplexer + go func() { + for { + // Wait for a service request or a shutdown + select { + case <-quit: + return + + case request := <-requests: + task := <-request + + task.Bitsets = make([][]byte, len(task.Sections)) + for i, section := range task.Sections { + if rand.Int()%4 != 0 { // Handle occasional missing deliveries + task.Bitsets[i] = generateBitset(task.Bit, section) + atomic.AddUint32(retrievals, 1) + } + } + request <- task + } + } + }() + } +} + +// generateBitset generates the rotated bitset for the given bloom bit and section +// numbers. +func generateBitset(bit uint, section uint64) []byte { + bitset := make([]byte, testSectionSize/8) + for i := 0; i < len(bitset); i++ { + for b := 0; b < 8; b++ { + blockIdx := section*testSectionSize + uint64(i*8+b) + bitset[i] += bitset[i] + if (blockIdx % uint64(bit)) == 0 { + bitset[i]++ + } + } + } + return bitset +} + +func expMatch1(filter bloomIndexes, i uint64) bool { + for _, ii := range filter { + if (i % uint64(ii)) != 0 { + return false + } + } + return true +} + +func expMatch2(filter []bloomIndexes, i uint64) bool { + for _, ii := range filter { + if expMatch1(ii, i) { + return true + } + } + return false +} + +func expMatch3(filter [][]bloomIndexes, i uint64) bool { + for _, ii := range filter { + if !expMatch2(ii, i) { + return false + } + } + return true +} diff --git a/core/bloombits/scheduler.go b/core/bloombits/scheduler.go new file mode 100644 index 0000000000..6449c7465a --- /dev/null +++ b/core/bloombits/scheduler.go @@ -0,0 +1,181 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "sync" +) + +// request represents a bloom retrieval task to prioritize and pull from the local +// database or remotely from the network. +type request struct { + section uint64 // Section index to retrieve the a bit-vector from + bit uint // Bit index within the section to retrieve the vector of +} + +// response represents the state of a requested bit-vector through a scheduler. +type response struct { + cached []byte // Cached bits to dedup multiple requests + done chan struct{} // Channel to allow waiting for completion +} + +// scheduler handles the scheduling of bloom-filter retrieval operations for +// entire section-batches belonging to a single bloom bit. Beside scheduling the +// retrieval operations, this struct also deduplicates the requests and caches +// the results to minimize network/database overhead even in complex filtering +// scenarios. +type scheduler struct { + bit uint // Index of the bit in the bloom filter this scheduler is responsible for + responses map[uint64]*response // Currently pending retrieval requests or already cached responses + lock sync.Mutex // Lock protecting the responses from concurrent access +} + +// newScheduler creates a new bloom-filter retrieval scheduler for a specific +// bit index. +func newScheduler(idx uint) *scheduler { + return &scheduler{ + bit: idx, + responses: make(map[uint64]*response), + } +} + +// run creates a retrieval pipeline, receiving section indexes from sections and +// returning the results in the same order through the done channel. Concurrent +// runs of the same scheduler are allowed, leading to retrieval task deduplication. +func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) { + // Create a forwarder channel between requests and responses of the same size as + // the distribution channel (since that will block the pipeline anyway). + pend := make(chan uint64, cap(dist)) + + // Start the pipeline schedulers to forward between user -> distributor -> user + wg.Add(2) + go s.scheduleRequests(sections, dist, pend, quit, wg) + go s.scheduleDeliveries(pend, done, quit, wg) +} + +// reset cleans up any leftovers from previous runs. This is required before a +// restart to ensure the no previously requested but never delivered state will +// cause a lockup. +func (s *scheduler) reset() { + s.lock.Lock() + defer s.lock.Unlock() + + for section, res := range s.responses { + if res.cached == nil { + delete(s.responses, section) + } + } +} + +// scheduleRequests reads section retrieval requests from the input channel, +// deduplicates the stream and pushes unique retrieval tasks into the distribution +// channel for a database or network layer to honour. +func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) { + // Clean up the goroutine and pipeline when done + defer wg.Done() + defer close(pend) + + // Keep reading and scheduling section requests + for { + select { + case <-quit: + return + + case section, ok := <-reqs: + // New section retrieval requested + if !ok { + return + } + // Deduplicate retrieval requests + unique := false + + s.lock.Lock() + if s.responses[section] == nil { + s.responses[section] = &response{ + done: make(chan struct{}), + } + unique = true + } + s.lock.Unlock() + + // Schedule the section for retrieval and notify the deliverer to expect this section + if unique { + select { + case <-quit: + return + case dist <- &request{bit: s.bit, section: section}: + } + } + select { + case <-quit: + return + case pend <- section: + } + } + } +} + +// scheduleDeliveries reads section acceptance notifications and waits for them +// to be delivered, pushing them into the output data buffer. +func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) { + // Clean up the goroutine and pipeline when done + defer wg.Done() + defer close(done) + + // Keep reading notifications and scheduling deliveries + for { + select { + case <-quit: + return + + case idx, ok := <-pend: + // New section retrieval pending + if !ok { + return + } + // Wait until the request is honoured + s.lock.Lock() + res := s.responses[idx] + s.lock.Unlock() + + select { + case <-quit: + return + case <-res.done: + } + // Deliver the result + select { + case <-quit: + return + case done <- res.cached: + } + } + } +} + +// deliver is called by the request distributor when a reply to a request arrives. +func (s *scheduler) deliver(sections []uint64, data [][]byte) { + s.lock.Lock() + defer s.lock.Unlock() + + for i, section := range sections { + if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries + res.cached = data[i] + close(res.done) + } + } +} diff --git a/core/bloombits/scheduler_test.go b/core/bloombits/scheduler_test.go new file mode 100644 index 0000000000..8a159c2370 --- /dev/null +++ b/core/bloombits/scheduler_test.go @@ -0,0 +1,105 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "bytes" + "math/big" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Tests that the scheduler can deduplicate and forward retrieval requests to +// underlying fetchers and serve responses back, irrelevant of the concurrency +// of the requesting clients or serving data fetchers. +func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) } +func TestSchedulerSingleClientMultiFetcher(t *testing.T) { testScheduler(t, 1, 10, 5000) } +func TestSchedulerMultiClientSingleFetcher(t *testing.T) { testScheduler(t, 10, 1, 5000) } +func TestSchedulerMultiClientMultiFetcher(t *testing.T) { testScheduler(t, 10, 10, 5000) } + +func testScheduler(t *testing.T, clients int, fetchers int, requests int) { + f := newScheduler(0) + + // Create a batch of handler goroutines that respond to bloom bit requests and + // deliver them to the scheduler. + var fetchPend sync.WaitGroup + fetchPend.Add(fetchers) + defer fetchPend.Wait() + + fetch := make(chan *request, 16) + defer close(fetch) + + var delivered uint32 + for i := 0; i < fetchers; i++ { + go func() { + defer fetchPend.Done() + + for req := range fetch { + time.Sleep(time.Duration(rand.Intn(int(100 * time.Microsecond)))) + atomic.AddUint32(&delivered, 1) + + f.deliver([]uint64{ + req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds) + req.section, // Requested data + req.section, // Duplicated data (ensure it doesn't double close anything) + }, [][]byte{ + []byte{}, + new(big.Int).SetUint64(req.section).Bytes(), + new(big.Int).SetUint64(req.section).Bytes(), + }) + } + }() + } + // Start a batch of goroutines to concurrently run scheduling tasks + quit := make(chan struct{}) + + var pend sync.WaitGroup + pend.Add(clients) + + for i := 0; i < clients; i++ { + go func() { + defer pend.Done() + + in := make(chan uint64, 16) + out := make(chan []byte, 16) + + f.run(in, fetch, out, quit, &pend) + + go func() { + for j := 0; j < requests; j++ { + in <- uint64(j) + } + close(in) + }() + + for j := 0; j < requests; j++ { + bits := <-out + if want := new(big.Int).SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) { + t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want) + } + } + }() + } + pend.Wait() + + if have := atomic.LoadUint32(&delivered); int(have) != requests { + t.Errorf("request count mismatch: have %v, want %v", have, requests) + } +} diff --git a/core/bloombits/utils.go b/core/bloombits/utils.go deleted file mode 100644 index d0755cb65c..0000000000 --- a/core/bloombits/utils.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . -package bloombits - -import ( - "github.com/ethereum/go-ethereum/core/types" -) - -const BloomLength = 2048 - -// BloomBitsCreator takes SectionSize number of header bloom filters and calculates the bloomBits vectors of the section -type BloomBitsCreator struct { - blooms [BloomLength][]byte - sectionSize, bitIndex uint64 -} - -func NewBloomBitsCreator(sectionSize uint64) *BloomBitsCreator { - b := &BloomBitsCreator{sectionSize: sectionSize} - for i, _ := range b.blooms { - b.blooms[i] = make([]byte, sectionSize/8) - } - return b -} - -// AddHeaderBloom takes a single bloom filter and sets the corresponding bit column in memory accordingly -func (b *BloomBitsCreator) AddHeaderBloom(bloom types.Bloom) { - if b.bitIndex >= b.sectionSize { - panic("too many header blooms added") - } - - byteIdx := b.bitIndex / 8 - bitMask := byte(1) << byte(7-b.bitIndex%8) - for bloomBitIdx, _ := range b.blooms { - bloomByteIdx := BloomLength/8 - 1 - bloomBitIdx/8 - bloomBitMask := byte(1) << byte(bloomBitIdx%8) - if (bloom[bloomByteIdx] & bloomBitMask) != 0 { - b.blooms[bloomBitIdx][byteIdx] |= bitMask - } - } - b.bitIndex++ -} - -// GetBitVector returns the bit vector belonging to the given bit index after header blooms have been added -func (b *BloomBitsCreator) GetBitVector(idx uint) []byte { - if b.bitIndex != b.sectionSize { - panic("not enough header blooms added") - } - - return b.blooms[idx][:] -} diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 56360b59a3..f4c207dcca 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -36,14 +36,13 @@ import ( type ChainIndexerBackend interface { // Reset initiates the processing of a new chain segment, potentially terminating // any partially completed operations (in case of a reorg). - Reset(section uint64, lastSectionHead common.Hash) + Reset(section uint64) // Process crunches through the next header in the chain segment. The caller // will ensure a sequential order of headers. Process(header *types.Header) - // Commit finalizes the section metadata and stores it into the database. This - // interface will usually be a batch writer. + // Commit finalizes the section metadata and stores it into the database. Commit() error } @@ -101,34 +100,11 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken return c } -// AddKnownSectionHead marks a new section head as known/processed if it is newer -// than the already known best section head -func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) { - c.lock.Lock() - defer c.lock.Unlock() - - if section < c.storedSections { - return - } - c.setSectionHead(section, shead) - c.setValidSections(section + 1) -} - -// IndexerChain interface is used for connecting the indexer to a blockchain -type IndexerChain interface { - CurrentHeader() *types.Header - SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription -} - // Start creates a goroutine to feed chain head events into the indexer for // cascading background processing. Children do not need to be started, they // are notified about new events by their parents. -func (c *ChainIndexer) Start(chain IndexerChain) { - ch := make(chan ChainEvent, 10) - sub := chain.SubscribeChainEvent(ch) - currentHeader := chain.CurrentHeader() - - go c.eventLoop(currentHeader, ch, sub) +func (c *ChainIndexer) Start(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) { + go c.eventLoop(currentHeader, chainEventer) } // Close tears down all goroutines belonging to the indexer and returns any error @@ -149,14 +125,12 @@ func (c *ChainIndexer) Close() error { errs = append(errs, err) } } - // Close all children for _, child := range c.children { if err := child.Close(); err != nil { errs = append(errs, err) } } - // Return any failures switch { case len(errs) == 0: @@ -173,10 +147,12 @@ func (c *ChainIndexer) Close() error { // eventLoop is a secondary - optional - event loop of the indexer which is only // started for the outermost indexer to push chain head events into a processing // queue. -func (c *ChainIndexer) eventLoop(currentHeader *types.Header, ch chan ChainEvent, sub event.Subscription) { +func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) { // Mark the chain indexer as active, requiring an additional teardown atomic.StoreUint32(&c.active, 1) + events := make(chan ChainEvent, 10) + sub := chainEventer(events) defer sub.Unsubscribe() // Fire the initial new head event to start any outstanding processing @@ -193,7 +169,7 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, ch chan ChainEvent errc <- nil return - case ev, ok := <-ch: + case ev, ok := <-events: // Received a new event, ensure it's not nil (closing) and update if !ok { errc := <-c.quit @@ -257,10 +233,9 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { // down into the processing backend. func (c *ChainIndexer) updateLoop() { var ( - updated time.Time - updateMsg bool + updating bool + updated time.Time ) - for { select { case errc := <-c.quit: @@ -275,7 +250,7 @@ func (c *ChainIndexer) updateLoop() { // Periodically print an upgrade log message to the user if time.Since(updated) > 8*time.Second { if c.knownSections > c.storedSections+1 { - updateMsg = true + updating = true c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections) } updated = time.Now() @@ -284,7 +259,7 @@ func (c *ChainIndexer) updateLoop() { section := c.storedSections var oldHead common.Hash if section > 0 { - oldHead = c.SectionHead(section - 1) + oldHead = c.sectionHead(section - 1) } // Process the newly defined section in the background c.lock.Unlock() @@ -295,11 +270,11 @@ func (c *ChainIndexer) updateLoop() { c.lock.Lock() // If processing succeeded and no reorgs occcurred, mark the section completed - if err == nil && oldHead == c.SectionHead(section-1) { + if err == nil && oldHead == c.sectionHead(section-1) { c.setSectionHead(section, newHead) c.setValidSections(section + 1) - if c.storedSections == c.knownSections && updateMsg { - updateMsg = false + if c.storedSections == c.knownSections && updating { + updating = false c.log.Info("Finished upgrading chain index") } @@ -336,7 +311,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com c.log.Trace("Processing new chain section", "section", section) // Reset and partial processing - c.backend.Reset(section, lastHead) + c.backend.Reset(section) for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { hash := GetCanonicalHash(c.chainDb, number) @@ -366,7 +341,7 @@ func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) { c.lock.Lock() defer c.lock.Unlock() - return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1) + return c.storedSections, c.storedSections*c.sectionSize - 1, c.sectionHead(c.storedSections - 1) } // AddChildIndexer adds a child ChainIndexer that can use the output of this one @@ -408,7 +383,7 @@ func (c *ChainIndexer) setValidSections(sections uint64) { // sectionHead retrieves the last block hash of a processed section from the // index database. -func (c *ChainIndexer) SectionHead(section uint64) common.Hash { +func (c *ChainIndexer) sectionHead(section uint64) common.Hash { var data [8]byte binary.BigEndian.PutUint64(data[:], section) diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 247f52cf9e..b761e8a5b2 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" ) @@ -209,7 +208,7 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { return b.stored * b.indexer.sectionSize } -func (b *testChainIndexBackend) Reset(section uint64, lastSectionHead common.Hash) { +func (b *testChainIndexBackend) Reset(section uint64) { b.section = section b.headerCnt = 0 } diff --git a/core/database_util.go b/core/database_util.go index 179d6f1b2d..989071104e 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -33,21 +33,41 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +// DatabaseReader wraps the Get method of a backing data store. +type DatabaseReader interface { + Get(key []byte) (value []byte, err error) +} + +// DatabaseWriter wraps the Put method of a backing data store. +type DatabaseWriter interface { + Put(key, value []byte) error +} + +// DatabaseDeleter wraps the Delete method of a backing data store. +type DatabaseDeleter interface { + Delete(key []byte) error +} + var ( headHeaderKey = []byte("LastHeader") headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header - tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td - numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash - blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) - bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body - blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata - preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`). + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata + bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits - configPrefix = []byte("ethereum-config-") // config prefix for the db + preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage + configPrefix = []byte("ethereum-config-") // config prefix for the db + + // Chain index prefixes (use `i` + single byte to avoid mixing data types). + BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress // used by old db, now only used for conversion oldReceiptsPrefix = []byte("receipts-") @@ -57,8 +77,6 @@ var ( preimageCounter = metrics.NewCounter("db/preimage/total") preimageHitCounter = metrics.NewCounter("db/preimage/hits") - - bloomBitsPrefix = []byte("bloomBits-") ) // txLookupEntry is a positional metadata to help looking up the data content of @@ -77,7 +95,7 @@ func encodeBlockNumber(number uint64) []byte { } // GetCanonicalHash retrieves a hash assigned to a canonical block number. -func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { +func GetCanonicalHash(db DatabaseReader, number uint64) common.Hash { data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) if len(data) == 0 { return common.Hash{} @@ -91,7 +109,7 @@ const missingNumber = uint64(0xffffffffffffffff) // GetBlockNumber returns the block number assigned to a block hash // if the corresponding header is present in the database -func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { +func GetBlockNumber(db DatabaseReader, hash common.Hash) uint64 { data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) if len(data) != 8 { return missingNumber @@ -104,7 +122,7 @@ func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { // last block hash is only updated upon a full block import, the last header // hash is updated already at header import, allowing head tracking for the // light synchronization mechanism. -func GetHeadHeaderHash(db ethdb.Database) common.Hash { +func GetHeadHeaderHash(db DatabaseReader) common.Hash { data, _ := db.Get(headHeaderKey) if len(data) == 0 { return common.Hash{} @@ -113,7 +131,7 @@ func GetHeadHeaderHash(db ethdb.Database) common.Hash { } // GetHeadBlockHash retrieves the hash of the current canonical head block. -func GetHeadBlockHash(db ethdb.Database) common.Hash { +func GetHeadBlockHash(db DatabaseReader) common.Hash { data, _ := db.Get(headBlockKey) if len(data) == 0 { return common.Hash{} @@ -125,7 +143,7 @@ func GetHeadBlockHash(db ethdb.Database) common.Hash { // fast synchronization. The difference between this and GetHeadBlockHash is that // whereas the last block hash is only updated upon a full block import, the last // fast hash is updated when importing pre-processed blocks. -func GetHeadFastBlockHash(db ethdb.Database) common.Hash { +func GetHeadFastBlockHash(db DatabaseReader) common.Hash { data, _ := db.Get(headFastKey) if len(data) == 0 { return common.Hash{} @@ -135,14 +153,14 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. -func GetHeaderRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { +func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) return data } // GetHeader retrieves the block header corresponding to the hash, nil if none // found. -func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header { +func GetHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header { data := GetHeaderRLP(db, hash, number) if len(data) == 0 { return nil @@ -156,14 +174,14 @@ func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header } // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { +func GetBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) return data } // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { +func GetBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { data := GetBodyRLP(db, hash, number) if len(data) == 0 { return nil @@ -178,7 +196,7 @@ func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. -func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { +func GetTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), tdSuffix...)) if len(data) == 0 { return nil @@ -197,7 +215,7 @@ func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { // // Note, due to concurrent download of header and block body the header and thus // canonical hash can be stored in the database but the body data not (yet). -func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { +func GetBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { // Retrieve the block header and body contents header := GetHeader(db, hash, number) if header == nil { @@ -213,7 +231,7 @@ func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types.Receipts { +func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { return nil @@ -232,7 +250,7 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types. // GetTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. -func GetTxLookupEntry(db ethdb.Database, hash common.Hash) (common.Hash, uint64, uint64) { +func GetTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { // Load the positional metadata from disk and bail if it fails data, _ := db.Get(append(lookupPrefix, hash.Bytes()...)) if len(data) == 0 { @@ -249,7 +267,7 @@ func GetTxLookupEntry(db ethdb.Database, hash common.Hash) (common.Hash, uint64, // GetTransaction retrieves a specific transaction from the database, along with // its added positional metadata. -func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { +func GetTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { // Retrieve the lookup metadata and resolve the transaction from the body blockHash, blockNumber, txIndex := GetTxLookupEntry(db, hash) @@ -284,7 +302,7 @@ func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, co // GetReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. -func GetReceipt(db ethdb.Database, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { +func GetReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { // Retrieve the lookup metadata and resolve the receipt from the receipts blockHash, blockNumber, receiptIndex := GetTxLookupEntry(db, hash) @@ -309,8 +327,20 @@ func GetReceipt(db ethdb.Database, hash common.Hash) (*types.Receipt, common.Has return (*types.Receipt)(&receipt), common.Hash{}, 0, 0 } +// GetBloomBits retrieves the compressed bloom bit vector belonging to the given +// section and bit index from the. +func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) []byte { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + bits, _ := db.Get(key) + return bits +} + // WriteCanonicalHash stores the canonical hash for the given block number. -func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { +func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) error { key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) @@ -319,7 +349,7 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro } // WriteHeadHeaderHash stores the head header's hash. -func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { log.Crit("Failed to store last header's hash", "err", err) } @@ -327,7 +357,7 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last block's hash", "err", err) } @@ -335,7 +365,7 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { } // WriteHeadFastBlockHash stores the fast head block's hash. -func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { log.Crit("Failed to store last fast block's hash", "err", err) } @@ -343,7 +373,7 @@ func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { } // WriteHeader serializes a block header into the database. -func WriteHeader(db ethdb.Database, header *types.Header) error { +func WriteHeader(db DatabaseWriter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) if err != nil { return err @@ -363,7 +393,7 @@ func WriteHeader(db ethdb.Database, header *types.Header) error { } // WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error { +func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err @@ -372,7 +402,7 @@ func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.B } // WriteBodyRLP writes a serialized body of a block into the database. -func WriteBodyRLP(db ethdb.Database, hash common.Hash, number uint64, rlp rlp.RawValue) error { +func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) error { key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, rlp); err != nil { log.Crit("Failed to store block body", "err", err) @@ -381,7 +411,7 @@ func WriteBodyRLP(db ethdb.Database, hash common.Hash, number uint64, rlp rlp.Ra } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) error { +func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err @@ -394,7 +424,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) er } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db ethdb.Database, block *types.Block) error { +func WriteBlock(db DatabaseWriter, block *types.Block) error { // Store the body first to retain database consistency if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err @@ -409,7 +439,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error { +func WriteBlockReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -454,29 +484,42 @@ func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { return nil } +// WriteBloomBits writes the compressed bloom bits vector belonging to the given +// section and bit index. +func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + if err := db.Put(key, bits); err != nil { + log.Crit("Failed to store bloom bits", "err", err) + } +} + // DeleteCanonicalHash removes the number to hash canonical mapping. -func DeleteCanonicalHash(db ethdb.Database, number uint64) { +func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(blockHashPrefix, hash.Bytes()...)) db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...)) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) { DeleteBlockReceipts(db, hash, number) DeleteHeader(db, hash, number) DeleteBody(db, hash, number) @@ -484,12 +527,12 @@ func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { } // DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteBlockReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTxLookupEntry removes all transaction data associated with a hash. -func DeleteTxLookupEntry(db ethdb.Database, hash common.Hash) { +func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { db.Delete(append(lookupPrefix, hash.Bytes()...)) } @@ -521,7 +564,7 @@ func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash] } // GetBlockChainVersion reads the version number from db. -func GetBlockChainVersion(db ethdb.Database) int { +func GetBlockChainVersion(db DatabaseReader) int { var vsn uint enc, _ := db.Get([]byte("BlockchainVersion")) rlp.DecodeBytes(enc, &vsn) @@ -529,13 +572,13 @@ func GetBlockChainVersion(db ethdb.Database) int { } // WriteBlockChainVersion writes vsn as the version number to db. -func WriteBlockChainVersion(db ethdb.Database, vsn int) { +func WriteBlockChainVersion(db DatabaseWriter, vsn int) { enc, _ := rlp.EncodeToBytes(uint(vsn)) db.Put([]byte("BlockchainVersion"), enc) } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *params.ChainConfig) error { +func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) error { // short circuit and ignore if nil config. GetChainConfig // will return a default. if cfg == nil { @@ -551,7 +594,7 @@ func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *params.ChainConf } // GetChainConfig will fetch the network settings based on the given hash. -func GetChainConfig(db ethdb.Database, hash common.Hash) (*params.ChainConfig, error) { +func GetChainConfig(db DatabaseReader, hash common.Hash) (*params.ChainConfig, error) { jsonChainConfig, _ := db.Get(append(configPrefix, hash[:]...)) if len(jsonChainConfig) == 0 { return nil, ErrChainConfigNotFound @@ -566,7 +609,7 @@ func GetChainConfig(db ethdb.Database, hash common.Hash) (*params.ChainConfig, e } // FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db ethdb.Database, a, b *types.Header) *types.Header { +func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header { for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { a = GetHeader(db, a.ParentHash, a.Number.Uint64()-1) if a == nil { @@ -591,22 +634,3 @@ func FindCommonAncestor(db ethdb.Database, a, b *types.Header) *types.Header { } return a } - -// GetBloomBits reads the compressed bloomBits vector belonging to the given section and bit index from the db -func GetBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, sectionHead common.Hash) ([]byte, error) { - var encKey [10]byte - binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx)) - binary.BigEndian.PutUint64(encKey[2:10], sectionIdx) - key := append(append(bloomBitsPrefix, encKey[:]...), sectionHead.Bytes()...) - bloomBits, err := db.Get(key) - return bloomBits, err -} - -// StoreBloomBits writes the compressed bloomBits vector belonging to the given section and bit index to the db -func StoreBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, sectionHead common.Hash, bloomBits []byte) { - var encKey [10]byte - binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx)) - binary.BigEndian.PutUint64(encKey[2:10], sectionIdx) - key := append(append(bloomBitsPrefix, encKey[:]...), sectionHead.Bytes()...) - db.Put(key, bloomBits) -} diff --git a/core/types/bloom9.go b/core/types/bloom9.go index bdc6e60e79..a76b6f33c5 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -28,10 +28,16 @@ type bytesBacked interface { Bytes() []byte } -const bloomLength = 256 +const ( + // BloomByteLength represents the number of bytes used in a header log bloom. + BloomByteLength = 256 -// Bloom represents a 256 bit bloom filter. -type Bloom [bloomLength]byte + // BloomBitLength represents the number of bits used in a header log bloom. + BloomBitLength = 8 * BloomByteLength +) + +// Bloom represents a 2048 bit bloom filter. +type Bloom [BloomByteLength]byte // BytesToBloom converts a byte slice to a bloom filter. // It panics if b is not of suitable size. @@ -47,7 +53,7 @@ func (b *Bloom) SetBytes(d []byte) { if len(b) < len(d) { panic(fmt.Sprintf("bloom bytes too big %d %d", len(b), len(d))) } - copy(b[bloomLength-len(d):], d) + copy(b[BloomByteLength-len(d):], d) } // Add adds d to the filter. Future calls of Test(d) will return true. @@ -106,20 +112,6 @@ func LogsBloom(logs []*Log) *big.Int { return bin } -type BloomIndexList [3]uint - -// BloomIndexes returns the bloom filter bit indexes belonging to the given key -func BloomIndexes(b []byte) BloomIndexList { - b = crypto.Keccak256(b[:]) - - var r [3]uint - for i, _ := range r { - r[i] = (uint(b[i+i+1]) + (uint(b[i+i]) << 8)) & 2047 - } - - return r -} - func bloom9(b []byte) *big.Int { b = crypto.Keccak256(b[:]) diff --git a/eth/api_backend.go b/eth/api_backend.go index fa3cf3f805..91f392f94f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -24,11 +24,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -196,28 +196,13 @@ func (b *EthApiBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } -func (b *EthApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { - results := make([][]byte, len(sectionIdxList)) - var err error - for i, sectionIdx := range sectionIdxList { - sectionHead := core.GetCanonicalHash(b.eth.chainDb, (sectionIdx+1)*bloomBitsSection-1) - results[i], err = core.GetBloomBits(b.eth.chainDb, bitIdx, sectionIdx, sectionHead) - if err != nil { - return nil, err - } - } - return results, nil +func (b *EthApiBackend) BloomStatus() (uint64, uint64) { + sections, _, _ := b.eth.bloomIndexer.Sections() + return params.BloomBitsBlocks, sections } -func (b *EthApiBackend) BloomBitsSections() uint64 { - sections, _, _ := b.eth.bbIndexer.Sections() - return sections -} - -func (b *EthApiBackend) BloomBitsConfig() filters.BloomConfig { - return filters.BloomConfig{ - SectionSize: bloomBitsSection, - MaxRequestLen: 16, - MaxRequestWait: 0, +func (b *EthApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + for i := 0; i < bloomFilterThreads; i++ { + go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) } } diff --git a/eth/backend.go b/eth/backend.go index efc0a2317f..99a1928fec 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -77,7 +78,8 @@ type Ethereum struct { engine consensus.Engine accountManager *accounts.Manager - bbIndexer *core.ChainIndexer + bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests + bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports ApiBackend *EthApiBackend @@ -127,7 +129,8 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { networkId: config.NetworkId, gasPrice: config.GasPrice, etherbase: config.Etherbase, - bbIndexer: NewBloomBitsProcessor(chainDb, bloomBitsSection), + bloomRequests: make(chan chan *bloombits.Retrieval), + bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks), } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) @@ -151,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.blockchain.SetHead(compat.RewindTo) core.WriteChainConfig(chainDb, genesisHash, chainConfig) } - eth.bbIndexer.Start(eth.blockchain) + eth.bloomIndexer.Start(eth.blockchain.CurrentHeader(), eth.blockchain.SubscribeChainEvent) if config.TxPool.Journal != "" { config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal) @@ -261,7 +264,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, false, bloomBitsSection), + Service: filters.NewPublicFilterAPI(s.ApiBackend, false), Public: true, }, { Namespace: "admin", @@ -359,14 +362,17 @@ func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManage func (s *Ethereum) Protocols() []p2p.Protocol { if s.lesServer == nil { return s.protocolManager.SubProtocols - } else { - return append(s.protocolManager.SubProtocols, s.lesServer.Protocols()...) } + return append(s.protocolManager.SubProtocols, s.lesServer.Protocols()...) } // Start implements node.Service, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *Ethereum) Start(srvr *p2p.Server) error { + // Start the bloom bits servicing goroutines + s.startBloomHandlers() + + // Start the RPC service s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) // Figure out a max peers count based on the server limits @@ -377,6 +383,7 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { maxPeers = srvr.MaxPeers / 2 } } + // Start the networking layer and the light server if requested s.protocolManager.Start(maxPeers) if s.lesServer != nil { s.lesServer.Start(srvr) @@ -390,7 +397,7 @@ func (s *Ethereum) Stop() error { if s.stopDbUpgrade != nil { s.stopDbUpgrade() } - s.bbIndexer.Close() + s.bloomIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() if s.lesServer != nil { diff --git a/eth/bloombits.go b/eth/bloombits.go new file mode 100644 index 0000000000..02de408b0e --- /dev/null +++ b/eth/bloombits.go @@ -0,0 +1,142 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +const ( + // bloomServiceThreads is the number of goroutines used globally by an Ethereum + // instance to service bloombits lookups for all running filters. + bloomServiceThreads = 16 + + // bloomFilterThreads is the number of goroutines used locally per filter to + // multiplex requests onto the global servicing goroutines. + bloomFilterThreads = 3 + + // bloomRetrievalBatch is the maximum number of bloom bit retrievals to service + // in a single batch. + bloomRetrievalBatch = 16 + + // bloomRetrievalWait is the maximum time to wait for enough bloom bit requests + // to accumulate request an entire batch (avoiding hysteresis). + bloomRetrievalWait = time.Duration(0) +) + +// startBloomHandlers starts a batch of goroutines to accept bloom bit database +// retrievals from possibly a range of filters and serving the data to satisfy. +func (eth *Ethereum) startBloomHandlers() { + for i := 0; i < bloomServiceThreads; i++ { + go func() { + for { + select { + case <-eth.shutdownChan: + return + + case request := <-eth.bloomRequests: + task := <-request + + task.Bitsets = make([][]byte, len(task.Sections)) + for i, section := range task.Sections { + head := core.GetCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) + blob, err := bitutil.DecompressBytes(core.GetBloomBits(eth.chainDb, task.Bit, section, head), int(params.BloomBitsBlocks)/8) + if err != nil { + panic(err) + } + task.Bitsets[i] = blob + } + request <- task + } + } + }() + } +} + +const ( + // bloomConfirms is the number of confirmation blocks before a bloom section is + // considered probably final and its rotated bits are calculated. + bloomConfirms = 256 + + // bloomThrottling is the time to wait between processing two consecutive index + // sections. It's useful during chain upgrades to prevent disk overload. + bloomThrottling = 100 * time.Millisecond +) + +// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index +// for the Ethereum header bloom filters, permitting blazing fast filtering. +type BloomIndexer struct { + size uint64 // section size to generate bloombits for + + db ethdb.Database // database instance to write index data and metadata into + gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index + + section uint64 // Section is the section number being processed currently + head common.Hash // Head is the hash of the last header processed +} + +// NewBloomIndexer returns a chain indexer that generates bloom bits data for the +// canonical chain for fast logs filtering. +func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer { + backend := &BloomIndexer{ + db: db, + size: size, + } + table := ethdb.NewTable(db, string(core.BloomBitsIndexPrefix)) + + return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits") +} + +// Reset implements core.ChainIndexerBackend, starting a new bloombits index +// section. +func (b *BloomIndexer) Reset(section uint64) { + gen, err := bloombits.NewGenerator(uint(b.size)) + if err != nil { + panic(err) + } + b.gen, b.section, b.head = gen, section, common.Hash{} +} + +// Process implements core.ChainIndexerBackend, adding a new header's bloom into +// the index. +func (b *BloomIndexer) Process(header *types.Header) { + b.gen.AddBloom(header.Bloom) + b.head = header.Hash() +} + +// Commit implements core.ChainIndexerBackend, finalizing the bloom section and +// writing it out into the database. +func (b *BloomIndexer) Commit() error { + batch := b.db.NewBatch() + + for i := 0; i < types.BloomBitLength; i++ { + bits, err := b.gen.Bitset(uint(i)) + if err != nil { + return err + } + core.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) + } + return batch.Write() +} diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go index ce8ce699ab..d41afa17c0 100644 --- a/eth/db_upgrade.go +++ b/eth/db_upgrade.go @@ -22,10 +22,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -136,38 +133,3 @@ func upgradeDeduplicateData(db ethdb.Database) func() error { return <-errc } } - -// BloomBitsIndex implements ChainIndex -type BloomBitsIndex struct { - db ethdb.Database - bc *bloombits.BloomBitsCreator - section, sectionSize uint64 - sectionHead common.Hash -} - -// number of confirmation blocks before a section is considered probably final and its bloom bits are calculated -const bloomBitsConfirmations = 256 - -// NewBloomBitsProcessor returns a chain processor that generates bloom bits data for the canonical chain -func NewBloomBitsProcessor(db ethdb.Database, sectionSize uint64) *core.ChainIndexer { - backend := &BloomBitsIndex{db: db, sectionSize: sectionSize} - return core.NewChainIndexer(db, ethdb.NewTable(db, "bbIndex-"), backend, sectionSize, bloomBitsConfirmations, time.Millisecond*100, "bloombits") -} - -func (b *BloomBitsIndex) Reset(section uint64, lastSectionHead common.Hash) { - b.bc = bloombits.NewBloomBitsCreator(b.sectionSize) - b.section = section -} - -func (b *BloomBitsIndex) Process(header *types.Header) { - b.bc.AddHeaderBloom(header.Bloom) - b.sectionHead = header.Hash() -} - -func (b *BloomBitsIndex) Commit() error { - for i := 0; i < bloombits.BloomLength; i++ { - compVector := bitutil.CompressBytes(b.bc.GetBitVector(uint(i))) - core.StoreBloomBits(b.db, uint64(i), b.section, b.sectionHead, compVector) - } - return nil -} diff --git a/eth/filters/api.go b/eth/filters/api.go index 11767753e3..6e1d48adb6 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -51,27 +51,24 @@ type filter struct { // PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. type PublicFilterAPI struct { - backend Backend - bloomBitsSection uint64 - mux *event.TypeMux - quit chan struct{} - chainDb ethdb.Database - events *EventSystem - filtersMu sync.Mutex - filters map[rpc.ID]*filter + backend Backend + mux *event.TypeMux + quit chan struct{} + chainDb ethdb.Database + events *EventSystem + filtersMu sync.Mutex + filters map[rpc.ID]*filter } // NewPublicFilterAPI returns a new PublicFilterAPI instance. -func NewPublicFilterAPI(backend Backend, lightMode bool, bloomBitsSection uint64) *PublicFilterAPI { +func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI { api := &PublicFilterAPI{ - backend: backend, - bloomBitsSection: bloomBitsSection, - mux: backend.EventMux(), - chainDb: backend.ChainDb(), - events: NewEventSystem(backend.EventMux(), backend, lightMode), - filters: make(map[rpc.ID]*filter), + backend: backend, + mux: backend.EventMux(), + chainDb: backend.ChainDb(), + events: NewEventSystem(backend.EventMux(), backend, lightMode), + filters: make(map[rpc.ID]*filter), } - go api.timeoutLoop() return api @@ -326,16 +323,20 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { + // Convert the RPC block numbers into internal representations if crit.FromBlock == nil { crit.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) } if crit.ToBlock == nil { crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) } - + // Create and run the filter to get all the logs filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics) - logs, err := filter.Find(ctx) + logs, err := filter.Logs(ctx) + if err != nil { + return nil, err + } return returnLogs(logs), err } @@ -369,20 +370,18 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty return nil, fmt.Errorf("filter not found") } - var begin, end int64 + begin := rpc.LatestBlockNumber.Int64() if f.crit.FromBlock != nil { begin = f.crit.FromBlock.Int64() - } else { - begin = rpc.LatestBlockNumber.Int64() } + end := rpc.LatestBlockNumber.Int64() if f.crit.ToBlock != nil { end = f.crit.ToBlock.Int64() - } else { - end = rpc.LatestBlockNumber.Int64() } + // Create and run the filter to get all the logs filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) - logs, err := filter.Find(ctx) + logs, err := filter.Logs(ctx) if err != nil { return nil, err } @@ -390,7 +389,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty } // GetFilterChanges returns the logs for the filter with the given id since -// last time is was called. This can be used for polling. +// last time it was called. This can be used for polling. // // For pending transaction and block filters the result is []common.Hash. // (pending)Log filters return []Log. diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 2487bc0eb3..d994378fc1 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -31,82 +31,41 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/node" - "github.com/golang/snappy" ) func BenchmarkBloomBits512(b *testing.B) { - benchmarkBloomBitsForSize(b, 512) + benchmarkBloomBits(b, 512) } func BenchmarkBloomBits1k(b *testing.B) { - benchmarkBloomBitsForSize(b, 1024) + benchmarkBloomBits(b, 1024) } func BenchmarkBloomBits2k(b *testing.B) { - benchmarkBloomBitsForSize(b, 2048) + benchmarkBloomBits(b, 2048) } func BenchmarkBloomBits4k(b *testing.B) { - benchmarkBloomBitsForSize(b, 4096) + benchmarkBloomBits(b, 4096) } func BenchmarkBloomBits8k(b *testing.B) { - benchmarkBloomBitsForSize(b, 8192) + benchmarkBloomBits(b, 8192) } func BenchmarkBloomBits16k(b *testing.B) { - benchmarkBloomBitsForSize(b, 16384) + benchmarkBloomBits(b, 16384) } func BenchmarkBloomBits32k(b *testing.B) { - benchmarkBloomBitsForSize(b, 32768) -} - -func benchmarkBloomBitsForSize(b *testing.B, sectionSize uint64) { - benchmarkBloomBits(b, sectionSize, 0) - benchmarkBloomBits(b, sectionSize, 1) - benchmarkBloomBits(b, sectionSize, 2) + benchmarkBloomBits(b, 32768) } const benchFilterCnt = 2000 -func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { +func benchmarkBloomBits(b *testing.B, sectionSize uint64) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" - fmt.Println("Running bloombits benchmark section size:", sectionSize, " compression method:", comp) - - var ( - compressFn func([]byte) []byte - decompressFn func([]byte, int) ([]byte, error) - ) - switch comp { - case 0: - // no compression - compressFn = func(data []byte) []byte { - return data - } - decompressFn = func(data []byte, target int) ([]byte, error) { - if len(data) != target { - panic(nil) - } - return data, nil - } - case 1: - // bitutil/compress.go - compressFn = bitutil.CompressBytes - decompressFn = bitutil.DecompressBytes - case 2: - // go snappy - compressFn = func(data []byte) []byte { - return snappy.Encode(nil, data) - } - decompressFn = func(data []byte, target int) ([]byte, error) { - decomp, err := snappy.Decode(nil, data) - if err != nil || len(decomp) != target { - panic(err) - } - return decomp, nil - } - } + fmt.Println("Running bloombits benchmark section size:", sectionSize) db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) if err != nil { @@ -128,7 +87,10 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { cnt := (headNum - 512) / sectionSize var dataSize, compSize uint64 for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { - bc := bloombits.NewBloomBitsCreator(sectionSize) + bc, err := bloombits.NewGenerator(uint(sectionSize)) + if err != nil { + b.Fatalf("failed to create generator: %v", err) + } var header *types.Header for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { hash := core.GetCanonicalHash(db, i) @@ -136,15 +98,18 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { if header == nil { b.Fatalf("Error creating bloomBits data") } - bc.AddHeaderBloom(header.Bloom) + bc.AddBloom(header.Bloom) } sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) - for i := 0; i < bloombits.BloomLength; i++ { - data := bc.GetBitVector(uint(i)) - comp := compressFn(data) + for i := 0; i < types.BloomBitLength; i++ { + data, err := bc.Bitset(uint(i)) + if err != nil { + b.Fatalf("failed to retrieve bitset: %v", err) + } + comp := bitutil.CompressBytes(data) dataSize += uint64(len(data)) compSize += uint64(len(comp)) - core.StoreBloomBits(db, uint64(i), sectionIdx, sectionHead, comp) + core.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) } //if sectionIdx%50 == 0 { // fmt.Println(" section", sectionIdx, "/", cnt) @@ -171,8 +136,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { addr[0] = byte(i) addr[1] = byte(i / 256) filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) - filter.decompress = decompressFn - if _, err := filter.Find(context.Background()); err != nil { + if _, err := filter.Logs(context.Background()); err != nil { b.Error("filter.Find error:", err) } } @@ -229,7 +193,7 @@ func BenchmarkNoBloomBits(b *testing.B) { mux := new(event.TypeMux) backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} filter := New(backend, 0, int64(headNum), []common.Address{common.Address{}}, nil) - filter.Find(context.Background()) + filter.Logs(context.Background()) d := time.Since(start) fmt.Println("Finished running filter benchmarks") fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks") diff --git a/eth/filters/filter.go b/eth/filters/filter.go index ea9ccf2f92..3a2226f6be 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -19,11 +19,9 @@ package filters import ( "context" "math/big" - "sync" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" @@ -37,140 +35,143 @@ type Backend interface { EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - BloomBitsSections() uint64 - BloomBitsConfig() BloomConfig + SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) -} -type BloomConfig struct { - SectionSize uint64 - MaxRequestLen int - MaxRequestWait time.Duration + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) } // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend - bloomBitsConfig BloomConfig + backend Backend db ethdb.Database begin, end int64 addresses []common.Address topics [][]common.Hash - decompress func([]byte, int) ([]byte, error) - matcher *bloombits.Matcher + matcher *bloombits.Matcher } // New creates a new filter which uses a bloom filter on blocks to figure out whether // a particular block is interesting or not. func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { + size, _ := backend.BloomStatus() + return &Filter{ - backend: backend, - begin: begin, - end: end, - addresses: addresses, - topics: topics, - bloomBitsConfig: backend.BloomBitsConfig(), - db: backend.ChainDb(), - matcher: bloombits.NewMatcher(backend.BloomBitsConfig().SectionSize, addresses, topics), - decompress: bitutil.DecompressBytes, + backend: backend, + begin: begin, + end: end, + addresses: addresses, + topics: topics, + db: backend.ChainDb(), + matcher: bloombits.NewMatcher(size, addresses, topics), } } -// FindOnce searches the blockchain for matching log entries, returning -// all matching entries from the first block that contains matches, -// updating the start point of the filter accordingly. If no results are -// found, a nil slice is returned. -func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) { - head, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) - if head == nil { +// Logs searches the blockchain for matching log entries, returning all from the +// first block that contains matches, updating the start of the filter accordingly. +func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { + // Figure out the limits of the filter range + header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + if header == nil { return nil, nil } - headBlockNumber := head.Number.Uint64() + head := header.Number.Uint64() - var beginBlockNo uint64 = uint64(f.begin) if f.begin == -1 { - beginBlockNo = headBlockNumber + f.begin = int64(head) } - var endBlockNo uint64 = uint64(f.end) + end := uint64(f.end) if f.end == -1 { - endBlockNo = headBlockNumber + end = head } - - logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo) - f.begin = int64(blockNumber + 1) + // Gather all indexed logs, and finish with non indexed ones + var ( + logs []*types.Log + err error + ) + size, sections := f.backend.BloomStatus() + if indexed := sections * size; indexed > uint64(f.begin) { + if indexed > end { + logs, err = f.indexedLogs(ctx, end) + } else { + logs, err = f.indexedLogs(ctx, indexed-1) + } + if err != nil { + return logs, err + } + } + rest, err := f.unindexedLogs(ctx, end) + logs = append(logs, rest...) return logs, err } -// Run filters logs with the current parameters set -func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) { +// indexedLogs returns the logs matching the filter criteria based on the bloom +// bits indexed available locally or via the network. +func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) { + // Create a matcher session and request servicing from the backend + matches := make(chan uint64, 64) + + session, err := f.matcher.Start(uint64(f.begin), end, matches) + if err != nil { + return nil, err + } + defer session.Close(time.Second) + + f.backend.ServiceFilter(ctx, session) + + // Iterate over the matches until exhausted or context closed + var logs []*types.Log + for { - newLogs, err := f.FindOnce(ctx) - if len(newLogs) == 0 || err != nil { + select { + case number, ok := <-matches: + // Abort if all matches have been fulfilled + if !ok { + f.begin = int64(end) + 1 + return logs, nil + } + // Retrieve the suggested block and pull any truly matching logs + header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + if header == nil || err != nil { + return logs, err + } + found, err := f.checkMatches(ctx, header) + if err != nil { + return logs, err + } + logs = append(logs, found...) + + case <-ctx.Done(): + return logs, ctx.Err() + } + } +} + +// indexedLogs returns the logs matching the filter criteria based on raw block +// iteration and bloom matching. +func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) { + var logs []*types.Log + + for ; f.begin <= int64(end); f.begin++ { + header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + if header == nil || err != nil { return logs, err } - logs = append(logs, newLogs...) - } -} - -// nextRequest returns the next request to retrieve for the bloombits matcher -func (f *Filter) nextRequest() (bloombits uint, sections []uint64) { - bloomIndex, ok := f.matcher.AllocSectionQueue() - if !ok { - return 0, nil - } - if f.bloomBitsConfig.MaxRequestWait > 0 && - (f.bloomBitsConfig.MaxRequestLen <= 1 || // SectionCount is always greater than zero after a successful alloc - f.matcher.SectionCount(bloomIndex) < f.bloomBitsConfig.MaxRequestLen) { - time.Sleep(f.bloomBitsConfig.MaxRequestWait) - } - return bloomIndex, f.matcher.FetchSections(bloomIndex, f.bloomBitsConfig.MaxRequestLen) -} - -// serveMatcher serves the bloombits matcher by fetching the requested vectors -// through the filter backend -func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}, wg *sync.WaitGroup) chan error { - errChn := make(chan error, 1) - wg.Add(10) - for i := 0; i < 10; i++ { - go func(i int) { - defer wg.Done() - - for { - b, s := f.nextRequest() - if s == nil { - return - } - data, err := f.backend.GetBloomBits(ctx, uint64(b), s) - if err != nil { - select { - case errChn <- err: - case <-stop: - } - return - } - decomp := make([][]byte, len(data)) - for i, d := range data { - var err error - if decomp[i], err = f.decompress(d, int(f.bloomBitsConfig.SectionSize/8)); err != nil { - select { - case errChn <- err: - case <-stop: - } - return - } - } - f.matcher.Deliver(b, s, decomp) + if bloomFilter(header.Bloom, f.addresses, f.topics) { + found, err := f.checkMatches(ctx, header) + if err != nil { + return logs, err } - }(i) + logs = append(logs, found...) + } } - - return errChn + return logs, nil } // checkMatches checks if the receipts belonging to the given header contain any log events that @@ -192,83 +193,6 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ return nil, nil } -func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) { - haveBloomBitsBefore := f.backend.BloomBitsSections() * f.bloomBitsConfig.SectionSize - if haveBloomBitsBefore > start { - e := end - if haveBloomBitsBefore <= e { - e = haveBloomBitsBefore - 1 - } - - stop := make(chan struct{}) - var wg sync.WaitGroup - matches := f.matcher.Start(start, e) - errChn := f.serveMatcher(ctx, stop, &wg) - - defer func() { - f.matcher.Stop() - close(stop) - wg.Wait() - }() - - loop: - for { - select { - case i, ok := <-matches: - if !ok { - break loop - } - - blockNumber := rpc.BlockNumber(i) - header, err := f.backend.HeaderByNumber(ctx, blockNumber) - if header == nil || err != nil { - return logs, end, err - } - - logs, err := f.checkMatches(ctx, header) - if err != nil { - return nil, end, err - } - if logs != nil { - return logs, i, nil - } - case err := <-errChn: - return logs, end, err - case <-ctx.Done(): - return nil, end, ctx.Err() - } - } - - if end < haveBloomBitsBefore { - return logs, end, nil - } - start = haveBloomBitsBefore - } - - // search the rest with regular block-by-block bloom filtering - for i := start; i <= end; i++ { - blockNumber := rpc.BlockNumber(i) - header, err := f.backend.HeaderByNumber(ctx, blockNumber) - if header == nil || err != nil { - return logs, end, err - } - - // Use bloom filtering to see if this block is interesting given the - // current parameters - if f.bloomFilter(header.Bloom) { - logs, err := f.checkMatches(ctx, header) - if err != nil { - return nil, end, err - } - if logs != nil { - return logs, i, nil - } - } - } - - return logs, end, nil -} - func includes(addresses []common.Address, a common.Address) bool { for _, addr := range addresses { if addr == a { @@ -323,10 +247,6 @@ Logs: return ret } -func (f *Filter) bloomFilter(bloom types.Bloom) bool { - return bloomFilter(bloom, f.addresses, f.topics) -} - func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool { if len(addresses) > 0 { var included bool diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 140fad555c..664ce07a5f 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -20,12 +20,14 @@ import ( "context" "fmt" "math/big" + "math/rand" "reflect" "testing" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -85,29 +87,35 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } -func (b *testBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { - results := make([][]byte, len(sectionIdxList)) - var err error - for i, sectionIdx := range sectionIdxList { - sectionHead := core.GetCanonicalHash(b.db, (sectionIdx+1)*testBloomBitsSection-1) - results[i], err = core.GetBloomBits(b.db, bitIdx, sectionIdx, sectionHead) - if err != nil { - return nil, err +func (b *testBackend) BloomStatus() (uint64, uint64) { + return params.BloomBitsBlocks, b.sections +} + +func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + requests := make(chan chan *bloombits.Retrieval) + + go session.Multiplex(16, 0, requests) + go func() { + for { + // Wait for a service request or a shutdown + select { + case <-ctx.Done(): + return + + case request := <-requests: + task := <-request + + task.Bitsets = make([][]byte, len(task.Sections)) + for i, section := range task.Sections { + if rand.Int()%4 != 0 { // Handle occasional missing deliveries + head := core.GetCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1) + task.Bitsets[i] = core.GetBloomBits(b.db, task.Bit, section, head) + } + } + request <- task + } } - } - return results, nil -} - -func (b *testBackend) BloomBitsSections() uint64 { - return b.sections -} - -func (b *testBackend) BloomBitsConfig() BloomConfig { - return BloomConfig{ - SectionSize: testBloomBitsSection, - MaxRequestLen: 16, - MaxRequestWait: 0, - } + }() } // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. @@ -126,7 +134,7 @@ func TestBlockSubscription(t *testing.T) { logsFeed = new(event.Feed) chainFeed = new(event.Feed) backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false, 0) + api = NewPublicFilterAPI(backend, false) genesis = new(core.Genesis).MustCommit(db) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {}) chainEvents = []core.ChainEvent{} @@ -183,7 +191,7 @@ func TestPendingTxFilter(t *testing.T) { logsFeed = new(event.Feed) chainFeed = new(event.Feed) backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false, 0) + api = NewPublicFilterAPI(backend, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), new(big.Int), new(big.Int), nil), @@ -246,7 +254,7 @@ func TestLogFilterCreation(t *testing.T) { logsFeed = new(event.Feed) chainFeed = new(event.Feed) backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false, 0) + api = NewPublicFilterAPI(backend, false) testCases = []struct { crit FilterCriteria @@ -295,7 +303,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { logsFeed = new(event.Feed) chainFeed = new(event.Feed) backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false, 0) + api = NewPublicFilterAPI(backend, false) ) // different situations where log filter creation should fail. @@ -325,7 +333,7 @@ func TestLogFilter(t *testing.T) { logsFeed = new(event.Feed) chainFeed = new(event.Feed) backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false, 0) + api = NewPublicFilterAPI(backend, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -442,7 +450,7 @@ func TestPendingLogsSubscription(t *testing.T) { logsFeed = new(event.Feed) chainFeed = new(event.Feed) backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false, 0) + api = NewPublicFilterAPI(backend, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index f1c6481d7e..11235e95a7 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -32,8 +32,6 @@ import ( "github.com/ethereum/go-ethereum/params" ) -const testBloomBitsSection = 4096 - func makeReceipt(addr common.Address) *types.Receipt { receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -101,7 +99,7 @@ func BenchmarkFilters(b *testing.B) { filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { - logs, _ := filter.Find(context.Background()) + logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { b.Fatal("expected 4 logs, got", len(logs)) } @@ -189,13 +187,13 @@ func TestFilters(t *testing.T) { filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) - logs, _ := filter.Find(context.Background()) + logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) } @@ -204,7 +202,7 @@ func TestFilters(t *testing.T) { } filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) } @@ -214,7 +212,7 @@ func TestFilters(t *testing.T) { filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { t.Error("expected 2 log, got", len(logs)) } @@ -222,7 +220,7 @@ func TestFilters(t *testing.T) { failHash := common.BytesToHash([]byte("fail")) filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } @@ -230,14 +228,14 @@ func TestFilters(t *testing.T) { failAddr := common.BytesToAddress([]byte("failmenow")) filter = New(backend, 0, -1, []common.Address{failAddr}, nil) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } diff --git a/eth/handler.go b/eth/handler.go index aadd771df5..28ae208c06 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -49,8 +49,6 @@ const ( // txChanSize is the size of channel listening to TxPreEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 - - bloomBitsSection = 4096 ) var ( @@ -94,8 +92,6 @@ type ProtocolManager struct { quitSync chan struct{} noMorePeers chan struct{} - lesServer LesServer - // wait group is used for graceful shutdowns during downloading // and processing wg sync.WaitGroup @@ -118,7 +114,6 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } - // Figure out whether to allow fast sync or not if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 { log.Warn("Blockchain not empty, fast sync disabled") diff --git a/les/api_backend.go b/les/api_backend.go index c2ba270283..0d2d31b673 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -19,17 +19,16 @@ package les import ( "context" "math/big" - "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -174,18 +173,9 @@ func (b *LesApiBackend) AccountManager() *accounts.Manager { return b.eth.accountManager } -func (b *LesApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { - return nil, nil // implemented in a subsequent PR +func (b *LesApiBackend) BloomStatus() (uint64, uint64) { + return params.BloomBitsBlocks, 0 } -func (b *LesApiBackend) BloomBitsSections() uint64 { - return 0 -} - -func (b *LesApiBackend) BloomBitsConfig() filters.BloomConfig { - return filters.BloomConfig{ - SectionSize: 32768, - MaxRequestLen: 16, - MaxRequestWait: time.Microsecond * 100, - } +func (b *LesApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { } diff --git a/les/backend.go b/les/backend.go index a3670b5ac7..4c33417c03 100644 --- a/les/backend.go +++ b/les/backend.go @@ -169,7 +169,7 @@ func (s *LightEthereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, true, 0), + Service: filters.NewPublicFilterAPI(s.ApiBackend, true), Public: true, }, { Namespace: "net", diff --git a/params/network_params.go b/params/network_params.go new file mode 100644 index 0000000000..536a46d3df --- /dev/null +++ b/params/network_params.go @@ -0,0 +1,26 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package params + +// These are network parameters that need to be constant between clients, but +// aren't necesarilly consensus related. + +const ( + // BloomBitsBlocks is the number of blocks a single bloom bit section vector + // contains. + BloomBitsBlocks uint64 = 4096 +) From 6ff2c02991e60a8db54c2f60027442277fd889c0 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Wed, 6 Sep 2017 01:43:00 +0200 Subject: [PATCH 18/45] core/bloombits: AddBloom index parameter and fixes variable names --- core/bloombits/generator.go | 13 ++++++++----- core/bloombits/generator_test.go | 2 +- eth/bloombits.go | 2 +- eth/filters/bench_test.go | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/bloombits/generator.go b/core/bloombits/generator.go index 04a7f5146c..540085450d 100644 --- a/core/bloombits/generator.go +++ b/core/bloombits/generator.go @@ -49,21 +49,24 @@ func NewGenerator(sections uint) (*Generator, error) { // AddBloom takes a single bloom filter and sets the corresponding bit column // in memory accordingly. -func (b *Generator) AddBloom(bloom types.Bloom) error { +func (b *Generator) AddBloom(index uint, bloom types.Bloom) error { // Make sure we're not adding more bloom filters than our capacity if b.nextBit >= b.sections { return errSectionOutOfBounds } + if b.nextBit != index { + return errors.New("bloom filter with unexpected index") + } // Rotate the bloom and insert into our collection - byteMask := b.nextBit / 8 + byteIndex := b.nextBit / 8 bitMask := byte(1) << byte(7-b.nextBit%8) for i := 0; i < types.BloomBitLength; i++ { - bloomByteMask := types.BloomByteLength - 1 - i/8 + bloomByteIndex := types.BloomByteLength - 1 - i/8 bloomBitMask := byte(1) << byte(i%8) - if (bloom[bloomByteMask] & bloomBitMask) != 0 { - b.blooms[i][byteMask] |= bitMask + if (bloom[bloomByteIndex] & bloomBitMask) != 0 { + b.blooms[i][byteIndex] |= bitMask } } b.nextBit++ diff --git a/core/bloombits/generator_test.go b/core/bloombits/generator_test.go index f4aa9551c2..f9bcef96e0 100644 --- a/core/bloombits/generator_test.go +++ b/core/bloombits/generator_test.go @@ -44,7 +44,7 @@ func TestGenerator(t *testing.T) { t.Fatalf("failed to create bloombit generator: %v", err) } for i, bloom := range input { - if err := gen.AddBloom(bloom); err != nil { + if err := gen.AddBloom(uint(i), bloom); err != nil { t.Fatalf("bloom %d: failed to add: %v", i, err) } } diff --git a/eth/bloombits.go b/eth/bloombits.go index 02de408b0e..32f6c7b318 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -122,7 +122,7 @@ func (b *BloomIndexer) Reset(section uint64) { // Process implements core.ChainIndexerBackend, adding a new header's bloom into // the index. func (b *BloomIndexer) Process(header *types.Header) { - b.gen.AddBloom(header.Bloom) + b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) b.head = header.Hash() } diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index d994378fc1..abbf4593e9 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -98,7 +98,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { if header == nil { b.Fatalf("Error creating bloomBits data") } - bc.AddBloom(header.Bloom) + bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) } sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) for i := 0; i < types.BloomBitLength; i++ { From 451ffdb62b43bab66188f3f7eeb2131b65415ccb Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Wed, 6 Sep 2017 02:33:10 +0200 Subject: [PATCH 19/45] core/bloombits: use general filters instead of addresses and topics --- core/bloombits/matcher.go | 56 +++++++++------------------------- core/bloombits/matcher_test.go | 6 ++-- eth/filters/filter.go | 19 +++++++++++- 3 files changed, 34 insertions(+), 47 deletions(-) diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index e365fd6d04..df0967a124 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -24,7 +24,6 @@ import ( "sync/atomic" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/crypto" ) @@ -68,8 +67,7 @@ type Retrieval struct { type Matcher struct { sectionSize uint64 // Size of the data batches to filter on - addresses []bloomIndexes // Addresses the system is filtering for - topics [][]bloomIndexes // Topics the system is filtering for + filters [][]bloomIndexes // Filter the system is matching for schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits retrievers chan chan uint // Retriever processes waiting for bit allocations @@ -82,7 +80,8 @@ type Matcher struct { // NewMatcher creates a new pipeline for retrieving bloom bit streams and doing // address and topic filtering on them. -func NewMatcher(sectionSize uint64, addresses []common.Address, topics [][]common.Hash) *Matcher { +func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher { + // Create the matcher instance m := &Matcher{ sectionSize: sectionSize, schedulers: make(map[uint]*scheduler), @@ -91,48 +90,25 @@ func NewMatcher(sectionSize uint64, addresses []common.Address, topics [][]commo retrievals: make(chan chan *Retrieval), deliveries: make(chan *Retrieval), } - m.setAddresses(addresses) - m.setTopics(topics) - return m -} + // Calculate the bloom bit indexes for the groups we're interested in + m.filters = nil -// setAddresses configures the matcher to only return logs that are generated -// from addresses that are included in the given list. -func (m *Matcher) setAddresses(addresses []common.Address) { - // Calculate the bloom bit indexes for the addresses we're interested in - m.addresses = make([]bloomIndexes, len(addresses)) - for i, address := range addresses { - m.addresses[i] = calcBloomIndexes(address.Bytes()) + for _, filter := range filters { + bloomBits := make([]bloomIndexes, len(filter)) + for i, clause := range filter { + bloomBits[i] = calcBloomIndexes(clause) + } + m.filters = append(m.filters, bloomBits) } // For every bit, create a scheduler to load/download the bit vectors - for _, bloomIndexList := range m.addresses { - for _, bloomIndex := range bloomIndexList { - m.addScheduler(bloomIndex) - } - } -} - -// setTopics configures the matcher to only return logs that have topics matching -// the given list. -func (m *Matcher) setTopics(topicsList [][]common.Hash) { - // Calculate the bloom bit indexes for the topics we're interested in - m.topics = nil - - for _, topics := range topicsList { - bloomBits := make([]bloomIndexes, len(topics)) - for i, topic := range topics { - bloomBits[i] = calcBloomIndexes(topic.Bytes()) - } - m.topics = append(m.topics, bloomBits) - } - // For every bit, create a scheduler to load/download the bit vectors - for _, bloomIndexLists := range m.topics { + for _, bloomIndexLists := range m.filters { for _, bloomIndexList := range bloomIndexLists { for _, bloomIndex := range bloomIndexList { m.addScheduler(bloomIndex) } } } + return m } // addScheduler adds a bit stream retrieval scheduler for the given bit index if @@ -250,14 +226,10 @@ func (m *Matcher) run(begin, end uint64, buffer int, session *MatcherSession) ch } }() // Assemble the daisy-chained filtering pipeline - blooms := m.topics - if len(m.addresses) > 0 { - blooms = append([][]bloomIndexes{m.addresses}, blooms...) - } next := source dist := make(chan *request, buffer) - for _, bloom := range blooms { + for _, bloom := range m.filters { next = m.subMatch(next, dist, bloom, session) } // Start the request distribution diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index fc49b43b8f..177e1b792b 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -94,10 +94,8 @@ func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, // number of requests made for cross validation between different modes. func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { // Create a new matcher an simulate our explicit random bitsets - matcher := NewMatcher(testSectionSize, nil, nil) - - matcher.addresses = filter[0] - matcher.topics = filter[1:] + matcher := NewMatcher(testSectionSize, nil) + matcher.filters = filter for _, rule := range filter { for _, topic := range rule { diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 3a2226f6be..4f6c30058e 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -60,6 +60,23 @@ type Filter struct { // New creates a new filter which uses a bloom filter on blocks to figure out whether // a particular block is interesting or not. func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { + // Flatten the address and topic filter clauses into a single filter system + var filters [][][]byte + if len(addresses) > 0 { + filter := make([][]byte, len(addresses)) + for i, address := range addresses { + filter[i] = address.Bytes() + } + filters = append(filters, filter) + } + for _, topicList := range topics { + filter := make([][]byte, len(topicList)) + for i, topic := range topicList { + filter[i] = topic.Bytes() + } + filters = append(filters, filter) + } + // Assemble and return the filter size, _ := backend.BloomStatus() return &Filter{ @@ -69,7 +86,7 @@ func New(backend Backend, begin, end int64, addresses []common.Address, topics [ addresses: addresses, topics: topics, db: backend.ChainDb(), - matcher: bloombits.NewMatcher(size, addresses, topics), + matcher: bloombits.NewMatcher(size, filters), } } From 564c8f3ae6f80d039ef27479e5ad15145f488710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 6 Sep 2017 11:12:53 +0300 Subject: [PATCH 20/45] core/bloombits: drop nil-matcher special case --- core/bloombits/matcher.go | 12 ++---------- core/bloombits/matcher_test.go | 5 +++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index df0967a124..f3ed405a61 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -17,6 +17,7 @@ package bloombits import ( + "bytes" "errors" "math" "sort" @@ -171,15 +172,6 @@ func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession } // Iterate over all the blocks in the section and return the matching ones for i := first; i <= last; i++ { - // If the bitset is nil, we're a special match-all cornercase - if res.bitset == nil { - select { - case <-session.quit: - return - case results <- i: - } - continue - } // Skip the entire byte if no matches are found inside next := res.bitset[(i-sectionStart)/8] if next == 0 { @@ -221,7 +213,7 @@ func (m *Matcher) run(begin, end uint64, buffer int, session *MatcherSession) ch select { case <-session.quit: return - case source <- &partialMatches{i, nil}: + case source <- &partialMatches{i, bytes.Repeat([]byte{0xff}, int(m.sectionSize/8))}: } } }() diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index 177e1b792b..f0198c4e38 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -51,6 +51,11 @@ func TestMatcherRandom(t *testing.T) { } } +// Tests that matching on everything doesn't crash (special case internally). +func TestWildcardMatcher(t *testing.T) { + testMatcherBothModes(t, nil, 10000, 0) +} + // makeRandomIndexes generates a random filter system, composed on multiple filter // criteria, each having one bloom list component for the address and arbitrarilly // many topic bloom list components. From f30179d62e239521290690ac0688da8648ddebc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 6 Sep 2017 15:02:44 +0300 Subject: [PATCH 21/45] eth: disable fast sync after pivot is committed --- eth/downloader/downloader.go | 3 +-- eth/sync.go | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 6ac58140a3..f2dbf0f88a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -1170,7 +1170,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // If we're already past the pivot point, this could be an attack, thread carefully if rollback[len(rollback)-1].Number.Uint64() > pivot { - // If we didn't ever fail, lock in te pivot header (must! not! change!) + // If we didn't ever fail, lock in the pivot header (must! not! change!) if atomic.LoadUint32(&d.fsPivotFails) == 0 { for _, header := range rollback { if header.Number.Uint64() == pivot { @@ -1392,7 +1392,6 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { stateSync.Cancel() if err := d.commitPivotBlock(P); err != nil { return err - } } if err := d.importBlockResults(afterP); err != nil { diff --git a/eth/sync.go b/eth/sync.go index 7442f912c7..a8ae646170 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -188,7 +188,17 @@ func (pm *ProtocolManager) synchronise(peer *peer) { atomic.StoreUint32(&pm.fastSync, 1) mode = downloader.FastSync } - if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { + // Run the sync cycle, and disable fast sync if we've went past the pivot block + err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode) + + if atomic.LoadUint32(&pm.fastSync) == 1 { + // Disable fast sync if we indeed have something in our chain + if pm.blockchain.CurrentBlock().NumberU64() > 0 { + log.Info("Fast sync complete, auto disabling") + atomic.StoreUint32(&pm.fastSync, 0) + } + } + if err != nil { return } atomic.StoreUint32(&pm.acceptTxs, 1) // Mark initial sync done @@ -201,12 +211,4 @@ func (pm *ProtocolManager) synchronise(peer *peer) { // more reliably update peers or the local TD state. go pm.BroadcastBlock(head, false) } - // If fast sync was enabled, and we synced up, disable it - if atomic.LoadUint32(&pm.fastSync) == 1 { - // Disable fast sync if we indeed have something in our chain - if pm.blockchain.CurrentBlock().NumberU64() > 0 { - log.Info("Fast sync complete, auto disabling") - atomic.StoreUint32(&pm.fastSync, 0) - } - } } From 69c8be7c86d5747e9ef2a59ed769d769fd9249f5 Mon Sep 17 00:00:00 2001 From: Pawan Singh Pal Date: Fri, 8 Sep 2017 02:38:06 +0530 Subject: [PATCH 22/45] core: delete dao.go (#15113) - dao.go is already present in consensus/misc - core/dao.go is not used anywhere in the codebase --- core/dao.go | 75 ----------------------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 core/dao.go diff --git a/core/dao.go b/core/dao.go deleted file mode 100644 index ff42a0e9d3..0000000000 --- a/core/dao.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// ValidateDAOHeaderExtraData validates the extra-data field of a block header to -// ensure it conforms to DAO hard-fork rules. -// -// DAO hard-fork extension to the header validity: -// a) if the node is no-fork, do not accept blocks in the [fork, fork+10) range -// with the fork specific extra-data set -// b) if the node is pro-fork, require blocks in the specific range to have the -// unique extra-data set. -func ValidateDAOHeaderExtraData(config *params.ChainConfig, header *types.Header) error { - // Short circuit validation if the node doesn't care about the DAO fork - if config.DAOForkBlock == nil { - return nil - } - // Make sure the block is within the fork's modified extra-data range - limit := new(big.Int).Add(config.DAOForkBlock, params.DAOForkExtraRange) - if header.Number.Cmp(config.DAOForkBlock) < 0 || header.Number.Cmp(limit) >= 0 { - return nil - } - // Depending whether we support or oppose the fork, validate the extra-data contents - if config.DAOForkSupport { - if !bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - return fmt.Errorf("DAO pro-fork bad block extra-data: 0x%x", header.Extra) - } - } else { - if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - return fmt.Errorf("DAO no-fork bad block extra-data: 0x%x", header.Extra) - } - } - // All ok, header has the same extra-data we expect - return nil -} - -// ApplyDAOHardFork modifies the state database according to the DAO hard-fork -// rules, transferring all balances of a set of DAO accounts to a single refund -// contract. -func ApplyDAOHardFork(statedb *state.StateDB) { - // Retrieve the contract to refund balances into - if !statedb.Exist(params.DAORefundContract) { - statedb.CreateAccount(params.DAORefundContract) - } - - // Move every DAO account and extra-balance account funds into the refund contract - for _, addr := range params.DAODrainList() { - statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr)) - statedb.SetBalance(addr, new(big.Int)) - } -} From 2dcb22afec1616c3eb8897be96f01b0a5ab185c9 Mon Sep 17 00:00:00 2001 From: Fiisio Date: Fri, 8 Sep 2017 05:10:51 +0800 Subject: [PATCH 23/45] swarm/fuse: use Equal instead of Compare (#15097) --- swarm/fuse/swarmfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 69f3cc6157..93f1d4c2fd 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -140,7 +140,7 @@ func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo if err != nil { t.Fatalf("Could not readfile %v : %v", fname, err) } - if bytes.Compare(fileContents, finfo.contents) != 0 { + if !bytes.Equal(fileContents, finfo.contents) { t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents) } From 02b4d074f688293a3443d5ca73c357c1ded22ddb Mon Sep 17 00:00:00 2001 From: Fiisio Date: Fri, 8 Sep 2017 05:11:48 +0800 Subject: [PATCH 24/45] core/asm: use ContainsRune instead of IndexRune (#15098) --- core/asm/lexer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/asm/lexer.go b/core/asm/lexer.go index d784e5d504..a34b2cbd8f 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -145,7 +145,7 @@ func (l *lexer) ignore() { // Accepts checks whether the given input matches the next rune func (l *lexer) accept(valid string) bool { - if strings.IndexRune(valid, l.next()) >= 0 { + if strings.ContainsRune(valid, l.next()) { return true } @@ -157,7 +157,7 @@ func (l *lexer) accept(valid string) bool { // acceptRun will continue to advance the seeker until valid // can no longer be met. func (l *lexer) acceptRun(valid string) { - for strings.IndexRune(valid, l.next()) >= 0 { + for strings.ContainsRune(valid, l.next()) { } l.backup() } @@ -166,7 +166,7 @@ func (l *lexer) acceptRun(valid string) { // to advance the seeker until the rune has been found. func (l *lexer) acceptRunUntil(until rune) bool { // Continues running until a rune is found - for i := l.next(); strings.IndexRune(string(until), i) == -1; i = l.next() { + for i := l.next(); !strings.ContainsRune(string(until), i); i = l.next() { if i == 0 { return false } From e3db1236de65619d26e87eb7c886e565fcc2eafc Mon Sep 17 00:00:00 2001 From: Benoit Verkindt Date: Thu, 7 Sep 2017 14:14:47 -0700 Subject: [PATCH 25/45] contracts/chequebook: fix race in test (#15058) --- contracts/chequebook/cheque_test.go | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 5f6a54a1c1..b7555d0815 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -232,8 +232,8 @@ func TestDeposit(t *testing.T) { balance := new(big.Int).SetUint64(42) chbook.Deposit(balance) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } amount := common.Big1 @@ -243,8 +243,8 @@ func TestDeposit(t *testing.T) { } backend.Commit() exp := new(big.Int).SetUint64(41) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } // autodeposit on each issue @@ -259,8 +259,8 @@ func TestDeposit(t *testing.T) { t.Fatalf("expected no error, got %v", err) } backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } // autodeposit off @@ -277,11 +277,11 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(40) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } - // autodeposit every 10ms if new cheque issued + // autodeposit every 30ms if new cheque issued interval := 30 * time.Millisecond chbook.AutoDeposit(interval, common.Big1, balance) _, err = chbook.Issue(addr1, amount) @@ -296,14 +296,14 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(38) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } time.Sleep(3 * interval) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } exp = new(big.Int).SetUint64(40) @@ -319,8 +319,8 @@ func TestDeposit(t *testing.T) { } time.Sleep(3 * interval) backend.Commit() - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } _, err = chbook.Issue(addr1, amount) @@ -330,8 +330,8 @@ func TestDeposit(t *testing.T) { time.Sleep(1 * interval) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } chbook.AutoDeposit(1*interval, common.Big0, balance) @@ -352,8 +352,8 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(39) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } } From c1740e454015e61e8207cdeb34c48eee3adbb4d3 Mon Sep 17 00:00:00 2001 From: Mark Date: Thu, 7 Sep 2017 16:22:27 -0500 Subject: [PATCH 26/45] core/types, miner: avoid tx sender miscaching (#14773) --- core/types/transaction.go | 22 ++++++++++++---------- core/types/transaction_test.go | 2 +- miner/worker.go | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 947fc85d60..7f54860fc0 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -381,28 +381,32 @@ func (s *TxByPrice) Pop() interface{} { // transactions in a profit-maximising sorted order, while supporting removing // entire batches of transactions for non-executable accounts. type TransactionsByPriceAndNonce struct { - txs map[common.Address]Transactions // Per account nonce-sorted list of transactions - heads TxByPrice // Next transaction for each unique account (price heap) + txs map[common.Address]Transactions // Per account nonce-sorted list of transactions + heads TxByPrice // Next transaction for each unique account (price heap) + signer Signer // Signer for the set of transactions } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve // price sorted transactions in a nonce-honouring way. // // Note, the input map is reowned so the caller should not interact any more with -// if after providng it to the constructor. -func NewTransactionsByPriceAndNonce(txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { +// if after providing it to the constructor. +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { // Initialize a price based heap with the head transactions heads := make(TxByPrice, 0, len(txs)) - for acc, accTxs := range txs { + for _, accTxs := range txs { heads = append(heads, accTxs[0]) + // Ensure the sender address is from the signer + acc, _ := Sender(signer, accTxs[0]) txs[acc] = accTxs[1:] } heap.Init(&heads) // Assemble and return the transaction set return &TransactionsByPriceAndNonce{ - txs: txs, - heads: heads, + txs: txs, + heads: heads, + signer: signer, } } @@ -416,9 +420,7 @@ func (t *TransactionsByPriceAndNonce) Peek() *Transaction { // Shift replaces the current best head with the next one from the same account. func (t *TransactionsByPriceAndNonce) Shift() { - signer := deriveSigner(t.heads[0].data.V) - // derive signer but don't cache. - acc, _ := Sender(signer, t.heads[0]) // we only sort valid txs so this cannot fail + acc, _ := Sender(t.signer, t.heads[0]) if txs, ok := t.txs[acc]; ok && len(txs) > 0 { t.heads[0], t.txs[acc] = txs[0], txs[1:] heap.Fix(&t.heads, 0) diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index edbe171e6f..df9d7ffd18 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -143,7 +143,7 @@ func TestTransactionPriceNonceSort(t *testing.T) { } } // Sort the transactions and cross check the nonce ordering - txset := NewTransactionsByPriceAndNonce(groups) + txset := NewTransactionsByPriceAndNonce(signer, groups) txs := Transactions{} for { diff --git a/miner/worker.go b/miner/worker.go index e1154ac068..2919d4a485 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -268,7 +268,7 @@ func (self *worker) update() { self.currentMu.Lock() acc, _ := types.Sender(self.current.signer, ev.Tx) txs := map[common.Address]types.Transactions{acc: {ev.Tx}} - txset := types.NewTransactionsByPriceAndNonce(txs) + txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) self.currentMu.Unlock() @@ -471,7 +471,7 @@ func (self *worker) commitNewWork() { log.Error("Failed to fetch pending transactions", "err", err) return } - txs := types.NewTransactionsByPriceAndNonce(pending) + txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) work.commitTransactions(self.mux, txs, self.chain, self.coinbase) // compute uncles for the new block. From fc87bc5f523ad78974e0a6ce1c5651b98a1e8a52 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 7 Sep 2017 23:32:59 +0200 Subject: [PATCH 27/45] common: improve documentation of Hash.SetBytes (#15062) Fixes #15004 --- common/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/types.go b/common/types.go index eaf8352fbc..d31bbf741b 100644 --- a/common/types.go +++ b/common/types.go @@ -88,7 +88,7 @@ func (h Hash) MarshalText() ([]byte, error) { return hexutil.Bytes(h[:]).MarshalText() } -// Sets the hash to the value of b. If b is larger than len(h) it will panic +// Sets the hash to the value of b. If b is larger than len(h), 'b' will be cropped (from the left). func (h *Hash) SetBytes(b []byte) { if len(b) > len(h) { b = b[len(b)-HashLength:] @@ -97,7 +97,7 @@ func (h *Hash) SetBytes(b []byte) { copy(h[HashLength-len(b):], b) } -// Set string `s` to h. If s is larger than len(h) it will panic +// Set string `s` to h. If s is larger than len(h) s will be cropped (from left) to fit. func (h *Hash) SetString(s string) { h.SetBytes([]byte(s)) } // Sets h to other From 5ba9225fe3b06d5512976b808bc87d92d03d54ba Mon Sep 17 00:00:00 2001 From: nkbai Date: Fri, 8 Sep 2017 05:34:45 +0800 Subject: [PATCH 28/45] accounts/abi/bind: pass non-empty directory when calling goimports (#15070) --- accounts/abi/bind/bind.go | 2 +- accounts/abi/bind/bind_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 73e95e02a1..f587580884 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -122,7 +122,7 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La } // For Go bindings pass the code through goimports to clean it up and double check if lang == LangGo { - code, err := imports.Process("", buffer.Bytes(), nil) + code, err := imports.Process(".", buffer.Bytes(), nil) if err != nil { return "", fmt.Errorf("%v\n%s", err, buffer) } diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 8ea3eca415..43ed53b922 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -459,7 +459,7 @@ func TestBindings(t *testing.T) { } // Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845) linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}") - linkTestDeps, err := imports.Process("", []byte(linkTestCode), nil) + linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil) if err != nil { t.Fatalf("failed check for goimports symlink bug: %v", err) } From ac193e36ce4bce752717124433a8ce84c347dbf7 Mon Sep 17 00:00:00 2001 From: holisticode Date: Fri, 8 Sep 2017 13:29:09 -0500 Subject: [PATCH 29/45] swarm/api/http: add error pages (#14967) --- swarm/api/api.go | 1 + swarm/api/http/error.go | 116 +++++++++ swarm/api/http/error_templates.go | 376 ++++++++++++++++++++++++++++++ swarm/api/http/error_test.go | 143 ++++++++++++ swarm/api/http/server.go | 37 +-- swarm/api/http/server_test.go | 9 +- 6 files changed, 662 insertions(+), 20 deletions(-) create mode 100644 swarm/api/http/error.go create mode 100644 swarm/api/http/error_templates.go create mode 100644 swarm/api/http/error_test.go diff --git a/swarm/api/api.go b/swarm/api/api.go index 7d185ab3ca..a5941fb5cd 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -132,6 +132,7 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) { func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { trie, err := loadManifest(self.dpa, key, nil) if err != nil { + status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return } diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go new file mode 100644 index 0000000000..ebb5e3ebe2 --- /dev/null +++ b/swarm/api/http/error.go @@ -0,0 +1,116 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Show nicely (but simple) formatted HTML error pages (or respond with JSON +if the appropriate `Accept` header is set)) for the http package. +*/ +package http + +import ( + "encoding/json" + "fmt" + "html/template" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +//templateMap holds a mapping of an HTTP error code to a template +var templateMap map[int]*template.Template + +//parameters needed for formatting the correct HTML page +type ErrorParams struct { + Msg string + Code int + Timestamp string + template *template.Template + Details template.HTML +} + +//we init the error handling right on boot time, so lookup and http response is fast +func init() { + initErrHandling() +} + +func initErrHandling() { + //pages are saved as strings - get these strings + genErrPage := GetGenericErrorPage() + notFoundPage := GetNotFoundErrorPage() + //map the codes to the available pages + tnames := map[int]string{ + 0: genErrPage, //default + 400: genErrPage, + 404: notFoundPage, + 500: genErrPage, + } + templateMap = make(map[int]*template.Template) + for code, tname := range tnames { + //assign formatted HTML to the code + templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname)) + } +} + +//ShowError is used to show an HTML error page to a client. +//If there is an `Accept` header of `application/json`, JSON will be returned instead +//The function just takes a string message which will be displayed in the error page. +//The code is used to evaluate which template will be displayed +//(and return the correct HTTP status code) +func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { + if code == http.StatusInternalServerError { + log.Error(msg) + } + respond(w, r, &ErrorParams{ + Code: code, + Msg: msg, + Timestamp: time.Now().Format(time.RFC1123), + template: getTemplate(code), + }) +} + +//evaluate if client accepts html or json response +func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) { + w.WriteHeader(params.Code) + if r.Header.Get("Accept") == "application/json" { + respondJson(w, params) + } else { + respondHtml(w, params) + } +} + +//return a HTML page +func respondHtml(w http.ResponseWriter, params *ErrorParams) { + err := params.template.Execute(w, params) + if err != nil { + log.Error(err.Error()) + } +} + +//return JSON +func respondJson(w http.ResponseWriter, params *ErrorParams) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(params) +} + +//get the HTML template for a given code +func getTemplate(code int) *template.Template { + if val, tmpl := templateMap[code]; tmpl { + return val + } else { + return templateMap[0] + } +} diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go new file mode 100644 index 0000000000..29bd3bfbb1 --- /dev/null +++ b/swarm/api/http/error_templates.go @@ -0,0 +1,376 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +We use html templates to handle simple but as informative as possible error pages. + +To eliminate circular dependency in case of an error, we don't store error pages on swarm. +We can't save the error pages as html files on disk, or when deploying compiled binaries +they won't be found. + +For this reason we resort to save the HTML error pages as strings, which then can be +parsed by Go's html/template package +*/ +package http + +//This returns the HTML for generic errors +func GetGenericErrorPage() string { + page := ` + + + + + + + + + + + + Swarm::HTTP Error Page + + + + +
+ +
+
+ +
+
+

There was a problem serving the requested page

+
+
+
{{.Timestamp}}
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
+ Hmmmmm....Swarm was not able to serve your request! +
+ Error message: +
+ {{.Msg}} +
+ Error code: +
+ {{.Code}} +
+
+ + +
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + +
+ + + +` + return page +} + +//This returns the HTML for a 404 Not Found error +func GetNotFoundErrorPage() string { + page := ` + + + + + + + + + + + + Swarm::404 HTTP Not Found + + + + +
+ +
+
+ +
+
+

Resource Not Found

+
+
+
{{.Timestamp}}
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
+ Unfortunately, the resource you were trying to access could not be found on swarm. +
+
+ {{.Msg}} +
+ Error code: +
+ {{.Code}} +
+
+ + +
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + +
+ + + +` + return page +} diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go new file mode 100644 index 0000000000..465fbf4cad --- /dev/null +++ b/swarm/api/http/error_test.go @@ -0,0 +1,143 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package http_test + +import ( + "encoding/json" + "golang.org/x/net/html" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +func TestError(t *testing.T) { + + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/this_should_fail_as_no_bzz_protocol_present" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") { + t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func Test404Page(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/1234567890123456789012345678901234567890123456789012345678901234" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 404 || !strings.Contains(string(respbody), "404") { + t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func Test500Page(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/thisShouldFailWith500Code" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 500 || !strings.Contains(string(respbody), "500") { + t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func TestJsonResponse(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/thisShouldFailWith500Code/" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + req.Header.Set("Accept", "application/json") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 500 { + t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + } + + if !isJSON(string(respbody)) { + t.Fatalf("Expected repsonse to be JSON, received invalid JSON: %s", string(respbody)) + } + +} + +func isJSON(s string) bool { + var js map[string]interface{} + return json.Unmarshal([]byte(s), &js) == nil +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 0b4ec7e18e..a637b87351 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -332,7 +332,7 @@ func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) @@ -341,8 +341,7 @@ func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { - s.logDebug("key not found %s: %s", key, err) - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) return } @@ -534,16 +533,20 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { return } - reader, contentType, _, err := s.api.Get(key, r.uri.Path) + reader, contentType, status, err := s.api.Get(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + switch status { + case http.StatusNotFound: + s.NotFound(w, r, err) + default: + s.Error(w, r, err) + } return } // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { - s.logDebug("file not found %s: %s", r.uri, err) - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) return } @@ -556,14 +559,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + req := &Request{Request: *r, uri: uri} if err != nil { s.logError("Invalid URI %q: %s", r.URL.Path, err) - http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest) + s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) return } s.logDebug("%s request received for %s", r.Method, uri) - req := &Request{Request: *r, uri: uri} switch r.Method { case "POST": if uri.Raw() { @@ -579,7 +582,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // strictly a traditional PUT request which replaces content // at a URI, and POST is more ubiquitous) if uri.Raw() { - http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return } else { s.HandlePostFiles(w, req) @@ -587,7 +590,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "DELETE": if uri.Raw() { - http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) return } s.HandleDelete(w, req) @@ -611,7 +614,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.HandleGetFile(w, req) default: - http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + ShowError(w, r, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed) } } @@ -643,11 +646,13 @@ func (s *Server) logError(format string, v ...interface{}) { } func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { - s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason) - http.Error(w, reason, http.StatusBadRequest) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest) } func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { - s.logError("error serving %s %s: %s", r.Method, r.uri, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) +} + +func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) { + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index d3374594b6..ffeaf6e0d8 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "sync" "testing" @@ -110,9 +111,9 @@ func TestBzzrGetPath(t *testing.T) { } nonhashresponses := []string{ - "error resolving name: no DNS to resolve name: \"name\"\n", - "error resolving nonhash: immutable address not a content hash: \"nonhash\"\n", - "error resolving nonhash: no DNS to resolve name: \"nonhash\"\n", + "error resolving name: no DNS to resolve name: "name"", + "error resolving nonhash: immutable address not a content hash: "nonhash"", + "error resolving nonhash: no DNS to resolve name: "nonhash"", } for i, url := range nonhashtests { @@ -129,7 +130,7 @@ func TestBzzrGetPath(t *testing.T) { if err != nil { t.Fatalf("ReadAll failed: %v", err) } - if string(respbody) != nonhashresponses[i] { + if !strings.Contains(string(respbody), nonhashresponses[i]) { t.Fatalf("Non-Hash response body does not match, expected: %v, got: %v", nonhashresponses[i], string(respbody)) } } From 10181b57a9fb648f5fd424ca611820a3cf42c42b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 9 Sep 2017 18:03:07 +0200 Subject: [PATCH 30/45] core, eth/downloader: commit block data using batches (#15115) * ethdb: add Putter interface and Has method * ethdb: improve docs and add IdealBatchSize * ethdb: remove memory batch lock Batches are not safe for concurrent use. * core: use ethdb.Putter for Write* functions This covers the easy cases. * core/state: simplify StateSync * trie: optimize local node check * ethdb: add ValueSize to Batch * core: optimize HasHeader check This avoids one random database read get the block number. For many uses of HasHeader, the expectation is that it's actually there. Using Has avoids a load + decode of the value. * core: write fast sync block data in batches Collect writes into batches up to the ideal size instead of issuing many small, concurrent writes. * eth/downloader: commit larger state batches Collect nodes into a batch up to the ideal size instead of committing whenever a node is received. * core: optimize HasBlock check This avoids a random database read to get the number. * core: use numberCache in HasHeader numberCache has higher capacity, increasing the odds of finding the header without a database lookup. * core: write imported block data using a batch Restore batch writes of state and add blocks, tx entries, receipts to the same batch. The change also simplifies the miner. This commit also removes posting of logs when a forked block is imported. * core: fix DB write error handling * ethdb: use RLock for Has * core: fix HasBlock comment --- cmd/utils/cmd.go | 2 +- core/blockchain.go | 224 +++++++++++++----------------- core/database_util.go | 53 ++++--- core/headerchain.go | 13 +- core/state/sync.go | 42 +----- eth/api.go | 2 +- eth/downloader/downloader.go | 8 +- eth/downloader/downloader_test.go | 2 +- eth/downloader/statesync.go | 77 +++++----- eth/handler.go | 4 +- ethdb/database.go | 24 +++- ethdb/interface.go | 18 ++- ethdb/memory_database.go | 21 ++- les/handler.go | 2 +- light/lightchain.go | 8 +- miner/worker.go | 21 +-- trie/sync.go | 5 +- trie/trie.go | 1 + 18 files changed, 247 insertions(+), 280 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 17c258c6c8..23b10c2d74 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -164,7 +164,7 @@ func ImportChain(chain *core.BlockChain, fn string) error { func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { for _, b := range bs { - if !chain.HasBlock(b.Hash()) { + if !chain.HasBlock(b.Hash(), b.NumberU64()) { return false } } diff --git a/core/blockchain.go b/core/blockchain.go index d74b3520b1..25be8d762a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -23,7 +23,6 @@ import ( "io" "math/big" mrand "math/rand" - "runtime" "sync" "sync/atomic" "time" @@ -515,10 +514,13 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { return body } -// HasBlock checks if a block is fully present in the database or not, caching -// it if present. -func (bc *BlockChain) HasBlock(hash common.Hash) bool { - return bc.GetBlockByHash(hash) != nil +// HasBlock checks if a block is fully present in the database or not. +func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { + if bc.blockCache.Contains(hash) { + return true + } + ok, _ := bc.chainDb.Has(blockBodyKey(hash, number)) + return ok } // HasBlockAndState checks if a block and associated state trie is fully present @@ -693,108 +695,73 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty // InsertReceiptChain attempts to complete an already existing header chain with // transaction and receipt data. -// XXX should this be moved to the test? func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { + bc.wg.Add(1) + defer bc.wg.Done() + // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(blockChain); i++ { if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() { - // Chain broke ancestry, log a messge (programming error) and skip insertion log.Error("Non contiguous receipt insert", "number", blockChain[i].Number(), "hash", blockChain[i].Hash(), "parent", blockChain[i].ParentHash(), "prevnumber", blockChain[i-1].Number(), "prevhash", blockChain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, blockChain[i-1].NumberU64(), blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4]) } } - // Pre-checks passed, start the block body and receipt imports - bc.wg.Add(1) - defer bc.wg.Done() - // Collect some import statistics to report on - stats := struct{ processed, ignored int32 }{} - start := time.Now() + var ( + stats = struct{ processed, ignored int32 }{} + start = time.Now() + bytes = 0 + batch = bc.chainDb.NewBatch() + ) + for i, block := range blockChain { + receipts := receiptChain[i] + // Short circuit insertion if shutting down or processing failed + if atomic.LoadInt32(&bc.procInterrupt) == 1 { + return 0, nil + } + // Short circuit if the owner header is unknown + if !bc.HasHeader(block.Hash(), block.NumberU64()) { + return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) + } + // Skip if the entire data is already known + if bc.HasBlock(block.Hash(), block.NumberU64()) { + stats.ignored++ + continue + } + // Compute all the non-consensus fields of the receipts + SetReceiptsData(bc.config, block, receipts) + // Write all the data out into the database + if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil { + return i, fmt.Errorf("failed to write block body: %v", err) + } + if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { + return i, fmt.Errorf("failed to write block receipts: %v", err) + } + if err := WriteTxLookupEntries(batch, block); err != nil { + return i, fmt.Errorf("failed to write lookup metadata: %v", err) + } + stats.processed++ - // Create the block importing task queue and worker functions - tasks := make(chan int, len(blockChain)) - for i := 0; i < len(blockChain) && i < len(receiptChain); i++ { - tasks <- i - } - close(tasks) - - errs, failed := make([]error, len(tasks)), int32(0) - process := func(worker int) { - for index := range tasks { - block, receipts := blockChain[index], receiptChain[index] - - // Short circuit insertion if shutting down or processing failed - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - return + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return 0, err } - if atomic.LoadInt32(&failed) > 0 { - return - } - // Short circuit if the owner header is unknown - if !bc.HasHeader(block.Hash()) { - errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) - atomic.AddInt32(&failed, 1) - return - } - // Skip if the entire data is already known - if bc.HasBlock(block.Hash()) { - atomic.AddInt32(&stats.ignored, 1) - continue - } - // Compute all the non-consensus fields of the receipts - SetReceiptsData(bc.config, block, receipts) - // Write all the data out into the database - if err := WriteBody(bc.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { - errs[index] = fmt.Errorf("failed to write block body: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write block body", "err", err) - return - } - if err := WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - errs[index] = fmt.Errorf("failed to write block receipts: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write block receipts", "err", err) - return - } - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { - errs[index] = fmt.Errorf("failed to write lookup metadata: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write lookup metadata", "err", err) - return - } - atomic.AddInt32(&stats.processed, 1) + bytes += batch.ValueSize() + batch = bc.chainDb.NewBatch() } } - // Start as many worker threads as goroutines allowed - pending := new(sync.WaitGroup) - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - pending.Add(1) - go func(id int) { - defer pending.Done() - process(id) - }(i) - } - pending.Wait() - - // If anything failed, report - if failed > 0 { - for i, err := range errs { - if err != nil { - return i, err - } + if batch.ValueSize() > 0 { + bytes += batch.ValueSize() + if err := batch.Write(); err != nil { + return 0, err } } - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - log.Debug("Premature abort during receipts processing") - return 0, nil - } + // Update the head fast sync block if better bc.mu.Lock() - - head := blockChain[len(errs)-1] + head := blockChain[len(blockChain)-1] if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 { if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil { @@ -805,16 +772,18 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } bc.mu.Unlock() - // Report some public statistics so the user has a clue what's going on - last := blockChain[len(blockChain)-1] - log.Info("Imported new block receipts", "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), - "number", last.Number(), "hash", last.Hash(), "ignored", stats.ignored) - + log.Info("Imported new block receipts", + "count", stats.processed, + "elapsed", common.PrettyDuration(time.Since(start)), + "bytes", bytes, + "number", head.Number(), + "hash", head.Hash(), + "ignored", stats.ignored) return 0, nil } // WriteBlock writes the block to the chain. -func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) { +func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -827,7 +796,7 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er bc.mu.Lock() defer bc.mu.Unlock() - if bc.HasBlock(block.Hash()) { + if bc.HasBlock(block.Hash(), block.NumberU64()) { log.Trace("Block existed", "hash", block.Hash()) return } @@ -837,10 +806,18 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er // Irrelevant of the canonical status, write the block itself to the database if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { - log.Crit("Failed to write block total difficulty", "err", err) + return NonStatTy, err } - if err := WriteBlock(bc.chainDb, block); err != nil { - log.Crit("Failed to write block contents", "err", err) + // Write other block data using a batch. + batch := bc.chainDb.NewBatch() + if err := WriteBlock(batch, block); err != nil { + return NonStatTy, err + } + if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil { + return NonStatTy, err + } + if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { + return NonStatTy, err } // If the total difficulty is higher than our known, add it to the canonical chain @@ -853,15 +830,28 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er return NonStatTy, err } } - bc.insert(block) // Insert the block as the new head of the chain + // Write the positional metadata for transaction and receipt lookups + if err := WriteTxLookupEntries(batch, block); err != nil { + return NonStatTy, err + } + // Write hash preimages + if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { + return NonStatTy, err + } status = CanonStatTy } else { status = SideStatTy } + if err := batch.Write(); err != nil { + return NonStatTy, err + } + // Set new head. + if status == CanonStatTy { + bc.insert(block) + } bc.futureBlocks.Remove(block.Hash()) - - return + return status, nil } // InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. If an error is returned @@ -975,29 +965,18 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { bc.reportBlock(block, receipts, err) return i, err } - // Write state changes to database - if _, err = state.CommitTo(bc.chainDb, bc.config.IsEIP158(block.Number())); err != nil { - return i, err - } - // coalesce logs for later processing - coalescedLogs = append(coalescedLogs, logs...) - - if err = WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - return i, err - } - - // write the block to the chain and get the status - status, err := bc.WriteBlock(block) + // Write the block to the chain and get the status. + status, err := bc.WriteBlockAndState(block, receipts, state) if err != nil { return i, err } - switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart))) - + // coalesce logs for later processing + coalescedLogs = append(coalescedLogs, logs...) blockInsertTimer.UpdateSince(bstart) events = append(events, ChainEvent{block, block.Hash(), logs}) // We need some control over the mining operation. Acquiring locks and waiting @@ -1006,15 +985,6 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { if bc.LastBlockHash() == block.Hash() { events = append(events, ChainHeadEvent{block}) } - - // Write the positional metadata for transaction and receipt lookups - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { - return i, err - } - // Write hash preimages - if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { - return i, err - } case SideStatTy: log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles())) @@ -1357,8 +1327,8 @@ func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { // HasHeader checks if a block header is present in the database or not, caching // it if present. -func (bc *BlockChain) HasHeader(hash common.Hash) bool { - return bc.hc.HasHeader(hash) +func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { + return bc.hc.HasHeader(hash, number) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given diff --git a/core/database_util.go b/core/database_util.go index 989071104e..1730a048e7 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -38,11 +38,6 @@ type DatabaseReader interface { Get(key []byte) (value []byte, err error) } -// DatabaseWriter wraps the Put method of a backing data store. -type DatabaseWriter interface { - Put(key, value []byte) error -} - // DatabaseDeleter wraps the Delete method of a backing data store. type DatabaseDeleter interface { Delete(key []byte) error @@ -154,7 +149,7 @@ func GetHeadFastBlockHash(db DatabaseReader) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(headerKey(hash, number)) return data } @@ -175,10 +170,18 @@ func GetHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func GetBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(blockBodyKey(hash, number)) return data } +func headerKey(hash common.Hash, number uint64) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +func blockBodyKey(hash common.Hash, number uint64) []byte { + return append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. func GetBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { @@ -340,7 +343,7 @@ func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) } // WriteCanonicalHash stores the canonical hash for the given block number. -func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) error { +func WriteCanonicalHash(db ethdb.Putter, hash common.Hash, number uint64) error { key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) @@ -349,7 +352,7 @@ func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) erro } // WriteHeadHeaderHash stores the head header's hash. -func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadHeaderHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { log.Crit("Failed to store last header's hash", "err", err) } @@ -357,7 +360,7 @@ func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadBlockHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last block's hash", "err", err) } @@ -365,7 +368,7 @@ func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { } // WriteHeadFastBlockHash stores the fast head block's hash. -func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { log.Crit("Failed to store last fast block's hash", "err", err) } @@ -373,7 +376,7 @@ func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { } // WriteHeader serializes a block header into the database. -func WriteHeader(db DatabaseWriter, header *types.Header) error { +func WriteHeader(db ethdb.Putter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) if err != nil { return err @@ -393,7 +396,7 @@ func WriteHeader(db DatabaseWriter, header *types.Header) error { } // WriteBody serializes the body of a block into the database. -func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) error { +func WriteBody(db ethdb.Putter, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err @@ -402,7 +405,7 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B } // WriteBodyRLP writes a serialized body of a block into the database. -func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) error { +func WriteBodyRLP(db ethdb.Putter, hash common.Hash, number uint64, rlp rlp.RawValue) error { key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, rlp); err != nil { log.Crit("Failed to store block body", "err", err) @@ -411,7 +414,7 @@ func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.Ra } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) error { +func WriteTd(db ethdb.Putter, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err @@ -424,7 +427,7 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) er } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db DatabaseWriter, block *types.Block) error { +func WriteBlock(db ethdb.Putter, block *types.Block) error { // Store the body first to retain database consistency if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err @@ -439,7 +442,7 @@ func WriteBlock(db DatabaseWriter, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.Putter, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -459,9 +462,7 @@ func WriteBlockReceipts(db DatabaseWriter, hash common.Hash, number uint64, rece // WriteTxLookupEntries stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. -func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { - batch := db.NewBatch() - +func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error { // Iterate over each transaction and encode its metadata for i, tx := range block.Transactions() { entry := txLookupEntry{ @@ -473,20 +474,16 @@ func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { if err != nil { return err } - if err := batch.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { + if err := db.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { return err } } - // Write the scheduled data into the database - if err := batch.Write(); err != nil { - log.Crit("Failed to store lookup entries", "err", err) - } return nil } // WriteBloomBits writes the compressed bloom bits vector belonging to the given // section and bit index. -func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { +func WriteBloomBits(db ethdb.Putter, bit uint, section uint64, head common.Hash, bits []byte) { key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) binary.BigEndian.PutUint16(key[1:], uint16(bit)) @@ -572,13 +569,13 @@ func GetBlockChainVersion(db DatabaseReader) int { } // WriteBlockChainVersion writes vsn as the version number to db. -func WriteBlockChainVersion(db DatabaseWriter, vsn int) { +func WriteBlockChainVersion(db ethdb.Putter, vsn int) { enc, _ := rlp.EncodeToBytes(uint(vsn)) db.Put([]byte("BlockchainVersion"), enc) } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) error { +func WriteChainConfig(db ethdb.Putter, hash common.Hash, cfg *params.ChainConfig) error { // short circuit and ignore if nil config. GetChainConfig // will return a default. if cfg == nil { diff --git a/core/headerchain.go b/core/headerchain.go index 6ec44b61d5..0e5215293d 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -267,7 +267,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCa return i, errors.New("aborted") } // If the header's already known, skip it, otherwise store - if hc.GetHeader(header.Hash(), header.Number.Uint64()) != nil { + if hc.HasHeader(header.Hash(), header.Number.Uint64()) { stats.ignored++ continue } @@ -361,10 +361,13 @@ func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { return hc.GetHeader(hash, hc.GetBlockNumber(hash)) } -// HasHeader checks if a block header is present in the database or not, caching -// it if present. -func (hc *HeaderChain) HasHeader(hash common.Hash) bool { - return hc.GetHeaderByHash(hash) != nil +// HasHeader checks if a block header is present in the database or not. +func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool { + if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) { + return true + } + ok, _ := hc.chainDb.Has(headerKey(hash, number)) + return ok } // GetHeaderByNumber retrieves a block header from the database by number, diff --git a/core/state/sync.go b/core/state/sync.go index 2c29d706aa..28fcf6ae05 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -18,60 +18,24 @@ package state import ( "bytes" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) -// StateSync is the main state synchronisation scheduler, which provides yet the -// unknown state hashes to retrieve, accepts node data associated with said hashes -// and reconstructs the state database step by step until all is done. -type StateSync trie.TrieSync - // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database trie.DatabaseReader) *StateSync { +func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync { var syncer *trie.TrieSync - callback := func(leaf []byte, parent common.Hash) error { - var obj struct { - Nonce uint64 - Balance *big.Int - Root common.Hash - CodeHash []byte - } + var obj Account if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } syncer.AddSubTrie(obj.Root, 64, parent, nil) syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent) - return nil } syncer = trie.NewTrieSync(root, database, callback) - return (*StateSync)(syncer) -} - -// Missing retrieves the known missing nodes from the state trie for retrieval. -func (s *StateSync) Missing(max int) []common.Hash { - return (*trie.TrieSync)(s).Missing(max) -} - -// Process injects a batch of retrieved trie nodes data, returning if something -// was committed to the memcache and also the index of an entry if processing of -// it failed. -func (s *StateSync) Process(list []trie.SyncResult) (bool, int, error) { - return (*trie.TrieSync)(s).Process(list) -} - -// Commit flushes the data stored in the internal memcache out to persistent -// storage, returning th enumber of items written and any occurred error. -func (s *StateSync) Commit(dbw trie.DatabaseWriter) (int, error) { - return (*trie.TrieSync)(s).Commit(dbw) -} - -// Pending returns the number of state entries currently pending for download. -func (s *StateSync) Pending() int { - return (*trie.TrieSync)(s).Pending() + return syncer } diff --git a/eth/api.go b/eth/api.go index 9904c6f536..a5b6e7076c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -241,7 +241,7 @@ func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { for _, b := range bs { - if !chain.HasBlock(b.Hash()) { + if !chain.HasBlock(b.Hash(), b.NumberU64()) { return false } } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f2dbf0f88a..38b66d9dd5 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -156,7 +156,7 @@ type Downloader struct { // LightChain encapsulates functions required to synchronise a light chain. type LightChain interface { // HasHeader verifies a header's presence in the local chain. - HasHeader(common.Hash) bool + HasHeader(h common.Hash, number uint64) bool // GetHeaderByHash retrieves a header from the local chain. GetHeaderByHash(common.Hash) *types.Header @@ -666,7 +666,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash())) { + if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) { number, hash = headers[i].Number.Uint64(), headers[i].Hash() // If every header is known, even future ones, the peer straight out lied about its head @@ -731,7 +731,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash())) { + if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) { end = check break } @@ -1256,7 +1256,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // Collect the yet unknown headers to mark them as uncertain unknown := make([]*types.Header, 0, len(headers)) for _, header := range chunk { - if !d.lightchain.HasHeader(header.Hash()) { + if !d.lightchain.HasHeader(header.Hash(), header.Number.Uint64()) { unknown = append(unknown, header) } } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index d66aafe94e..58f6e9a624 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -217,7 +217,7 @@ func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { } // HasHeader checks if a header is present in the testers canonical chain. -func (dl *downloadTester) HasHeader(hash common.Hash) bool { +func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool { return dl.GetHeaderByHash(hash) != nil } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index a5ce8c42df..eb5416f63c 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" ) @@ -187,10 +188,13 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { type stateSync struct { d *Downloader // Downloader instance to access and manage current peerset - sched *state.StateSync // State trie sync scheduler defining the tasks + sched *trie.TrieSync // State trie sync scheduler defining the tasks keccak hash.Hash // Keccak256 hasher to verify deliveries with tasks map[common.Hash]*stateTask // Set of tasks currently queued for retrieval + numUncommitted int + bytesUncommitted int + deliver chan *stateReq // Delivery channel multiplexing peer responses cancel chan struct{} // Channel to signal a termination request cancelOnce sync.Once // Ensures cancel only ever gets called once @@ -252,9 +256,10 @@ func (s *stateSync) loop() error { // Keep assigning new tasks until the sync completes or aborts for s.sched.Pending() > 0 { - if err := s.assignTasks(); err != nil { + if err := s.commit(false); err != nil { return err } + s.assignTasks() // Tasks assigned, wait for something to happen select { case <-newPeer: @@ -284,12 +289,28 @@ func (s *stateSync) loop() error { } } } + return s.commit(true) +} + +func (s *stateSync) commit(force bool) error { + if !force && s.bytesUncommitted < ethdb.IdealBatchSize { + return nil + } + start := time.Now() + b := s.d.stateDB.NewBatch() + s.sched.Commit(b) + if err := b.Write(); err != nil { + return fmt.Errorf("DB write error: %v", err) + } + s.updateStats(s.numUncommitted, 0, 0, time.Since(start)) + s.numUncommitted = 0 + s.bytesUncommitted = 0 return nil } // assignTasks attempts to assing new tasks to all idle peers, either from the // batch currently being retried, or fetching new data from the trie sync itself. -func (s *stateSync) assignTasks() error { +func (s *stateSync) assignTasks() { // Iterate over all idle peers and try to assign them state fetches peers, _ := s.d.peers.NodeDataIdlePeers() for _, p := range peers { @@ -301,7 +322,6 @@ func (s *stateSync) assignTasks() error { // If the peer was assigned tasks to fetch, send the network request if len(req.items) > 0 { req.peer.log.Trace("Requesting new batch of data", "type", "state", "count", len(req.items)) - select { case s.d.trackStateReq <- req: req.peer.FetchNodeData(req.items) @@ -309,7 +329,6 @@ func (s *stateSync) assignTasks() error { } } } - return nil } // fillTasks fills the given request object with a maximum of n state download @@ -347,11 +366,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) { // delivered. func (s *stateSync) process(req *stateReq) (bool, error) { // Collect processing stats and update progress if valid data was received - processed, written, duplicate, unexpected := 0, 0, 0, 0 + duplicate, unexpected := 0, 0 defer func(start time.Time) { - if processed+written+duplicate+unexpected > 0 { - s.updateStats(processed, written, duplicate, unexpected, time.Since(start)) + if duplicate > 0 || unexpected > 0 { + s.updateStats(0, duplicate, unexpected, time.Since(start)) } }(time.Now()) @@ -362,7 +381,9 @@ func (s *stateSync) process(req *stateReq) (bool, error) { prog, hash, err := s.processNodeData(blob) switch err { case nil: - processed++ + s.numUncommitted++ + s.bytesUncommitted += len(blob) + progress = progress || prog case trie.ErrNotRequested: unexpected++ case trie.ErrAlreadyProcessed: @@ -370,38 +391,20 @@ func (s *stateSync) process(req *stateReq) (bool, error) { default: return stale, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } - if prog { - progress = true - } // If the node delivered a requested item, mark the delivery non-stale if _, ok := req.tasks[hash]; ok { delete(req.tasks, hash) stale = false } } - // If some data managed to hit the database, flush and reset failure counters - if progress { - // Flush any accumulated data out to disk - batch := s.d.stateDB.NewBatch() - - count, err := s.sched.Commit(batch) - if err != nil { - return stale, err - } - if err := batch.Write(); err != nil { - return stale, err - } - written = count - - // If we're inside the critical section, reset fail counter since we progressed - if atomic.LoadUint32(&s.d.fsPivotFails) > 1 { - log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails)) - atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block - } + // If we're inside the critical section, reset fail counter since we progressed. + if progress && atomic.LoadUint32(&s.d.fsPivotFails) > 1 { + log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails)) + atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block } + // Put unfulfilled tasks back into the retry queue npeers := s.d.peers.Len() - for hash, task := range req.tasks { // If the node did deliver something, missing items may be due to a protocol // limit or a previous timeout + delayed delivery. Both cases should permit @@ -425,25 +428,25 @@ func (s *stateSync) process(req *stateReq) (bool, error) { // error occurred. func (s *stateSync) processNodeData(blob []byte) (bool, common.Hash, error) { res := trie.SyncResult{Data: blob} - s.keccak.Reset() s.keccak.Write(blob) s.keccak.Sum(res.Hash[:0]) - committed, _, err := s.sched.Process([]trie.SyncResult{res}) return committed, res.Hash, err } // updateStats bumps the various state sync progress counters and displays a log // message for the user to see. -func (s *stateSync) updateStats(processed, written, duplicate, unexpected int, duration time.Duration) { +func (s *stateSync) updateStats(written, duplicate, unexpected int, duration time.Duration) { s.d.syncStatsLock.Lock() defer s.d.syncStatsLock.Unlock() s.d.syncStatsState.pending = uint64(s.sched.Pending()) - s.d.syncStatsState.processed += uint64(processed) + s.d.syncStatsState.processed += uint64(written) s.d.syncStatsState.duplicate += uint64(duplicate) s.d.syncStatsState.unexpected += uint64(unexpected) - log.Info("Imported new state entries", "count", processed, "flushed", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + if written > 0 || duplicate > 0 || unexpected > 0 { + log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + } } diff --git a/eth/handler.go b/eth/handler.go index 28ae208c06..cee719ddb6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -609,7 +609,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Schedule all the unknown hashes for retrieval unknown := make(newBlockHashesData, 0, len(announces)) for _, block := range announces { - if !pm.blockchain.HasBlock(block.Hash) { + if !pm.blockchain.HasBlock(block.Hash, block.Number) { unknown = append(unknown, block) } } @@ -699,7 +699,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { return } // Otherwise if the block is indeed in out own chain, announce it - if pm.blockchain.HasBlock(hash) { + if pm.blockchain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) } diff --git a/ethdb/database.go b/ethdb/database.go index 7d5fb0b9ef..93755dd7e3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -109,6 +109,10 @@ func (db *LDBDatabase) Put(key []byte, value []byte) error { return db.db.Put(key, value, nil) } +func (db *LDBDatabase) Has(key []byte) (bool, error) { + return db.db.Has(key, nil) +} + // Get returns the given key if it's present. func (db *LDBDatabase) Get(key []byte) ([]byte, error) { // Measure the database get latency, if requested @@ -271,19 +275,19 @@ func (db *LDBDatabase) meter(refresh time.Duration) { } } -// TODO: remove this stuff and expose leveldb directly - func (db *LDBDatabase) NewBatch() Batch { return &ldbBatch{db: db.db, b: new(leveldb.Batch)} } type ldbBatch struct { - db *leveldb.DB - b *leveldb.Batch + db *leveldb.DB + b *leveldb.Batch + size int } func (b *ldbBatch) Put(key, value []byte) error { b.b.Put(key, value) + b.size += len(value) return nil } @@ -291,6 +295,10 @@ func (b *ldbBatch) Write() error { return b.db.Write(b.b, nil) } +func (b *ldbBatch) ValueSize() int { + return b.size +} + type table struct { db Database prefix string @@ -309,6 +317,10 @@ func (dt *table) Put(key []byte, value []byte) error { return dt.db.Put(append([]byte(dt.prefix), key...), value) } +func (dt *table) Has(key []byte) (bool, error) { + return dt.db.Has(append([]byte(dt.prefix), key...)) +} + func (dt *table) Get(key []byte) ([]byte, error) { return dt.db.Get(append([]byte(dt.prefix), key...)) } @@ -342,3 +354,7 @@ func (tb *tableBatch) Put(key, value []byte) error { func (tb *tableBatch) Write() error { return tb.batch.Write() } + +func (tb *tableBatch) ValueSize() int { + return tb.batch.ValueSize() +} diff --git a/ethdb/interface.go b/ethdb/interface.go index f4b787a52a..99a5b770d5 100644 --- a/ethdb/interface.go +++ b/ethdb/interface.go @@ -16,15 +16,29 @@ package ethdb -type Database interface { +// Code using batches should try to add this much data to the batch. +// The value was determined empirically. +const IdealBatchSize = 100 * 1024 + +// Putter wraps the database write operation supported by both batches and regular databases. +type Putter interface { Put(key []byte, value []byte) error +} + +// Database wraps all database operations. All methods are safe for concurrent use. +type Database interface { + Putter Get(key []byte) ([]byte, error) + Has(key []byte) (bool, error) Delete(key []byte) error Close() NewBatch() Batch } +// Batch is a write-only database that commits changes to its host database +// when Write is called. Batch cannot be used concurrently. type Batch interface { - Put(key, value []byte) error + Putter + ValueSize() int // amount of data in the batch Write() error } diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 11b0937248..699bd0c9fd 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -45,6 +45,14 @@ func (db *MemDatabase) Put(key []byte, value []byte) error { return nil } +func (db *MemDatabase) Has(key []byte) (bool, error) { + db.lock.RLock() + defer db.lock.RUnlock() + + _, ok := db.db[string(key)] + return ok, nil +} + func (db *MemDatabase) Get(key []byte) ([]byte, error) { db.lock.RLock() defer db.lock.RUnlock() @@ -93,21 +101,16 @@ type kv struct{ k, v []byte } type memBatch struct { db *MemDatabase writes []kv - lock sync.RWMutex + size int } func (b *memBatch) Put(key, value []byte) error { - b.lock.Lock() - defer b.lock.Unlock() - b.writes = append(b.writes, kv{common.CopyBytes(key), common.CopyBytes(value)}) + b.size += len(value) return nil } func (b *memBatch) Write() error { - b.lock.RLock() - defer b.lock.RUnlock() - b.db.lock.Lock() defer b.db.lock.Unlock() @@ -116,3 +119,7 @@ func (b *memBatch) Write() error { } return nil } + +func (b *memBatch) ValueSize() int { + return b.size +} diff --git a/les/handler.go b/les/handler.go index 1a75cd369d..df7eb6af51 100644 --- a/les/handler.go +++ b/les/handler.go @@ -70,7 +70,7 @@ func errResp(code errCode, format string, v ...interface{}) error { } type BlockChain interface { - HasHeader(hash common.Hash) bool + HasHeader(hash common.Hash, number uint64) bool GetHeader(hash common.Hash, number uint64) *types.Header GetHeaderByHash(hash common.Hash) *types.Header CurrentHeader() *types.Header diff --git a/light/lightchain.go b/light/lightchain.go index df194ecadf..3a763fc906 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -252,8 +252,8 @@ func (self *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.R // HasBlock checks if a block is fully present in the database or not, caching // it if present. -func (bc *LightChain) HasBlock(hash common.Hash) bool { - blk, _ := bc.GetBlockByHash(NoOdr, hash) +func (bc *LightChain) HasBlock(hash common.Hash, number uint64) bool { + blk, _ := bc.GetBlock(NoOdr, hash, number) return blk != nil } @@ -418,8 +418,8 @@ func (self *LightChain) GetHeaderByHash(hash common.Hash) *types.Header { // HasHeader checks if a block header is present in the database or not, caching // it if present. -func (bc *LightChain) HasHeader(hash common.Hash) bool { - return bc.hc.HasHeader(hash) +func (bc *LightChain) HasHeader(hash common.Hash, number uint64) bool { + return bc.hc.HasHeader(hash, number) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given diff --git a/miner/worker.go b/miner/worker.go index 2919d4a485..b48db2a30e 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -304,13 +304,8 @@ func (self *worker) wait() { } go self.mux.Post(core.NewMinedBlockEvent{Block: block}) } else { - work.state.CommitTo(self.chainDb, self.config.IsEIP158(block.Number())) - stat, err := self.chain.WriteBlock(block) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - // update block hash since it is now available and not when the receipt/log of individual transactions were created + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. for _, r := range work.receipts { for _, l := range r.Logs { l.BlockHash = block.Hash() @@ -319,15 +314,17 @@ func (self *worker) wait() { for _, log := range work.state.Logs() { log.BlockHash = block.Hash() } + stat, err := self.chain.WriteBlockAndState(block, work.receipts, work.state) + if err != nil { + log.Error("Failed writing block to chain", "err", err) + continue + } // check if canon block and write transactions if stat == core.CanonStatTy { - // This puts transactions in a extra db for rpc - core.WriteTxLookupEntries(self.chainDb, block) // implicit by posting ChainHeadEvent mustCommitNewWork = false } - // broadcast before waiting for validation go func(block *types.Block, logs []*types.Log, receipts []*types.Receipt) { self.mux.Post(core.NewMinedBlockEvent{Block: block}) @@ -336,16 +333,12 @@ func (self *worker) wait() { coalescedLogs []*types.Log ) events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - if stat == core.CanonStatTy { events = append(events, core.ChainHeadEvent{Block: block}) coalescedLogs = logs } // post blockchain events self.chain.PostChainEvents(events, coalescedLogs) - if err := core.WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - log.Warn("Failed writing block receipts", "err", err) - } }(block, work.state.Logs(), work.receipts) } // Insert the block into the set of pending ones to wait for confirmations diff --git a/trie/sync.go b/trie/sync.go index 1e4f8d87c1..fea10051f4 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -138,7 +138,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) if _, ok := s.membatch.batch[hash]; ok { return } - if blob, _ := s.database.Get(hash.Bytes()); blob != nil { + if ok, _ := s.database.Has(hash.Bytes()); ok { return } // Assemble the new sub-trie sync request @@ -296,8 +296,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) { if _, ok := s.membatch.batch[hash]; ok { continue } - blob, _ := s.database.Get(node) - if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil { + if ok, _ := s.database.Has(node); ok { continue } // Locally unknown node, schedule for retrieval diff --git a/trie/trie.go b/trie/trie.go index a3151b1ce7..7f69a3d1de 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -66,6 +66,7 @@ type Database interface { // DatabaseReader wraps the Get method of a backing store for the trie. type DatabaseReader interface { Get(key []byte) (value []byte, err error) + Has(key []byte) (bool, error) } // DatabaseWriter wraps the Put method of a backing store for the trie. From 5596b664c4ca5be199bb93f87155fa2c1fa7eab2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 11 Sep 2017 09:33:18 +0200 Subject: [PATCH 31/45] internal/debug: add debug_freeOSMemory (#15122) --- internal/debug/api.go | 5 ++ internal/web3ext/web3ext.go | 132 +++++++++++++++++------------------- 2 files changed, 67 insertions(+), 70 deletions(-) diff --git a/internal/debug/api.go b/internal/debug/api.go index 8b7693f6a4..7583878ed5 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -176,6 +176,11 @@ func (*HandlerT) Stacks() string { return string(buf) } +// FreeOSMemory returns unused memory to the OS. +func (*HandlerT) FreeOSMemory() { + debug.FreeOSMemory() +} + func writeProfile(name, file string) error { p := pprof.Lookup(name) log.Info("Writing profile records", "count", p.Count(), "type", name, "dump", file) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index bf1db88196..927085b441 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -34,58 +34,56 @@ var Modules = map[string]string{ const Chequebook_JS = ` web3._extend({ - property: 'chequebook', - methods: - [ - new web3._extend.Method({ - name: 'deposit', - call: 'chequebook_deposit', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Property({ + property: 'chequebook', + methods: [ + new web3._extend.Method({ + name: 'deposit', + call: 'chequebook_deposit', + params: 1, + inputFormatter: [null] + }), + new web3._extend.Property({ name: 'balance', getter: 'chequebook_balance', - outputFormatter: web3._extend.utils.toDecimal + outputFormatter: web3._extend.utils.toDecimal }), - new web3._extend.Method({ - name: 'cash', - call: 'chequebook_cash', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'issue', - call: 'chequebook_issue', - params: 2, - inputFormatter: [null, null] - }), - ] + new web3._extend.Method({ + name: 'cash', + call: 'chequebook_cash', + params: 1, + inputFormatter: [null] + }), + new web3._extend.Method({ + name: 'issue', + call: 'chequebook_issue', + params: 2, + inputFormatter: [null, null] + }), + ] }); ` const Clique_JS = ` web3._extend({ - property: 'clique', - methods: - [ + property: 'clique', + methods: [ new web3._extend.Method({ name: 'getSnapshot', call: 'clique_getSnapshot', params: 1, - inputFormatter: [null] + inputFormatter: [null] }), new web3._extend.Method({ name: 'getSnapshotAtHash', call: 'clique_getSnapshotAtHash', params: 1 }), - new web3._extend.Method({ - name: 'getSigners', - call: 'clique_getSigners', - params: 1, - inputFormatter: [null] - }), + new web3._extend.Method({ + name: 'getSigners', + call: 'clique_getSigners', + params: 1, + inputFormatter: [null] + }), new web3._extend.Method({ name: 'getSignersAtHash', call: 'clique_getSignersAtHash', @@ -100,10 +98,9 @@ web3._extend({ name: 'discard', call: 'clique_discard', params: 1 - }) - ], - properties: - [ + }), + ], + properties: [ new web3._extend.Property({ name: 'proposals', getter: 'clique_proposals' @@ -115,8 +112,7 @@ web3._extend({ const Admin_JS = ` web3._extend({ property: 'admin', - methods: - [ + methods: [ new web3._extend.Method({ name: 'addPeer', call: 'admin_addPeer', @@ -162,10 +158,9 @@ web3._extend({ new web3._extend.Method({ name: 'stopWS', call: 'admin_stopWS' - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'nodeInfo', getter: 'admin_nodeInfo' @@ -177,7 +172,7 @@ web3._extend({ new web3._extend.Property({ name: 'datadir', getter: 'admin_datadir' - }) + }), ] }); ` @@ -185,8 +180,7 @@ web3._extend({ const Debug_JS = ` web3._extend({ property: 'debug', - methods: - [ + methods: [ new web3._extend.Method({ name: 'printBlock', call: 'debug_printBlock', @@ -268,6 +262,11 @@ web3._extend({ params: 0, outputFormatter: console.log }), + new web3._extend.Method({ + name: 'freeOSMemory', + call: 'debug_freeOSMemory', + params: 0, + }), new web3._extend.Method({ name: 'memStats', call: 'debug_memStats', @@ -358,8 +357,7 @@ web3._extend({ const Eth_JS = ` web3._extend({ property: 'eth', - methods: - [ + methods: [ new web3._extend.Method({ name: 'sign', call: 'eth_sign', @@ -396,10 +394,9 @@ web3._extend({ }, params: 2, inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex] - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'pendingTransactions', getter: 'eth_pendingTransactions', @@ -411,7 +408,7 @@ web3._extend({ } return formatted; } - }) + }), ] }); ` @@ -419,8 +416,7 @@ web3._extend({ const Miner_JS = ` web3._extend({ property: 'miner', - methods: - [ + methods: [ new web3._extend.Method({ name: 'start', call: 'miner_start', @@ -451,7 +447,7 @@ web3._extend({ new web3._extend.Method({ name: 'getHashrate', call: 'miner_getHashrate' - }) + }), ], properties: [] }); @@ -461,12 +457,11 @@ const Net_JS = ` web3._extend({ property: 'net', methods: [], - properties: - [ + properties: [ new web3._extend.Property({ name: 'version', getter: 'net_version' - }) + }), ] }); ` @@ -474,8 +469,7 @@ web3._extend({ const Personal_JS = ` web3._extend({ property: 'personal', - methods: - [ + methods: [ new web3._extend.Method({ name: 'importRawKey', call: 'personal_importRawKey', @@ -501,14 +495,13 @@ web3._extend({ name: 'deriveAccount', call: 'personal_deriveAccount', params: 3 - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'listWallets', getter: 'personal_listWallets' - }) + }), ] }) ` @@ -517,12 +510,11 @@ const RPC_JS = ` web3._extend({ property: 'rpc', methods: [], - properties: - [ + properties: [ new web3._extend.Property({ name: 'modules', getter: 'rpc_modules' - }) + }), ] }); ` @@ -633,7 +625,7 @@ web3._extend({ name: 'newMessageFilter', call: 'shh_newMessageFilter', params: 1 - }) + }), ], properties: [ @@ -669,7 +661,7 @@ web3._extend({ name: 'listmounts', call: 'swarmfs_listmounts', params: 0 - }) + }), ] }); ` @@ -696,7 +688,7 @@ web3._extend({ status.queued = web3._extend.utils.toDecimal(status.queued); return status; } - }) + }), ] }); ` From 10b3f97c9dcc6f3711aa2d3b1bb43e67eb921223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 11 Sep 2017 13:13:05 +0300 Subject: [PATCH 32/45] core: only fire one chain head per batch (#15123) * core: only fire one chain head per batch * miner: announce chan events synchronously --- core/blockchain.go | 53 +++++++++++++++++------------ core/blockchain_test.go | 2 +- miner/worker.go | 75 +++++++++++++++++------------------------ 3 files changed, 63 insertions(+), 67 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 25be8d762a..235c1f3da2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -854,9 +854,22 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R return status, nil } -// InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. If an error is returned -// it will return the index number of the failing block as well an error describing what went wrong (for possible errors see core/errors.go). +// InsertChain attempts to insert the given batch of blocks in to the canonical +// chain or, otherwise, create a fork. If an error is returned it will return +// the index number of the failing block as well an error describing what went +// wrong. +// +// After insertion is done, all accumulated events will be fired. func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { + n, events, logs, err := bc.insertChain(chain) + bc.PostChainEvents(events, logs) + return n, err +} + +// insertChain will execute the actual chain insertion and event aggregation. The +// only reason this method exists as a separate one is to make locking cleaner +// with deferred statements. +func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() { @@ -864,7 +877,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(), "parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), + return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4]) } } @@ -881,6 +894,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { var ( stats = insertStats{startTime: mclock.Now()} events = make([]interface{}, 0, len(chain)) + lastCanon *types.Block coalescedLogs []*types.Log ) // Start the parallel header verifier @@ -904,7 +918,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { bc.reportBlock(block, nil, ErrBlacklistedHash) - return i, ErrBlacklistedHash + return i, events, coalescedLogs, ErrBlacklistedHash } // Wait for the block's verification to complete bstart := time.Now() @@ -925,7 +939,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // if given. max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) if block.Time().Cmp(max) > 0 { - return i, fmt.Errorf("future block: %v > %v", block.Time(), max) + return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max) } bc.futureBlocks.Add(block.Hash(), block) stats.queued++ @@ -939,7 +953,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } bc.reportBlock(block, nil, err) - return i, err + return i, events, coalescedLogs, err } // Create a new statedb using the parent block and report an // error if it fails. @@ -951,40 +965,35 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } state, err := state.New(parent.Root(), bc.stateCache) if err != nil { - return i, err + return i, events, coalescedLogs, err } // Process block using the parent state as reference point. receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { bc.reportBlock(block, receipts, err) - return i, err + return i, events, coalescedLogs, err } // Validate the state using the default validator err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas) if err != nil { bc.reportBlock(block, receipts, err) - return i, err + return i, events, coalescedLogs, err } - // Write the block to the chain and get the status. status, err := bc.WriteBlockAndState(block, receipts, state) if err != nil { - return i, err + return i, events, coalescedLogs, err } switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart))) - // coalesce logs for later processing + coalescedLogs = append(coalescedLogs, logs...) blockInsertTimer.UpdateSince(bstart) events = append(events, ChainEvent{block, block.Hash(), logs}) - // We need some control over the mining operation. Acquiring locks and waiting - // for the miner to create new block takes too long and in most cases isn't - // even necessary. - if bc.LastBlockHash() == block.Hash() { - events = append(events, ChainHeadEvent{block}) - } + lastCanon = block + case SideStatTy: log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles())) @@ -996,9 +1005,11 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { stats.usedGas += usedGas.Uint64() stats.report(chain, i) } - go bc.PostChainEvents(events, coalescedLogs) - - return 0, nil + // Append a single chain head event if we've progressed the chain + if lastCanon != nil && bc.LastBlockHash() == lastCanon.Hash() { + events = append(events, ChainHeadEvent{lastCanon}) + } + return 0, events, coalescedLogs, nil } // insertStats tracks and reports on block insertion. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 470974a0a7..700a30a16f 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -935,7 +935,7 @@ func TestReorgSideEvent(t *testing.T) { } gen.AddTx(tx) }) - chainSideCh := make(chan ChainSideEvent) + chainSideCh := make(chan ChainSideEvent, 64) blockchain.SubscribeChainSideEvent(chainSideCh) if _, err := blockchain.InsertChain(replacementBlocks); err != nil { t.Fatalf("failed to insert chain: %v", err) diff --git a/miner/worker.go b/miner/worker.go index b48db2a30e..bf24970f5c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -125,8 +125,6 @@ type worker struct { // atomic status counters mining int32 atWork int32 - - fullValidation bool } func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase common.Address, eth Backend, mux *event.TypeMux) *worker { @@ -146,7 +144,6 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com coinbase: coinbase, agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - fullValidation: false, } // Subscribe TxPreEvent for tx pool worker.txSub = eth.TxPool().SubscribeTxPreEvent(worker.txCh) @@ -297,50 +294,38 @@ func (self *worker) wait() { block := result.Block work := result.Work - if self.fullValidation { - if _, err := self.chain.InsertChain(types.Blocks{block}); err != nil { - log.Error("Mined invalid block", "err", err) - continue + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, r := range work.receipts { + for _, l := range r.Logs { + l.BlockHash = block.Hash() } - go self.mux.Post(core.NewMinedBlockEvent{Block: block}) - } else { - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - for _, r := range work.receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } - } - for _, log := range work.state.Logs() { - log.BlockHash = block.Hash() - } - stat, err := self.chain.WriteBlockAndState(block, work.receipts, work.state) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - - // check if canon block and write transactions - if stat == core.CanonStatTy { - // implicit by posting ChainHeadEvent - mustCommitNewWork = false - } - // broadcast before waiting for validation - go func(block *types.Block, logs []*types.Log, receipts []*types.Receipt) { - self.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - coalescedLogs []*types.Log - ) - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - if stat == core.CanonStatTy { - events = append(events, core.ChainHeadEvent{Block: block}) - coalescedLogs = logs - } - // post blockchain events - self.chain.PostChainEvents(events, coalescedLogs) - }(block, work.state.Logs(), work.receipts) } + for _, log := range work.state.Logs() { + log.BlockHash = block.Hash() + } + stat, err := self.chain.WriteBlockAndState(block, work.receipts, work.state) + if err != nil { + log.Error("Failed writing block to chain", "err", err) + continue + } + // check if canon block and write transactions + if stat == core.CanonStatTy { + // implicit by posting ChainHeadEvent + mustCommitNewWork = false + } + // Broadcast the block and announce chain insertion event + self.mux.Post(core.NewMinedBlockEvent{Block: block}) + var ( + events []interface{} + logs = work.state.Logs() + ) + events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) + if stat == core.CanonStatTy { + events = append(events, core.ChainHeadEvent{Block: block}) + } + self.chain.PostChainEvents(events, logs) + // Insert the block into the set of pending ones to wait for confirmations self.unconfirmed.Insert(block.NumberU64(), block.Hash()) From 48705f8aea04dffb7e772784122aa600bbc60cab Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 11 Sep 2017 12:29:47 +0200 Subject: [PATCH 33/45] internal/debug: add debug_setGCPercent --- internal/debug/api.go | 6 ++++++ internal/web3ext/web3ext.go | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/internal/debug/api.go b/internal/debug/api.go index 7583878ed5..3547b05641 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -181,6 +181,12 @@ func (*HandlerT) FreeOSMemory() { debug.FreeOSMemory() } +// SetGCPercent sets the garbage collection target percentage. It returns the previous +// setting. A negative value disables GC. +func (*HandlerT) SetGCPercent(v int) int { + return debug.SetGCPercent(v) +} + func writeProfile(name, file string) error { p := pprof.Lookup(name) log.Info("Writing profile records", "count", p.Count(), "type", name, "dump", file) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 927085b441..215eae7010 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -267,6 +267,11 @@ web3._extend({ call: 'debug_freeOSMemory', params: 0, }), + new web3._extend.Method({ + name: 'setGCPercent', + call: 'debug_setGCPercent', + params: 1, + }), new web3._extend.Method({ name: 'memStats', call: 'debug_memStats', From 7a7f6a4f29c7d30d861eb6b9c92374d370ce87cd Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Mon, 11 Sep 2017 23:36:16 +0200 Subject: [PATCH 34/45] light: new CHTs for mainnet and ropsten --- light/lightchain.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/light/lightchain.go b/light/lightchain.go index 3a763fc906..4c877a771a 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -97,9 +97,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. } if bc.genesisBlock.Hash() == params.MainnetGenesisHash { // add trusted CHT - WriteTrustedCht(bc.chainDb, TrustedCht{Number: 805, Root: common.HexToHash("85e4286fe0a730390245c49de8476977afdae0eb5530b277f62a52b12313d50f")}) + WriteTrustedCht(bc.chainDb, TrustedCht{Number: 1040, Root: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca")}) log.Info("Added trusted CHT for mainnet") } + if bc.genesisBlock.Hash() == params.TestnetGenesisHash { + // add trusted CHT + WriteTrustedCht(bc.chainDb, TrustedCht{Number: 400, Root: common.HexToHash("2a4befa19e4675d939c3dc22dca8c6ae9fcd642be1f04b06bd6e4203cc304660")}) + log.Info("Added trusted CHT for ropsten testnet") + } if err := bc.loadLastState(); err != nil { return nil, err From 66a7ef57e61f15d718c179d2a0ffaf8a5c38b2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 12 Sep 2017 11:35:35 +0300 Subject: [PATCH 35/45] cmd/puppeth: reserve "yournode" as a non-allowed ethstats user --- cmd/puppeth/module_ethstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index 5d3fa5fc03..6ce662f65f 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -42,7 +42,7 @@ RUN \ WORKDIR /eth-netstats EXPOSE 3000 -RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}]};' > lib/utils/config.js +RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}], reserved: ["yournode"]};' > lib/utils/config.js CMD ["npm", "start"] ` From f46adfac285741e943b97779d2053b22e66ce18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 12 Sep 2017 14:39:34 +0300 Subject: [PATCH 36/45] eth/downloader: track peer drops and deassign state sync tasks --- eth/downloader/peer.go | 22 ++++++++++++++++------ eth/downloader/statesync.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d0dc9a8aa6..e638744ea5 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -349,9 +349,10 @@ func (p *peerConnection) Lacks(hash common.Hash) bool { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peerConnection - newPeerFeed event.Feed - lock sync.RWMutex + peers map[string]*peerConnection + newPeerFeed event.Feed + peerDropFeed event.Feed + lock sync.RWMutex } // newPeerSet creates a new peer set top track the active download sources. @@ -361,10 +362,16 @@ func newPeerSet() *peerSet { } } +// SubscribeNewPeers subscribes to peer arrival events. func (ps *peerSet) SubscribeNewPeers(ch chan<- *peerConnection) event.Subscription { return ps.newPeerFeed.Subscribe(ch) } +// SubscribePeerDrops subscribes to peer departure events. +func (ps *peerSet) SubscribePeerDrops(ch chan<- *peerConnection) event.Subscription { + return ps.peerDropFeed.Subscribe(ch) +} + // Reset iterates over the current peer set, and resets each of the known peers // to prepare for a next batch of block retrieval. func (ps *peerSet) Reset() { @@ -419,12 +426,15 @@ func (ps *peerSet) Register(p *peerConnection) error { // actions to/from that particular entity. func (ps *peerSet) Unregister(id string) error { ps.lock.Lock() - defer ps.lock.Unlock() - - if _, ok := ps.peers[id]; !ok { + p, ok := ps.peers[id] + if !ok { + defer ps.lock.Unlock() return errNotRegistered } delete(ps.peers, id) + ps.lock.Unlock() + + ps.peerDropFeed.Send(p) return nil } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index eb5416f63c..a0b05c9be6 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -40,6 +40,7 @@ type stateReq struct { timer *time.Timer // Timer to fire when the RTT timeout expires peer *peerConnection // Peer that we're requesting from response [][]byte // Response data of the peer (nil for timeouts) + dropped bool // Flag whether the peer dropped off early } // timedOut returns if this request timed out. @@ -105,6 +106,11 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { go s.run() defer s.Cancel() + // Listen for peer departure events to cancel assigned tasks + peerDrop := make(chan *peerConnection, 1024) + peerSub := s.d.peers.SubscribePeerDrops(peerDrop) + defer peerSub.Unsubscribe() + for { // Enable sending of the first buffered element if there is one. var ( @@ -143,6 +149,20 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { finished = append(finished, req) delete(active, pack.PeerId()) + // Handle dropped peer connections: + case p := <-peerDrop: + // Skip if no request is currently pending + req := active[p.id] + if req == nil { + continue + } + // Finalize the request and queue up for processing + req.timer.Stop() + req.dropped = true + + finished = append(finished, req) + delete(active, p.id) + // Handle timed-out requests: case req := <-timeout: // If the peer is already requesting something else, ignore the stale timeout. @@ -167,6 +187,9 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { log.Warn("Busy peer assigned new state fetch", "peer", old.peer.id) // Make sure the previous one doesn't get siletly lost + old.timer.Stop() + old.dropped = true + finished = append(finished, old) } // Start a timer to notify the sync loop if the peer stalled. @@ -269,9 +292,9 @@ func (s *stateSync) loop() error { return errCancelStateFetch case req := <-s.deliver: - // Response or timeout triggered, drop the peer if stalling - log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "timeout", req.timedOut()) - if len(req.items) <= 2 && req.timedOut() { + // Response, disconnect or timeout triggered, drop the peer if stalling + log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "dropped", req.dropped, "timeout", !req.dropped && req.timedOut()) + if len(req.items) <= 2 && !req.dropped && req.timedOut() { // 2 items are the minimum requested, if even that times out, we've no use of // this peer at the moment. log.Warn("Stalling state sync, dropping peer", "peer", req.peer.id) From 72af509abe8e77ecbc7a1aee344c6dbc6992de66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 13 Sep 2017 11:25:50 +0300 Subject: [PATCH 37/45] params: add Infura bootnode to Rinkeby --- params/bootnodes.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/params/bootnodes.go b/params/bootnodes.go index 6c6c02f8cf..82ab6d48c2 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -41,12 +41,14 @@ var TestnetBootnodes = []string{ // Rinkeby test network. var RinkebyBootnodes = []string{ "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303", // IE + "enode://343149e4feefa15d882d9fe4ac7d88f885bd05ebb735e547f12e12080a9fa07c8014ca6fd7f373123488102fe5e34111f8509cf0b7de3f5b44339c9f25e87cb8@52.3.158.184:30303", // INFURA } // RinkebyV5Bootnodes are the enode URLs of the P2P bootstrap nodes running on the // Rinkeby test network for the experimental RLPx v5 topic-discovery network. var RinkebyV5Bootnodes = []string{ "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303?discport=30304", // IE + "enode://343149e4feefa15d882d9fe4ac7d88f885bd05ebb735e547f12e12080a9fa07c8014ca6fd7f373123488102fe5e34111f8509cf0b7de3f5b44339c9f25e87cb8@52.3.158.184:30303?discport=30304", // INFURA } // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the From 5bbd7fb390a539a7183bccc5f2b75b4e564ed398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Sep 2017 10:07:31 +0300 Subject: [PATCH 38/45] consensus, core, params: rebrand Metro to Byzantium --- consensus/ethash/consensus.go | 20 +++++----- core/state_processor.go | 2 +- core/types/receipt.go | 2 +- core/vm/contracts.go | 6 +-- core/vm/contracts_test.go | 4 +- core/vm/evm.go | 8 ++-- core/vm/gas_table.go | 2 +- core/vm/interpreter.go | 6 +-- core/vm/jump_table.go | 12 +++--- params/config.go | 72 +++++++++++++++++------------------ tests/init.go | 26 ++++++------- tests/transaction_test.go | 12 +++--- 12 files changed, 86 insertions(+), 86 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index b714204458..6a19d449f3 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -36,9 +36,9 @@ import ( // Ethash proof-of-work protocol constants. var ( - frontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - metropolisBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Metropolis - maxUncles = 2 // Maximum number of uncles allowed in a single block + frontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + byzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + maxUncles = 2 // Maximum number of uncles allowed in a single block ) // Various error messages to mark blocks invalid. These should be private to @@ -290,8 +290,8 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { - case config.IsMetropolis(next): - return calcDifficultyMetropolis(time, parent) + case config.IsByzantium(next): + return calcDifficultyByzantium(time, parent) case config.IsHomestead(next): return calcDifficultyHomestead(time, parent) default: @@ -310,10 +310,10 @@ var ( big2999999 = big.NewInt(2999999) ) -// calcDifficultyMetropolis is the difficulty adjustment algorithm. It returns +// calcDifficultyByzantium is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time given the -// parent block's time and difficulty. The calculation uses the Metropolis rules. -func calcDifficultyMetropolis(time uint64, parent *types.Header) *big.Int { +// parent block's time and difficulty. The calculation uses the Byzantium rules. +func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int { // https://github.com/ethereum/EIPs/issues/100. // algorithm: // diff = (parent_diff + @@ -530,8 +530,8 @@ var ( func AccumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) { // Select the correct block reward based on chain progression blockReward := frontierBlockReward - if config.IsMetropolis(header.Number) { - blockReward = metropolisBlockReward + if config.IsByzantium(header.Number) { + blockReward = byzantiumBlockReward } // Accumulate the rewards for the miner and any included uncles reward := new(big.Int).Set(blockReward) diff --git a/core/state_processor.go b/core/state_processor.go index 4115eab8cb..689c83785f 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -105,7 +105,7 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, author *common // Update the state with pending changes var root []byte - if config.IsMetropolis(header.Number) { + if config.IsByzantium(header.Number) { statedb.Finalise(true) } else { root = statedb.IntermediateRoot(config.IsEIP158(header.Number)).Bytes() diff --git a/core/types/receipt.go b/core/types/receipt.go index 4f3b7357ed..e179fe0cfd 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -79,7 +79,7 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed *big.Int) *Receipt { } // EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt -// into an RLP stream. If no post state is present, metropolis fork is assumed. +// into an RLP stream. If no post state is present, byzantium fork is assumed. func (r *Receipt) EncodeRLP(w io.Writer) error { return rlp.Encode(w, &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}) } diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 790d42bbe6..7344b6043c 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -46,9 +46,9 @@ var PrecompiledContractsHomestead = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{4}): &dataCopy{}, } -// PrecompiledContractsMetropolis contains the default set of pre-compiled Ethereum -// contracts used in the Metropolis release. -var PrecompiledContractsMetropolis = map[common.Address]PrecompiledContract{ +// PrecompiledContractsByzantium contains the default set of pre-compiled Ethereum +// contracts used in the Byzantium release. +var PrecompiledContractsByzantium = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{1}): &ecrecover{}, common.BytesToAddress([]byte{2}): &sha256hash{}, common.BytesToAddress([]byte{3}): &ripemd160hash{}, diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 880d7324ae..513651835a 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -321,7 +321,7 @@ var bn256PairingTests = []precompiledTest{ } func testPrecompiled(addr string, test precompiledTest, t *testing.T) { - p := PrecompiledContractsMetropolis[common.HexToAddress(addr)] + p := PrecompiledContractsByzantium[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) contract := NewContract(AccountRef(common.HexToAddress("1337")), nil, new(big.Int), p.RequiredGas(in)) @@ -338,7 +338,7 @@ func benchmarkPrecompiled(addr string, test precompiledTest, bench *testing.B) { if test.noBenchmark { return } - p := PrecompiledContractsMetropolis[common.HexToAddress(addr)] + p := PrecompiledContractsByzantium[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) reqGas := p.RequiredGas(in) contract := NewContract(AccountRef(common.HexToAddress("1337")), diff --git a/core/vm/evm.go b/core/vm/evm.go index 495d9beea1..b0a6f3d269 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -41,8 +41,8 @@ type ( func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, error) { if contract.CodeAddr != nil { precompiles := PrecompiledContractsHomestead - if evm.ChainConfig().IsMetropolis(evm.BlockNumber) { - precompiles = PrecompiledContractsMetropolis + if evm.ChainConfig().IsByzantium(evm.BlockNumber) { + precompiles = PrecompiledContractsByzantium } if p := precompiles[*contract.CodeAddr]; p != nil { return RunPrecompiledContract(p, input, contract) @@ -151,8 +151,8 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas ) if !evm.StateDB.Exist(addr) { precompiles := PrecompiledContractsHomestead - if evm.ChainConfig().IsMetropolis(evm.BlockNumber) { - precompiles = PrecompiledContractsMetropolis + if evm.ChainConfig().IsByzantium(evm.BlockNumber) { + precompiles = PrecompiledContractsByzantium } if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 { return nil, gas, nil diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 13712295c9..0d8e295a52 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -324,7 +324,7 @@ func gasCall(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem eip158 = evm.ChainConfig().IsEIP158(evm.BlockNumber) ) if eip158 { - if transfersValue && evm.StateDB.Empty(address) { + if transfersValue && evm.StateDB.Empty(address) { gas += params.CallNewAccountGas } } else if !evm.StateDB.Exist(address) { diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index d3e24a7a45..b0d796a44b 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -70,8 +70,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { switch { - case evm.ChainConfig().IsMetropolis(evm.BlockNumber): - cfg.JumpTable = metropolisInstructionSet + case evm.ChainConfig().IsByzantium(evm.BlockNumber): + cfg.JumpTable = byzantiumInstructionSet case evm.ChainConfig().IsHomestead(evm.BlockNumber): cfg.JumpTable = homesteadInstructionSet default: @@ -88,7 +88,7 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { } func (in *Interpreter) enforceRestrictions(op OpCode, operation operation, stack *Stack) error { - if in.evm.chainRules.IsMetropolis { + if in.evm.chainRules.IsByzantium { if in.readOnly { // If the interpreter is operating in readonly mode, make sure no // state-modifying operation is performed. The 3rd stack item diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index f0a922912f..9ef192fdf9 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -51,14 +51,14 @@ type operation struct { } var ( - frontierInstructionSet = NewFrontierInstructionSet() - homesteadInstructionSet = NewHomesteadInstructionSet() - metropolisInstructionSet = NewMetropolisInstructionSet() + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() + byzantiumInstructionSet = NewByzantiumInstructionSet() ) -// NewMetropolisInstructionSet returns the frontier, homestead and -// metropolis instructions. -func NewMetropolisInstructionSet() [256]operation { +// NewByzantiumInstructionSet returns the frontier, homestead and +// byzantium instructions. +func NewByzantiumInstructionSet() [256]operation { // instructions that can be executed during the homestead phase. instructionSet := NewHomesteadInstructionSet() instructionSet[STATICCALL] = operation{ diff --git a/params/config.go b/params/config.go index f4bb6172ba..324620ad5c 100644 --- a/params/config.go +++ b/params/config.go @@ -32,45 +32,45 @@ var ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Ethash: new(EthashConfig), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Ethash: new(EthashConfig), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), - HomesteadBlock: big.NewInt(1), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(2), - EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Clique: &CliqueConfig{ Period: 15, @@ -110,7 +110,7 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block - MetropolisBlock *big.Int `json:"metropolisBlock,omitempty"` // Metropolis switch block (nil = no fork, 0 = alraedy on homestead) + ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = alraedy on homestead) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -147,7 +147,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Metropolis: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -155,7 +155,7 @@ func (c *ChainConfig) String() string { c.EIP150Block, c.EIP155Block, c.EIP158Block, - c.MetropolisBlock, + c.ByzantiumBlock, engine, ) } @@ -182,8 +182,8 @@ func (c *ChainConfig) IsEIP158(num *big.Int) bool { return isForked(c.EIP158Block, num) } -func (c *ChainConfig) IsMetropolis(num *big.Int) bool { - return isForked(c.MetropolisBlock, num) +func (c *ChainConfig) IsByzantium(num *big.Int) bool { + return isForked(c.ByzantiumBlock, num) } // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). @@ -243,8 +243,8 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if c.IsEIP158(head) && !configNumEqual(c.ChainId, newcfg.ChainId) { return newCompatError("EIP158 chain ID", c.EIP158Block, newcfg.EIP158Block) } - if isForkIncompatible(c.MetropolisBlock, newcfg.MetropolisBlock, head) { - return newCompatError("Metropolis fork block", c.MetropolisBlock, newcfg.MetropolisBlock) + if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { + return newCompatError("Byzantium fork block", c.ByzantiumBlock, newcfg.ByzantiumBlock) } return nil } @@ -312,7 +312,7 @@ func (err *ConfigCompatError) Error() string { type Rules struct { ChainId *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool - IsMetropolis bool + IsByzantium bool } func (c *ChainConfig) Rules(num *big.Int) Rules { @@ -320,5 +320,5 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { if chainId == nil { chainId = new(big.Int) } - return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsMetropolis: c.IsMetropolis(num)} + return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsByzantium: c.IsByzantium(num)} } diff --git a/tests/init.go b/tests/init.go index 0c3fe61d13..a2c633ad67 100644 --- a/tests/init.go +++ b/tests/init.go @@ -45,13 +45,13 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), }, "Byzantium": ¶ms.ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - DAOForkBlock: big.NewInt(0), - MetropolisBlock: big.NewInt(0), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + DAOForkBlock: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), }, "FrontierToHomesteadAt5": ¶ms.ChainConfig{ ChainId: big.NewInt(1), @@ -69,12 +69,12 @@ var Forks = map[string]*params.ChainConfig{ DAOForkSupport: true, }, "EIP158ToByzantiumAt5": ¶ms.ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - MetropolisBlock: big.NewInt(5), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(5), }, } diff --git a/tests/transaction_test.go b/tests/transaction_test.go index 72d43c0ece..c743996c2a 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -37,12 +37,12 @@ func TestTransaction(t *testing.T) { EIP158Block: big.NewInt(0), ChainId: big.NewInt(1), }) - txt.config(`^Metropolis/`, params.ChainConfig{ - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - MetropolisBlock: big.NewInt(0), + txt.config(`^Byzantium/`, params.ChainConfig{ + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), }) txt.walk(t, transactionTestDir, func(t *testing.T, name string, test *TransactionTest) { From 9be07de5396527eb527f3ca0dd402213c0008a3e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 14 Sep 2017 09:35:54 +0200 Subject: [PATCH 39/45] params: Updated finalized gascosts for ECMUL/MODEXP (#15135) * params: Updated finalized gascosts for ECMUL/MODEXP * core,tests: Updates pending new tests * tests: Updated with new tests * core: revert state transition bugfix * tests: Add expected failures due to #15119 --- core/vm/evm.go | 6 +++--- params/protocol_params.go | 4 ++-- tests/block_test.go | 2 -- tests/state_test.go | 10 +++++++++- tests/testdata | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index b0a6f3d269..093c7d4c14 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -299,9 +299,6 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // Create creates a new contract using code as deployment code. func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { - return nil, common.Address{}, gas, nil - } // Depth check execution. Fail if we're trying to execute above the // limit. @@ -334,6 +331,9 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I contract := NewContract(caller, AccountRef(contractAddr), value, gas) contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code) + if evm.vmConfig.NoRecursion && evm.depth > 0 { + return nil, contractAddr, gas, nil + } ret, err = run(evm, snapshot, contract, nil) // check whether the max code size has been exceeded maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize diff --git a/params/protocol_params.go b/params/protocol_params.go index 9c84c7d348..c56faf56f8 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -64,9 +64,9 @@ const ( Ripemd160PerWordGas uint64 = 120 // Per-word price for a RIPEMD160 operation IdentityBaseGas uint64 = 15 // Base price for a data copy operation IdentityPerWordGas uint64 = 3 // Per-work price for a data copy operation - ModExpQuadCoeffDiv uint64 = 100 // Divisor for the quadratic particle of the big int modular exponentiation + ModExpQuadCoeffDiv uint64 = 20 // Divisor for the quadratic particle of the big int modular exponentiation Bn256AddGas uint64 = 500 // Gas needed for an elliptic curve addition - Bn256ScalarMulGas uint64 = 2000 // Gas needed for an elliptic curve scalar multiplication + Bn256ScalarMulGas uint64 = 40000 // Gas needed for an elliptic curve scalar multiplication Bn256PairingBaseGas uint64 = 100000 // Base price for an elliptic curve pairing check Bn256PairingPerPointGas uint64 = 80000 // Per-point price for an elliptic curve pairing check ) diff --git a/tests/block_test.go b/tests/block_test.go index 56e1e1e8da..669d3ca08a 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -32,8 +32,6 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) // Constantinople is not implemented yet. bt.skipLoad(`(?i)(constantinople)`) - // Expected failures: - bt.fails(`^TransitionTests/bcHomesteadToDao/DaoTransactions(|_UncleExtradata|_EmptyTransactionAndForkBlocksAhead)\.json`, "issue in test") // Still failing tests bt.skipLoad(`^bcWalletTest.*_Byzantium$`) diff --git a/tests/state_test.go b/tests/state_test.go index 1fb7f5908f..9a7430fbeb 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -39,6 +39,13 @@ func TestState(t *testing.T) { st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/Byzantium`, "bug in test") + st.fails( `^stRandom/randomStatetest645\.json/EIP150/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest645\.json/Frontier/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest645\.json/Homestead/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/EIP150/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/Frontier/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/Homestead/.*`, "known bug #15119") + st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { @@ -59,7 +66,8 @@ func TestState(t *testing.T) { } // Transactions with gasLimit above this value will not get a VM trace on failure. -const traceErrorLimit = 400000 +//const traceErrorLimit = 400000 +const traceErrorLimit = 0 func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { err := test(vm.Config{}) diff --git a/tests/testdata b/tests/testdata index 1d30b47956..ca41e90635 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 1d30b4795664f64b1b157971754e14a10cfd9115 +Subproject commit ca41e906351209481bce3a1b35501f25a79023c5 From 701d60c889f885b3995122a5f9bb9656b75857bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Sep 2017 10:59:05 +0300 Subject: [PATCH 40/45] params: enable Byzantium on Ropsten/tests, fix failures --- core/blockchain_test.go | 2 +- core/dao_test.go | 36 ++++++++++++++++++++++-------------- params/config.go | 6 +++--- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 700a30a16f..cb1df8d4b0 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -928,7 +928,7 @@ func TestReorgSideEvent(t *testing.T) { replacementBlocks, _ := GenerateChain(gspec.Config, genesis, db, 4, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), big.NewInt(1000000), new(big.Int), nil), signer, key1) if i == 2 { - gen.OffsetTime(-1) + gen.OffsetTime(-9) } if err != nil { t.Fatalf("failed to create tx: %v", err) diff --git a/core/dao_test.go b/core/dao_test.go index d6e11d78ae..b9898ff7c8 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -40,14 +40,22 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create the concurrent, conflicting two nodes proDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(proDb) - proConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true} - proBc, _ := NewBlockChain(proDb, proConf, ethash.NewFaker(), vm.Config{}) + + proConf := *params.TestChainConfig + proConf.DAOForkBlock = forkBlock + proConf.DAOForkSupport = true + + proBc, _ := NewBlockChain(proDb, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() conDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(conDb) - conConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false} - conBc, _ := NewBlockChain(conDb, conConf, ethash.NewFaker(), vm.Config{}) + + conConf := *params.TestChainConfig + conConf.DAOForkBlock = forkBlock + conConf.DAOForkSupport = false + + conBc, _ := NewBlockChain(conDb, &conConf, ethash.NewFaker(), vm.Config{}) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -61,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -71,19 +79,19 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err == nil { t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0]) } // Create a proper no-fork block for the contra-forker - blocks, _ = GenerateChain(conConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -93,12 +101,12 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err == nil { t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0]) } // Create a proper pro-fork block for the pro-forker - blocks, _ = GenerateChain(proConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accepted pro-fork block: %v", err) } @@ -106,7 +114,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -116,14 +124,14 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -133,7 +141,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err) } diff --git a/params/config.go b/params/config.go index 324620ad5c..5c6f0b6bf7 100644 --- a/params/config.go +++ b/params/config.go @@ -55,7 +55,7 @@ var ( EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), EIP155Block: big.NewInt(10), EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ByzantiumBlock: big.NewInt(1700000), Ethash: new(EthashConfig), } @@ -86,8 +86,8 @@ var ( // means that all fields must be set at all times. This forces // anyone adding flags to the config to also have to set these // fields. - AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(math.MaxInt64) /*disabled*/, new(EthashConfig), nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) From 6c6c7b2af3efdad4d2f64f70f3a724af434bbcd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Sep 2017 13:55:42 +0300 Subject: [PATCH 41/45] params: release Geth 1.7.0 - Megara --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 0896662606..b43cecf704 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 7 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 7 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From a989cf5bad3ec7661c1df17df6a67b6b340274d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Sep 2017 14:01:31 +0300 Subject: [PATCH 42/45] VERSION, params: begin 1.7.1 release cycle --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index bd8bf882d0..943f9cbc4e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 +1.7.1 diff --git a/params/version.go b/params/version.go index b43cecf704..80329c91d9 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 7 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 7 // Minor version component of the current release + VersionPatch = 1 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5705ad004e01054c6aac173348b74c1097b40ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Sep 2017 14:01:44 +0300 Subject: [PATCH 43/45] containers/docker: bump docker images to 1.7 release branch --- containers/docker/master-alpine/Dockerfile | 2 +- containers/docker/master-ubuntu/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile index b2467f1a78..44402361d7 100644 --- a/containers/docker/master-alpine/Dockerfile +++ b/containers/docker/master-alpine/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 RUN \ apk add --update go git make gcc musl-dev linux-headers ca-certificates && \ - git clone --depth 1 --branch release/1.6 https://github.com/ethereum/go-ethereum && \ + git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ apk del go git make gcc musl-dev linux-headers && \ diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile index 877ae94e99..cc4fad10c4 100644 --- a/containers/docker/master-ubuntu/Dockerfile +++ b/containers/docker/master-ubuntu/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:xenial RUN \ apt-get update && apt-get upgrade -q -y && \ apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \ - git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \ + git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \ From c197d805f71ea45eaed7f8c692401502ba92d978 Mon Sep 17 00:00:00 2001 From: Kyuntae Ethan Kim Date: Fri, 15 Sep 2017 18:30:17 +0900 Subject: [PATCH 44/45] ethereum: fix typos in interfaces.go (#15149) --- interfaces.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interfaces.go b/interfaces.go index 744f07b95a..67f236ef75 100644 --- a/interfaces.go +++ b/interfaces.go @@ -102,7 +102,7 @@ type SyncProgress struct { CurrentBlock uint64 // Current block number where sync is at HighestBlock uint64 // Highest alleged block number in the chain PulledStates uint64 // Number of state trie entries already downloaded - KnownStates uint64 // Total number os state trie entries known about + KnownStates uint64 // Total number of state trie entries known about } // ChainSyncReader wraps access to the node's current sync status. If there's no @@ -129,7 +129,7 @@ type ContractCaller interface { CallContract(ctx context.Context, call CallMsg, blockNumber *big.Int) ([]byte, error) } -// FilterQuery contains options for contact log filtering. +// FilterQuery contains options for contract log filtering. type FilterQuery struct { FromBlock *big.Int // beginning of the queried range, nil means genesis block ToBlock *big.Int // end of the range, nil means latest block From 019dca9ba2f0ad91047129b95d91b95a9e71650e Mon Sep 17 00:00:00 2001 From: Dave Appleton Date: Fri, 15 Sep 2017 21:20:29 +0800 Subject: [PATCH 45/45] accounts/abi/backends: add AdjustTime (#15077) --- accounts/abi/bind/backends/simulated.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 88fb3331e8..d6d16eb3f9 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "sync" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -284,6 +285,22 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa return nil } +// JumpTimeInSeconds adds skip seconds to the clock +func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { + b.mu.Lock() + defer b.mu.Unlock() + blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) { + for _, tx := range b.pendingBlock.Transactions() { + block.AddTx(tx) + } + block.OffsetTime(int64(adjustment.Seconds())) + }) + b.pendingBlock = blocks[0] + b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) + + return nil +} + // callmsg implements core.Message to allow passing it as a transaction simulator. type callmsg struct { ethereum.CallMsg