diff --git a/cmd/geth/main.go b/cmd/geth/main.go
index f2bb275520..abf0303484 100644
--- a/cmd/geth/main.go
+++ b/cmd/geth/main.go
@@ -291,6 +291,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso
utils.FastSyncFlag,
utils.CacheFlag,
utils.LightKDFFlag,
+ utils.StateRetentionFlag,
utils.JSpathFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,
diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go
index 7a6ff704c4..7c02544f41 100644
--- a/cmd/geth/usage.go
+++ b/cmd/geth/usage.go
@@ -69,6 +69,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.GenesisFileFlag,
utils.IdentityFlag,
utils.FastSyncFlag,
+ utils.StateRetentionFlag,
utils.LightKDFFlag,
utils.CacheFlag,
utils.BlockchainVersionFlag,
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 63efa08ee3..6246d35b99 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -163,6 +163,11 @@ var (
Name: "lightkdf",
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
}
+ StateRetentionFlag = cli.IntFlag{
+ Name: "retain",
+ Usage: "Number of recent state tries to retain (0 = keep all history)",
+ Value: 5000,
+ }
// Miner settings
// TODO: refactor CPU vs GPU mining flags
MiningEnabledFlag = cli.BoolFlag{
@@ -613,6 +618,7 @@ func MakeSystemNode(name, version string, extra []byte, ctx *cli.Context) *node.
ethConf := ð.Config{
Genesis: MakeGenesisBlock(ctx),
FastSync: ctx.GlobalBool(FastSyncFlag.Name),
+ StateRetention: ctx.GlobalInt(StateRetentionFlag.Name),
BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name),
DatabaseCache: ctx.GlobalInt(CacheFlag.Name),
NetworkId: ctx.GlobalInt(NetworkIdFlag.Name),
diff --git a/core/blockchain.go b/core/blockchain.go
index 95ed06d8d6..0878597b36 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -1195,11 +1195,9 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
return i, err
}
// Write state changes to database
- _, err = statedb.Commit()
- if err != nil {
+ if _, err = statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil {
return i, err
}
-
// coalesce logs for later processing
coalescedLogs = append(coalescedLogs, logs...)
diff --git a/core/blockchain_test.go b/core/blockchain_test.go
index b4ac1696a3..7afc829efd 100644
--- a/core/blockchain_test.go
+++ b/core/blockchain_test.go
@@ -153,7 +153,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
blockchain.mu.Lock()
WriteTd(blockchain.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), blockchain.GetTd(block.ParentHash())))
WriteBlock(blockchain.chainDb, block)
- statedb.Commit()
+ statedb.CommitIndexed([]common.Hash{block.Hash()})
blockchain.mu.Unlock()
}
return nil
diff --git a/core/chain_makers.go b/core/chain_makers.go
index c62618e6cb..761d61901d 100644
--- a/core/chain_makers.go
+++ b/core/chain_makers.go
@@ -171,12 +171,14 @@ func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int,
gen(i, b)
}
AccumulateRewards(statedb, h, b.uncles)
- root, err := statedb.Commit()
- if err != nil {
+
+ h.Root = statedb.IntermediateRoot()
+ block := types.NewBlock(h, b.txs, b.uncles, b.receipts)
+
+ if _, err := statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil {
panic(fmt.Sprintf("state write error: %v", err))
}
- h.Root = root
- return types.NewBlock(h, b.txs, b.uncles, b.receipts), b.receipts
+ return block, b.receipts
}
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), db)
diff --git a/core/genesis.go b/core/genesis.go
index d8c6e9cea3..ed4949194c 100644
--- a/core/genesis.go
+++ b/core/genesis.go
@@ -72,8 +72,6 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block,
statedb.SetState(address, common.HexToHash(key), common.HexToHash(value))
}
}
- root, stateBatch := statedb.CommitBatch()
-
difficulty := common.String2Big(genesis.Difficulty)
block := types.NewBlock(&types.Header{
Nonce: types.EncodeNonce(common.String2Big(genesis.Nonce).Uint64()),
@@ -84,7 +82,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block,
Difficulty: difficulty,
MixDigest: common.HexToHash(genesis.Mixhash),
Coinbase: common.HexToAddress(genesis.Coinbase),
- Root: root,
+ Root: statedb.IntermediateRoot(),
}, nil, nil, nil)
if block := GetBlock(chainDb, block.Hash()); block != nil {
@@ -95,8 +93,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block,
}
return block, nil
}
-
- if err := stateBatch.Write(); err != nil {
+ if _, err := statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil {
return nil, fmt.Errorf("cannot write state: %v", err)
}
if err := WriteTd(chainDb, block.Hash(), difficulty); err != nil {
@@ -123,15 +120,16 @@ func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big
statedb, _ := state.New(common.Hash{}, db)
obj := statedb.GetOrNewStateObject(addr)
obj.SetBalance(balance)
- root, err := statedb.Commit()
- if err != nil {
- panic(fmt.Sprintf("cannot write state: %v", err))
- }
+
block := types.NewBlock(&types.Header{
Difficulty: params.GenesisDifficulty,
GasLimit: params.GenesisGasLimit,
- Root: root,
+ Root: statedb.IntermediateRoot(),
}, nil, nil, nil)
+
+ if _, err := statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil {
+ panic(fmt.Sprintf("cannot write state: %v", err))
+ }
return block
}
diff --git a/core/index_test.go b/core/index_test.go
new file mode 100644
index 0000000000..2d4a8635b3
--- /dev/null
+++ b/core/index_test.go
@@ -0,0 +1,155 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package core
+
+import (
+ "bytes"
+ "math/big"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/event"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// blackHoleContract is a trivial Ethereum contract that can just store some data
+// points in its state trie. It's goal is to have a means to test functionality
+// depending on evolving state roots.
+//
+// contract BlackHole {
+// mapping (int256 => uint) public data;
+//
+// function set(int256 key, uint value) {
+// data[key] = value;
+// }
+// }
+var blackHoleContract = common.Hex2Bytes("6060604052605f8060106000396000f3606060405260e060020a60003504639398e0cd81146024578063a22c554014603b575b005b605560043560006020819052908152604090205481565b600435600090815260208190526040902060243590556022565b6060908152602090f3")
+var blackHoleSetter = common.Hex2Bytes("a22c5540")
+
+// Tests that state trie indexes are properly constructed for an entire chain of
+// imported blocks, cross referencing between various state roots, as well as
+// making sure that state updates do not lose reference entries.
+func TestChainIndex(t *testing.T) {
+ // Configure the test chain and ensure we have enough funds to play with
+ var (
+ key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+ key3, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
+ addr1 = crypto.PubkeyToAddress(key1.PublicKey)
+ addr2 = crypto.PubkeyToAddress(key2.PublicKey)
+ addr3 = crypto.PubkeyToAddress(key3.PublicKey)
+ db, _ = ethdb.NewMemDatabase()
+ )
+ genesis := WriteGenesisBlockForTesting(db, GenesisAccount{addr1, big.NewInt(10000000000)})
+
+ // Generate a chain will all kinds of events happening in it
+ var contract common.Address
+
+ chain, _ := GenerateChain(genesis, db, 8, func(i int, gen *BlockGen) {
+ switch i {
+ case 0:
+ // In block 1, addr1 sends addr2 some ether.
+ tx, _ := types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil).SignECDSA(key1)
+ gen.AddTx(tx)
+ case 1:
+ // In block 2, addr1 sends some more ether to addr2.
+ // addr2 passes it on to addr3.
+ tx1, _ := types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key1)
+ tx2, _ := types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key2)
+ gen.AddTx(tx1)
+ gen.AddTx(tx2)
+ case 2:
+ // Block 3 is empty but was mined by addr3.
+ gen.SetCoinbase(addr3)
+ gen.SetExtra([]byte("yeehaw"))
+ case 3:
+ // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data).
+ b2 := gen.PrevBlock(1).Header()
+ b2.Extra = []byte("foo")
+ gen.AddUncle(b2)
+ b3 := gen.PrevBlock(2).Header()
+ b3.Extra = []byte("foo")
+ gen.AddUncle(b3)
+ case 4:
+ // In block 5, we create a simple storage contract to add data entries to
+ tx, _ := types.NewContractCreation(gen.TxNonce(addr1), big.NewInt(31415), params.GenesisGasLimit, big.NewInt(1), blackHoleContract).SignECDSA(key1)
+ contract = crypto.CreateAddress(addr1, tx.Nonce())
+ gen.AddTx(tx)
+ case 5:
+ // In block 6, we store a single entry into the storage to check single update indexing
+ key := common.Hex2BytesFixed("01", 32)
+ val := common.Hex2BytesFixed("02", 32)
+
+ tx, _ := types.NewTransaction(gen.TxNonce(addr1), contract, nil, params.GenesisGasLimit, nil, append(append(blackHoleSetter, key...), val...)).SignECDSA(key1)
+ gen.AddTx(tx)
+ case 6:
+ // In block 7, we store a lot of entries into the storage to check multi update indexing
+ for i := int64(0); i < 50; i++ {
+ key := common.Hex2BytesFixed(common.Bytes2Hex(big.NewInt(i).Bytes()), 32)
+ val := common.Hex2BytesFixed(common.Bytes2Hex(big.NewInt(i+2).Bytes()), 32)
+
+ tx, _ := types.NewTransaction(gen.TxNonce(addr1), contract, nil, big.NewInt(45000), nil, append(append(blackHoleSetter, key...), val...)).SignECDSA(key1)
+ gen.AddTx(tx)
+ }
+ }
+ })
+ // Import the chain. This runs all block validation rules.
+ evmux := &event.TypeMux{}
+
+ blockchain, _ := NewBlockChain(db, FakePow{}, evmux)
+ if i, err := blockchain.InsertChain(chain); err != nil {
+ t.Fatalf("block %d: insert failed: %v\n", i, err)
+ return
+ }
+ // Iterate over all the blocks and check the state trie indexes
+ indexes := make(map[string]struct{})
+
+ for num := uint64(0); num <= blockchain.CurrentBlock().NumberU64(); num++ {
+ // Gather all the indexes that should be present in the state trie
+ block := blockchain.GetBlockByNumber(num)
+
+ stateDb, err := state.New(block.Root(), db)
+ if err != nil {
+ t.Fatalf("failed to create state trie at %x: %v", block.Root(), err)
+ }
+ for it := state.NewNodeIterator(stateDb); it.Next(); {
+ if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+ indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+ }
+ }
+ // Add the block->state index entries
+ indexes[string(trie.ParentReferenceIndexKey(block.Hash().Bytes(), block.Root().Bytes()))] = struct{}{}
+ }
+ // Cross check the indexes and the database itself
+ for index, _ := range indexes {
+ if _, err := db.Get([]byte(index)); err != nil {
+ t.Errorf("failed to retrieve reported index %x: %v", index, err)
+ }
+ }
+ for _, key := range db.Keys() {
+ if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+ if _, ok := indexes[string(key)]; !ok {
+ t.Errorf("index entry not reported %x", key)
+ }
+ }
+ }
+}
diff --git a/core/state/index_test.go b/core/state/index_test.go
new file mode 100644
index 0000000000..8b349857eb
--- /dev/null
+++ b/core/state/index_test.go
@@ -0,0 +1,67 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package state
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// Tests that all index entries are stored in the database after a state commit.
+func TestStateIndexDangling(t *testing.T) {
+ testStateIndex(t, nil)
+}
+func TestStateIndexRooted(t *testing.T) {
+ testStateIndex(t, []common.Hash{common.BytesToHash([]byte{0x01}), common.BytesToHash([]byte{0x02, 0x03})})
+}
+
+func testStateIndex(t *testing.T, referrers []common.Hash) {
+ // Create some arbitrary test state to iterate
+ db, root, _ := makeTestState(referrers)
+
+ state, err := New(root, db)
+ if err != nil {
+ t.Fatalf("failed to create state trie at %x: %v", root, err)
+ }
+ // Gather all the indexes that should be present in the database
+ indexes := make(map[string]struct{})
+ for it := NewNodeIterator(state); it.Next(); {
+ if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+ indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+ }
+ }
+ for _, referrer := range referrers {
+ indexes[string(trie.ParentReferenceIndexKey(referrer.Bytes(), root.Bytes()))] = struct{}{}
+ }
+ // Cross check the indexes and the database itself
+ for index, _ := range indexes {
+ if _, err := db.Get([]byte(index)); err != nil {
+ t.Errorf("failed to retrieve reported index %x: %v", index, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+ if _, ok := indexes[string(key)]; !ok {
+ t.Errorf("index entry not reported %x", key)
+ }
+ }
+ }
+}
diff --git a/core/state/iterator.go b/core/state/iterator.go
new file mode 100644
index 0000000000..84259ac512
--- /dev/null
+++ b/core/state/iterator.go
@@ -0,0 +1,160 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package state
+
+import (
+ "bytes"
+ "fmt"
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// NodeIteratorSubtreeCallback is a callback that is invoked by the trie node
+// iterator whenever it descends into a new subtree, giving the possibility of
+// skipping the branch if it would be non-useful (e.g. already processed before).
+type NodeIteratorSubtreeCallback func(hash, parent common.Hash) bool
+
+// NodeIterator is an iterator to traverse the entire state trie post-order,
+// including all of the contract code and contract state tries.
+type NodeIterator struct {
+ state *StateDB // State being iterated
+
+ stateIt *trie.NodeIterator // Primary iterator for the global state trie
+ dataIt *trie.NodeIterator // Secondary iterator for the data trie of a contract
+
+ accountHash common.Hash // Hash of the node containing the account
+ codeHash common.Hash // Hash of the contract source code
+ code []byte // Source code associated with a contract
+
+ // Callback method enabling the user to control subtree descent.
+ //
+ // Note: This hook is invoked pre-order to allow the user to cancel the traversal
+ // of a subtree, but the iterator itself is post-order. This means that the hook
+ // may be invoked multiple times during a single iteration step (when descending)
+ // or never in multiple iteration steps (when ascending).
+ PreOrderHook NodeIteratorSubtreeCallback
+
+ Hash common.Hash // Hash of the current entry being iterated (nil if not standalone)
+ Entry interface{} // Current state entry being iterated (internal representation)
+ Parent common.Hash // Hash of the first full ancestor node (nil if current is the root)
+}
+
+// NewNodeIterator creates an post-order state node iterator.
+func NewNodeIterator(state *StateDB) *NodeIterator {
+ return &NodeIterator{
+ state: state,
+ }
+}
+
+// Next moves the iterator to the next node, returning whether there are any
+// further nodes.
+func (it *NodeIterator) Next() bool {
+ it.step()
+ return it.retrieve()
+}
+
+// step moves the iterator to the next entry of the state trie.
+func (it *NodeIterator) step() {
+ // Abort if we reached the end of the iteration
+ if it.state == nil {
+ return
+ }
+ // Initialize the iterator if we've just started
+ if it.stateIt == nil {
+ it.stateIt = trie.NewNodeIterator(it.state.trie.Trie)
+ it.stateIt.PreOrderHook = trie.NodeIteratorSubtreeCallback(it.PreOrderHook)
+ }
+ // If we had data nodes previously, we surely have at least state nodes
+ if it.dataIt != nil {
+ if cont := it.dataIt.Next(); !cont {
+ it.dataIt = nil
+ }
+ return
+ }
+ // If we had source code previously, discard that
+ if it.code != nil {
+ it.code = nil
+ return
+ }
+ // Step to the next state trie node, terminating if we're out of nodes
+ if cont := it.stateIt.Next(); !cont {
+ it.state, it.stateIt = nil, nil
+ return
+ }
+ // If the state trie node is an internal entry, leave as is
+ if !it.stateIt.Leaf {
+ return
+ }
+ // Otherwise we've reached an account node, initiate data iteration
+ var account struct {
+ Nonce uint64
+ Balance *big.Int
+ Root common.Hash
+ CodeHash []byte
+ }
+ err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob), &account)
+ if err != nil {
+ panic(err)
+ }
+ it.accountHash = it.stateIt.Parent
+
+ if bytes.Compare(account.CodeHash, emptyCodeHash) != 0 {
+ if hash := common.BytesToHash(account.CodeHash); it.PreOrderHook == nil || it.PreOrderHook(hash, it.accountHash) {
+ it.codeHash = hash
+ if it.code, err = it.state.db.Get(account.CodeHash); err != nil {
+ panic(fmt.Sprintf("code %x: %v", account.CodeHash, err))
+ }
+ }
+ }
+ dataTrie, err := trie.New(account.Root, it.state.db)
+ if err != nil {
+ panic(err)
+ }
+ it.dataIt = trie.NewNodeIterator(dataTrie)
+ it.dataIt.PreOrderHook = trie.NodeIteratorSubtreeCallback(it.PreOrderHook)
+ if !it.dataIt.Next() {
+ it.dataIt = nil
+ }
+}
+
+// retrieve pulls and caches the current state entry the iterator is traversing.
+// The method returns whether there are any more data left for inspection.
+func (it *NodeIterator) retrieve() bool {
+ // Clear out any previously set values
+ it.Hash, it.Entry = common.Hash{}, nil
+
+ // If the iteration's done, return no available data
+ if it.state == nil {
+ return false
+ }
+ // Otherwise retrieve the current entry
+ switch {
+ case it.dataIt != nil:
+ it.Hash, it.Entry, it.Parent = it.dataIt.Hash, it.dataIt.Node, it.dataIt.Parent
+ if it.Parent == (common.Hash{}) {
+ it.Parent = it.accountHash
+ }
+ case it.code != nil:
+ it.Hash, it.Entry, it.Parent = it.codeHash, it.code, it.accountHash
+ case it.stateIt != nil:
+ it.Hash, it.Entry, it.Parent = it.stateIt.Hash, it.stateIt.Node, it.stateIt.Parent
+ }
+ return true
+}
diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go
new file mode 100644
index 0000000000..0e7aba78c6
--- /dev/null
+++ b/core/state/iterator_test.go
@@ -0,0 +1,121 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package state
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// Tests that the node iterator indeed walks over the entire database contents.
+func TestNodeIteratorCoverage(t *testing.T) {
+ // Create some arbitrary test state to iterate
+ db, root, _ := makeTestState(nil)
+
+ state, err := New(root, db)
+ if err != nil {
+ t.Fatalf("failed to create state trie at %x: %v", root, err)
+ }
+ // Gather all the node hashes found by the iterator
+ hashes := make(map[common.Hash]struct{})
+ for it := NewNodeIterator(state); it.Next(); {
+ if it.Hash != (common.Hash{}) {
+ hashes[it.Hash] = struct{}{}
+ }
+ }
+ // Cross check the hashes and the database itself
+ for hash, _ := range hashes {
+ if _, err := db.Get(hash.Bytes()); err != nil {
+ t.Errorf("failed to retrieve reported node %x: %v", hash, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if bytes.HasPrefix(key, []byte("secure-key-")) {
+ continue
+ }
+ if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+ continue
+ }
+ if _, ok := hashes[common.BytesToHash(key)]; !ok {
+ t.Errorf("state entry not reported %x", key)
+ }
+ }
+}
+
+// Tests that the node iterator hook is invoked for all the nodes of the state,
+// also testing that the iterator indeed avoids visiting aborted branches.
+func TestNodeIteratorHookCoverageFull(t *testing.T) { testNodeIteratorHookCoverage(t, false) }
+func TestNodeIteratorHookCoverageDedup(t *testing.T) { testNodeIteratorHookCoverage(t, true) }
+
+func testNodeIteratorHookCoverage(t *testing.T, dedup bool) {
+ // Create some arbitrary test state to iterate
+ db, root, _ := makeTestState(nil)
+
+ state, err := New(root, db)
+ if err != nil {
+ t.Fatalf("failed to create state trie at %x: %v", root, err)
+ }
+ // Gather all the node hashes found by the iterator
+ hashes := make(map[common.Hash]int)
+
+ it := NewNodeIterator(state)
+ it.PreOrderHook = func(hash, parent common.Hash) bool {
+ if !dedup {
+ return true
+ }
+ return hashes[hash] == 0
+ }
+ for it.Next() {
+ if it.Hash != (common.Hash{}) {
+ hashes[it.Hash]++
+ }
+ }
+ // Cross check the hashes and the database itself
+ for hash, _ := range hashes {
+ if _, err := db.Get(hash.Bytes()); err != nil {
+ t.Errorf("failed to retrieve reported node %x: %v", hash, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if bytes.HasPrefix(key, []byte("secure-key-")) {
+ continue
+ }
+ if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+ continue
+ }
+ if _, ok := hashes[common.BytesToHash(key)]; !ok {
+ t.Errorf("state entry not reported %x", key)
+ }
+ }
+ // Check whether duplicates were avoided or not
+ duplicates := 0
+ for hash, count := range hashes {
+ if count > 1 {
+ duplicates++
+ if dedup {
+ t.Errorf("duplicate (%d) iteration: %x", count, hash)
+ }
+ }
+ }
+ if !dedup && duplicates == 0 {
+ t.Errorf("iterator didn't traverse common subtrees")
+ }
+}
diff --git a/core/state/state_object.go b/core/state/state_object.go
index 47546112f4..292f32f301 100644
--- a/core/state/state_object.go
+++ b/core/state/state_object.go
@@ -118,7 +118,7 @@ func (c *StateObject) setAddr(addr []byte, value common.Hash) {
// if RLPing failed we better panic and not fail silently. This would be considered a consensus issue
panic(err)
}
- c.trie.Update(addr, v)
+ c.trie.UpdateIndexed(addr, v, nil)
}
func (self *StateObject) Storage() Storage {
diff --git a/core/state/state_test.go b/core/state/state_test.go
index 7ce341c361..040683ae9d 100644
--- a/core/state/state_test.go
+++ b/core/state/state_test.go
@@ -89,7 +89,7 @@ func TestNull(t *testing.T) {
//value := common.FromHex("0x823140710bf13990e4500136726d8b55")
var value common.Hash
state.SetState(address, common.Hash{}, value)
- state.Commit()
+ state.CommitIndexed(nil)
value = state.GetState(address, common.Hash{})
if !common.EmptyHash(value) {
t.Errorf("expected empty hash. got %x", value)
diff --git a/core/state/statedb.go b/core/state/statedb.go
index 8093472b59..1077e3ccd8 100644
--- a/core/state/statedb.go
+++ b/core/state/statedb.go
@@ -206,15 +206,17 @@ func (self *StateDB) Delete(addr common.Address) bool {
// Update the given state object and apply it to state trie
func (self *StateDB) UpdateStateObject(stateObject *StateObject) {
+ refs := [][]byte{stateObject.Root()}
if len(stateObject.code) > 0 {
self.db.Put(stateObject.codeHash, stateObject.code)
+ refs = append(refs, stateObject.codeHash)
}
addr := stateObject.Address()
data, err := rlp.EncodeToBytes(stateObject)
if err != nil {
panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
}
- self.trie.Update(addr[:], data)
+ self.trie.UpdateIndexed(addr[:], data, refs)
}
// Delete the given state object and delete it from the state trie
@@ -351,22 +353,22 @@ func (s *StateDB) IntermediateRoot() common.Hash {
return s.trie.Hash()
}
-// Commit commits all state changes to the database.
-func (s *StateDB) Commit() (root common.Hash, err error) {
- root, batch := s.CommitBatch()
+// CommitIndexed commits all state changes to the database.
+func (s *StateDB) CommitIndexed(refs []common.Hash) (root common.Hash, err error) {
+ root, batch := s.CommitBatchIndexed(refs)
return root, batch.Write()
}
-// CommitBatch commits all state changes to a write batch but does not
+// CommitBatchIndexed commits all state changes to a write batch but does not
// execute the batch. It is used to validate state changes against
// the root hash stored in a block.
-func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) {
+func (s *StateDB) CommitBatchIndexed(refs []common.Hash) (root common.Hash, batch ethdb.Batch) {
batch = s.db.NewBatch()
- root, _ = s.commit(batch)
+ root, _ = s.commit(batch, refs)
return root, batch
}
-func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) {
+func (s *StateDB) commit(db trie.DatabaseWriter, refs []common.Hash) (common.Hash, error) {
s.refund = new(big.Int)
for _, stateObject := range s.stateObjects {
@@ -381,7 +383,7 @@ func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) {
// This updates the trie root internally, so
// getting the root hash of the storage trie
// through UpdateStateObject is fast.
- if _, err := stateObject.trie.CommitTo(db); err != nil {
+ if _, err := stateObject.trie.CommitToIndexed(db, nil); err != nil {
return common.Hash{}, err
}
// Update the object in the account trie.
@@ -389,7 +391,11 @@ func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) {
}
stateObject.dirty = false
}
- return s.trie.CommitTo(db)
+ referrers := make([][]byte, len(refs))
+ for i, ref := range refs {
+ referrers[i] = ref.Bytes()
+ }
+ return s.trie.CommitToIndexed(db, referrers)
}
func (self *StateDB) Refunds() *big.Int {
diff --git a/core/state/sync.go b/core/state/sync.go
index ef2b4b84c5..5d77517e7e 100644
--- a/core/state/sync.go
+++ b/core/state/sync.go
@@ -32,7 +32,7 @@ import (
type StateSync trie.TrieSync
// NewStateSync create a new state trie download scheduler.
-func NewStateSync(root common.Hash, database ethdb.Database) *StateSync {
+func NewStateSync(root common.Hash, database ethdb.Database, parent common.Hash) *StateSync {
var syncer *trie.TrieSync
callback := func(leaf []byte, parent common.Hash) error {
@@ -50,7 +50,7 @@ func NewStateSync(root common.Hash, database ethdb.Database) *StateSync {
return nil
}
- syncer = trie.NewTrieSync(root, database, callback)
+ syncer = trie.NewTrieSync(root, database, parent, callback)
return (*StateSync)(syncer)
}
diff --git a/core/state/sync_test.go b/core/state/sync_test.go
index 0dab372ba0..e975a317ec 100644
--- a/core/state/sync_test.go
+++ b/core/state/sync_test.go
@@ -18,10 +18,12 @@ package state
import (
"bytes"
+ "fmt"
"math/big"
"testing"
"github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
)
@@ -35,31 +37,36 @@ type testAccount struct {
}
// makeTestState create a sample test state to test node-wise reconstruction.
-func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
+func makeTestState(referrers []common.Hash) (ethdb.Database, common.Hash, []*testAccount) {
// Create an empty state
db, _ := ethdb.NewMemDatabase()
state, _ := New(common.Hash{}, db)
// Fill it with some arbitrary data
accounts := []*testAccount{}
- for i := byte(0); i < 255; i++ {
- obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
- acc := &testAccount{address: common.BytesToAddress([]byte{i})}
+ for i := byte(0); i < 1; i++ {
+ for j := byte(0); j < 2; j++ {
+ obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i, j}))
+ acc := &testAccount{address: common.BytesToAddress([]byte{i, j})}
- obj.AddBalance(big.NewInt(int64(11 * i)))
- acc.balance = big.NewInt(int64(11 * i))
+ obj.AddBalance(big.NewInt(int64(11 * i)))
+ acc.balance = big.NewInt(int64(11 * i))
- obj.SetNonce(uint64(42 * i))
- acc.nonce = uint64(42 * i)
+ obj.SetNonce(uint64(42 * i))
+ acc.nonce = uint64(42 * i)
- if i%3 == 0 {
- obj.SetCode([]byte{i, i, i, i, i})
- acc.code = []byte{i, i, i, i, i}
+ if i%3 == 0 {
+ obj.SetCode([]byte{i, i, i, i, i})
+ acc.code = []byte{i, i, i, i, i}
+ }
+ state.UpdateStateObject(obj)
+ accounts = append(accounts, acc)
}
- state.UpdateStateObject(obj)
- accounts = append(accounts, acc)
}
- root, _ := state.Commit()
+ root, _ := state.CommitIndexed(referrers)
+
+ // Remove any potentially cached data from the test state creation
+ trie.ClearGlobalCache()
// Return the generated state
return db, root, accounts
@@ -67,10 +74,22 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
// checkStateAccounts cross references a reconstructed state with an expected
// account array.
-func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
- state, _ := New(root, db)
- for i, acc := range accounts {
+func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount, parent common.Hash) {
+ // Remove any potentially cached data from the state synchronisation
+ trie.ClearGlobalCache()
+ // Check root availability and state contents
+ state, err := New(root, db)
+ if err != nil {
+ t.Fatalf("failed to create state trie at %x: %v", root, err)
+ }
+ if err := checkStateConsistency(db, root); err != nil {
+ t.Fatalf("inconsistent state trie at %x: %v", root, err)
+ }
+ if err := checkStateIndex(db, root, parent); err != nil {
+ t.Fatalf("index error at %x: %v", root, err)
+ }
+ for i, acc := range accounts {
if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 {
t.Errorf("account %d: balance mismatch: have %v, want %v", i, balance, acc.balance)
}
@@ -83,27 +102,103 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou
}
}
+// checkStateConsistency checks that all nodes in a state trie and indeed present.
+func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error) {
+ // Capture any panics by the iterator
+ defer func() {
+ if r := recover(); r != nil {
+ failure = fmt.Errorf("%v", r)
+ }
+ }()
+ // Remove any potentially cached data from the test state creation or previous checks
+ trie.ClearGlobalCache()
+
+ // Create and iterate a state trie rooted in a sub-node
+ if _, err := db.Get(root.Bytes()); err != nil {
+ return
+ }
+ state, err := New(root, db)
+ if err != nil {
+ return
+ }
+ for it := NewNodeIterator(state); it.Next(); {
+ }
+ return nil
+}
+
+// checkStateIndex iterates over the entire state trie and checks that all required
+// database indexes have been generated by the synchronizer.
+func checkStateIndex(db ethdb.Database, root common.Hash, parent common.Hash) error {
+ // Remove any potentially cached data from the test state creation or previous checks
+ trie.ClearGlobalCache()
+
+ if _, err := db.Get(root.Bytes()); err != nil {
+ return err
+ }
+ state, err := New(root, db)
+ if err != nil {
+ return fmt.Errorf("failed to create state trie at %x: %v", root, err)
+ }
+ // Gather all the indexes that should be present in the database
+ indexes := make(map[string]struct{})
+ for it := NewNodeIterator(state); it.Next(); {
+ if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+ indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+ }
+ }
+ if parent != (common.Hash{}) {
+ indexes[string(trie.ParentReferenceIndexKey(parent.Bytes(), root.Bytes()))] = struct{}{}
+ }
+ // Cross check the indexes and the database itself
+ for index, _ := range indexes {
+ if _, err := db.Get([]byte(index)); err != nil {
+ return fmt.Errorf("failed to retrieve reported index %x: %v", index, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+ if _, ok := indexes[string(key)]; !ok {
+ return fmt.Errorf("index entry not reported %x", key)
+ }
+ }
+ }
+ return nil
+}
+
// Tests that an empty state is not scheduled for syncing.
-func TestEmptyStateSync(t *testing.T) {
+func TestEmptyStateSyncDangling(t *testing.T) { testEmptyStateSync(t, common.Hash{}) }
+func TestEmptyStateSyncRooted(t *testing.T) { testEmptyStateSync(t, common.BytesToHash([]byte{0x01})) }
+
+func testEmptyStateSync(t *testing.T, origin common.Hash) {
empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
db, _ := ethdb.NewMemDatabase()
- if req := NewStateSync(empty, db).Missing(1); len(req) != 0 {
+ if req := NewStateSync(empty, db, origin).Missing(1); len(req) != 0 {
t.Errorf("content requested for empty state: %v", req)
}
}
// Tests that given a root hash, a state can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go.
-func TestIterativeStateSyncIndividual(t *testing.T) { testIterativeStateSync(t, 1) }
-func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, 100) }
+func TestIterativeStateSyncIndividualDangling(t *testing.T) {
+ testIterativeStateSync(t, 1, common.Hash{})
+}
+func TestIterativeStateSyncIndividualRooted(t *testing.T) {
+ testIterativeStateSync(t, 1, common.BytesToHash([]byte{0x02}))
+}
+func TestIterativeStateSyncBatchedDangling(t *testing.T) {
+ testIterativeStateSync(t, 100, common.Hash{})
+}
+func TestIterativeStateSyncBatchedRooted(t *testing.T) {
+ testIterativeStateSync(t, 100, common.BytesToHash([]byte{0x03}))
+}
-func testIterativeStateSync(t *testing.T, batch int) {
+func testIterativeStateSync(t *testing.T, batch int, origin common.Hash) {
// Create a random state to copy
- srcDb, srcRoot, srcAccounts := makeTestState()
+ srcDb, srcRoot, srcAccounts := makeTestState(nil)
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewStateSync(srcRoot, dstDb)
+ sched := NewStateSync(srcRoot, dstDb, origin)
queue := append([]common.Hash{}, sched.Missing(batch)...)
for len(queue) > 0 {
@@ -121,18 +216,25 @@ func testIterativeStateSync(t *testing.T, batch int) {
queue = append(queue[:0], sched.Missing(batch)...)
}
// Cross check that the two states are in sync
- checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned, and the others sent only later.
-func TestIterativeDelayedStateSync(t *testing.T) {
+func TestIterativeDelayedStateSyncDangling(t *testing.T) {
+ testIterativeDelayedStateSync(t, common.Hash{})
+}
+func TestIterativeDelayedStateSyncRooted(t *testing.T) {
+ testIterativeDelayedStateSync(t, common.BytesToHash([]byte{0x04}))
+}
+
+func testIterativeDelayedStateSync(t *testing.T, origin common.Hash) {
// Create a random state to copy
- srcDb, srcRoot, srcAccounts := makeTestState()
+ srcDb, srcRoot, srcAccounts := makeTestState(nil)
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewStateSync(srcRoot, dstDb)
+ sched := NewStateSync(srcRoot, dstDb, origin)
queue := append([]common.Hash{}, sched.Missing(0)...)
for len(queue) > 0 {
@@ -151,22 +253,32 @@ func TestIterativeDelayedStateSync(t *testing.T) {
queue = append(queue[len(results):], sched.Missing(0)...)
}
// Cross check that the two states are in sync
- checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin)
}
// Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go, however in a
// random order.
-func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) }
-func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) }
+func TestIterativeRandomStateSyncIndividualDangling(t *testing.T) {
+ testIterativeRandomStateSync(t, 1, common.Hash{})
+}
+func TestIterativeRandomStateSyncIndividualRooted(t *testing.T) {
+ testIterativeRandomStateSync(t, 1, common.BytesToHash([]byte{0x05}))
+}
+func TestIterativeRandomStateSyncBatchedDangling(t *testing.T) {
+ testIterativeRandomStateSync(t, 100, common.Hash{})
+}
+func TestIterativeRandomStateSyncBatchedRooted(t *testing.T) {
+ testIterativeRandomStateSync(t, 100, common.BytesToHash([]byte{0x06}))
+}
-func testIterativeRandomStateSync(t *testing.T, batch int) {
+func testIterativeRandomStateSync(t *testing.T, batch int, origin common.Hash) {
// Create a random state to copy
- srcDb, srcRoot, srcAccounts := makeTestState()
+ srcDb, srcRoot, srcAccounts := makeTestState(nil)
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewStateSync(srcRoot, dstDb)
+ sched := NewStateSync(srcRoot, dstDb, origin)
queue := make(map[common.Hash]struct{})
for _, hash := range sched.Missing(batch) {
@@ -192,18 +304,25 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
}
}
// Cross check that the two states are in sync
- checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned (Even those randomly), others sent only later.
-func TestIterativeRandomDelayedStateSync(t *testing.T) {
+func TestIterativeRandomDelayedStateSyncDangling(t *testing.T) {
+ testIterativeRandomDelayedStateSync(t, common.Hash{})
+}
+func TestIterativeRandomDelayedStateSyncRooted(t *testing.T) {
+ testIterativeRandomDelayedStateSync(t, common.BytesToHash([]byte{0x07}))
+}
+
+func testIterativeRandomDelayedStateSync(t *testing.T, origin common.Hash) {
// Create a random state to copy
- srcDb, srcRoot, srcAccounts := makeTestState()
+ srcDb, srcRoot, srcAccounts := makeTestState(nil)
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewStateSync(srcRoot, dstDb)
+ sched := NewStateSync(srcRoot, dstDb, origin)
queue := make(map[common.Hash]struct{})
for _, hash := range sched.Missing(0) {
@@ -234,5 +353,67 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
}
}
// Cross check that the two states are in sync
- checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin)
+}
+
+// Tests that at any point in time during a sync, only complete sub-tries are in
+// the database.
+func TestIncompleteStateSync(t *testing.T) {
+ // Create a random state to copy
+ srcDb, srcRoot, srcAccounts := makeTestState(nil)
+
+ // Create a destination state and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewStateSync(srcRoot, dstDb, common.Hash{})
+
+ added := []common.Hash{}
+ queue := append([]common.Hash{}, sched.Missing(1)...)
+ for len(queue) > 0 {
+ // Fetch a batch of state nodes
+ results := make([]trie.SyncResult, len(queue))
+ for i, hash := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results[i] = trie.SyncResult{hash, data}
+ }
+ // Process each of the state nodes
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ for _, result := range results {
+ added = append(added, result.Hash)
+ }
+ // Check that all known sub-tries in the synced state is complete
+ for _, root := range added {
+ // Skim through the accounts and make sure the root hash is not a code node
+ codeHash := false
+ for _, acc := range srcAccounts {
+ if bytes.Compare(root.Bytes(), crypto.Sha3(acc.code)) == 0 {
+ codeHash = true
+ break
+ }
+ }
+ // If the root is a real trie node, check consistency
+ if !codeHash {
+ if err := checkStateConsistency(dstDb, root); err != nil {
+ t.Fatalf("state inconsistent: %v", err)
+ }
+ }
+ }
+ // Fetch the next batch to retrieve
+ queue = append(queue[:0], sched.Missing(1)...)
+ }
+ // Sanity check that removing any node from the database is detected
+ for _, node := range added[1:] {
+ key := node.Bytes()
+ value, _ := dstDb.Get(key)
+
+ dstDb.Delete(key)
+ if err := checkStateConsistency(dstDb, added[0]); err == nil {
+ t.Fatalf("trie inconsistency not caught, missing: %x", key)
+ }
+ dstDb.Put(key, value)
+ }
}
diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go
index 00c42c5bc6..68d803e448 100644
--- a/core/types/derive_sha.go
+++ b/core/types/derive_sha.go
@@ -35,7 +35,7 @@ func DeriveSha(list DerivableList) common.Hash {
for i := 0; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
- trie.Update(keybuf.Bytes(), list.GetRlp(i))
+ trie.UpdateIndexed(keybuf.Bytes(), list.GetRlp(i), nil)
}
return trie.Hash()
}
diff --git a/eth/backend.go b/eth/backend.go
index abd1214caa..6e130a9c89 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/common/compiler"
"github.com/ethereum/go-ethereum/common/httpclient"
"github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/filters"
@@ -45,6 +46,7 @@ import (
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rlp"
rpc "github.com/ethereum/go-ethereum/rpc/v2"
+ "github.com/ethereum/go-ethereum/trie"
)
const (
@@ -61,9 +63,10 @@ var (
)
type Config struct {
- NetworkId int // Network ID to use for selecting peers to connect to
- Genesis string // Genesis JSON to seed the chain database with
- FastSync bool // Enables the state download based fast synchronisation algorithm
+ NetworkId int // Network ID to use for selecting peers to connect to
+ Genesis string // Genesis JSON to seed the chain database with
+ FastSync bool // Enables the state download based fast synchronisation algorithm
+ StateRetention int // Number of recent state tries to retain pruning
BlockChainVersion int
SkipBcVersionCheck bool // e.g. blockchain export
@@ -136,6 +139,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
const dbCount = 3
ethdb.OpenFileLimit = 128 / (dbCount + 1)
+ // Sanity check that the user doesn't throw away more data than what would hurt the network
+ if config.StateRetention != 0 && config.StateRetention < downloader.FsStateRetention {
+ return nil, fmt.Errorf("configured state retention (%d) too low (< %d) for network security", config.StateRetention, downloader.FsStateRetention)
+ }
// Open the chain database and perform any upgrades needed
chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache)
if err != nil {
@@ -150,7 +157,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
if err := addMipmapBloomBins(chainDb); err != nil {
return nil, err
}
-
+ if err := indexStateDatabase(chainDb, uint64(config.StateRetention)); err != nil {
+ return nil, err
+ }
dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache)
if err != nil {
return nil, err
@@ -578,3 +587,63 @@ func addMipmapBloomBins(db ethdb.Database) (err error) {
glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart))
return nil
}
+
+// indexStateDatabase iterates over the state tries of the topmost blocks in the
+// chain and inserts all child-parent references into the trie index to facilitate
+// state trie pruning afterwards.
+func indexStateDatabase(db ethdb.Database, retain uint64) error {
+ // Short circuit if the head block is already indexed
+ head := core.GetBlock(db, core.GetHeadBlockHash(db))
+ if head == nil {
+ return nil // Empty database, don't die on it
+ }
+ if _, err := db.Get(trie.ParentReferenceIndexKey(head.Hash().Bytes(), head.Root().Bytes())); err == nil {
+ return nil // Head already indexed, assume up to date database
+ }
+ // Iterate over the top blocks that are being retained and index all the state tries
+ glog.V(logger.Info).Infoln("Indexing database for state pruning (first block takes a few minutes, please be patient)...")
+
+ from, till := uint64(0), head.NumberU64()
+ if from+retain < till {
+ from = till - retain
+ }
+ for num := from; num <= till; num++ {
+ glog.V(logger.Info).Infof("%.2f%%: Indexing state of block #%d...", 100*float64(num-from)/float64(till-from+1), num)
+
+ // Load the block and ensure any database corruption is signaled
+ header := core.GetHeader(db, core.GetCanonicalHash(db, num))
+ if header == nil {
+ return fmt.Errorf("block #%d missing", num)
+ }
+ // Short circuit if the block has already been indexed previously
+ if _, err := db.Get(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes())); err == nil {
+ continue
+ }
+ // Otherwise iterate the entire state trie and insert the indices
+ stateDb, err := state.New(header.Root, db)
+ if err != nil {
+ return err
+ }
+ it := state.NewNodeIterator(stateDb)
+ it.PreOrderHook = func(hash, parent common.Hash) bool {
+ // If we've already indexed this branch, skip
+ _, err := db.Get(trie.ParentReferenceIndexKey(parent.Bytes(), hash.Bytes()))
+ if err == nil {
+ return false
+ }
+ return true
+ }
+ for it.Next() {
+ if it.Hash != (common.Hash{}) && it.Parent != (common.Hash{}) {
+ if err := db.Put(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()), nil); err != nil {
+ return err
+ }
+ }
+ }
+ if err := db.Put(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes()), nil); err != nil {
+ return err
+ }
+ }
+ glog.V(logger.Info).Infof("Database successfully indexed")
+ return nil
+}
diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go
index 1639947304..4ddd128777 100644
--- a/eth/downloader/downloader.go
+++ b/eth/downloader/downloader.go
@@ -67,6 +67,8 @@ var (
fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point
fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync
+
+ FsStateRetention = fsMinFullBlocks + fsPivotInterval + 1 // Number of state tries needed to reliably fast sync
)
var (
diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go
index 1e55560dbf..1f37ab6c00 100644
--- a/eth/downloader/queue.go
+++ b/eth/downloader/queue.go
@@ -356,7 +356,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
if q.mode == FastSync && header.Number.Uint64() == q.fastSyncPivot {
// Pivoting point of the fast sync, retrieve the state tries
q.stateSchedLock.Lock()
- q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase)
+ q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase, header.Hash())
q.stateSchedLock.Unlock()
}
inserts = append(inserts, header)
diff --git a/light/state_test.go b/light/state_test.go
index 2c2e6daea1..418dee83b5 100644
--- a/light/state_test.go
+++ b/light/state_test.go
@@ -65,7 +65,7 @@ func makeTestState() (common.Hash, ethdb.Database) {
so.Update()
st.UpdateStateObject(so)
}
- root, _ := st.Commit()
+ root, _ := st.CommitIndexed(nil)
return root, sdb
}
diff --git a/light/trie.go b/light/trie.go
index e9c96ea48e..d99fe3ca4c 100644
--- a/light/trie.go
+++ b/light/trie.go
@@ -89,19 +89,20 @@ func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error)
return
}
-// Update associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// UpdateIndexed associates key with value in the trie. Subsequent calls to Get
+// will return value. If value has length zero, any existing value is deleted
+// from the trie and calls to Get will return nil. In addition, state trie index
+// entries are also generated for all entities referencing the current node.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
-func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
+func (t *LightTrie) UpdateIndexed(ctx context.Context, key, value []byte, references [][]byte) (err error) {
err = t.do(ctx, key, func() (err error) {
if t.trie == nil {
t.trie, err = trie.NewSecure(t.originalRoot, t.db)
}
if err == nil {
- err = t.trie.TryUpdate(key, value)
+ err = t.trie.TryUpdateIndexed(key, value, references)
}
return
})
diff --git a/miner/worker.go b/miner/worker.go
index 1a411ae20b..648cbb57fd 100644
--- a/miner/worker.go
+++ b/miner/worker.go
@@ -284,7 +284,7 @@ func (self *worker) wait() {
}
go self.mux.Post(core.NewMinedBlockEvent{block})
} else {
- work.state.Commit()
+ work.state.CommitIndexed([]common.Hash{block.Hash()})
parent := self.chain.GetBlock(block.ParentHash())
if parent == nil {
glog.V(logger.Error).Infoln("Invalid block found during mining")
diff --git a/tests/block_test_util.go b/tests/block_test_util.go
index 473bc3419f..9503497380 100644
--- a/tests/block_test_util.go
+++ b/tests/block_test_util.go
@@ -250,7 +250,7 @@ func (t *BlockTest) InsertPreState(db ethdb.Database, am *accounts.Manager) (*st
}
}
- root, err := statedb.Commit()
+ root, err := statedb.CommitIndexed(nil)
if err != nil {
return nil, fmt.Errorf("error writing state: %v", err)
}
diff --git a/tests/state_test_util.go b/tests/state_test_util.go
index 352fe3570b..1e71fa93a5 100644
--- a/tests/state_test_util.go
+++ b/tests/state_test_util.go
@@ -201,7 +201,7 @@ func runStateTest(test VmTest) error {
}
}
- root, _ := statedb.Commit()
+ root, _ := statedb.CommitIndexed(nil)
if common.HexToHash(test.PostStateRoot) != root {
return fmt.Errorf("Post state root error. Expected %s, got %x", test.PostStateRoot, root)
}
@@ -245,7 +245,7 @@ func RunState(statedb *state.StateDB, env, tx map[string]string) ([]byte, vm.Log
if core.IsNonceErr(err) || core.IsInvalidTxErr(err) || core.IsGasLimitErr(err) {
statedb.Set(snapshot)
}
- statedb.Commit()
+ statedb.CommitIndexed(nil)
return ret, vmenv.state.Logs(), vmenv.Gas, err
}
diff --git a/trie/index.go b/trie/index.go
new file mode 100644
index 0000000000..705e605ea5
--- /dev/null
+++ b/trie/index.go
@@ -0,0 +1,68 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+import "bytes"
+
+var (
+ // ParentReferenceIndexPrefix is the database key prefix storing parent references index entries
+ ParentReferenceIndexPrefix = []byte("ref-")
+)
+
+// ParentReferenceIndexKey constructs a child->parent database index key.
+func ParentReferenceIndexKey(parent []byte, child []byte) []byte {
+ return append(append(ParentReferenceIndexPrefix, child...), parent...)
+}
+
+// storeParentReferences expands a trie node to find all its stored children and
+// adds a database reference pointing to the parent to permit reference tracking.
+func storeParentReferences(key []byte, node node, db DatabaseWriter) error {
+ switch node := node.(type) {
+ case fullNode:
+ for _, child := range node {
+ if child != nil {
+ if err := storeParentReferences(key, child, db); err != nil {
+ return err
+ }
+ }
+ }
+ case shortNode:
+ if child := node.Val; child != nil {
+ return storeParentReferences(key, child, db)
+ }
+ case hashNode:
+ return db.Put(ParentReferenceIndexKey(key, node), nil)
+
+ case valueNode:
+ for _, child := range node.refs {
+ if bytes.Compare(child, emptyRoot.Bytes()) == 0 {
+ continue // don't index an empty trie
+ }
+ if err := db.Put(ParentReferenceIndexKey(key, child), nil); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// storeParentReferenceEntry manually inserts a parent->child reference entry into
+// the database index without requiring direct, trie-internal descendancy. This is
+// useful to store references to external entities, like accounts or blocks.
+func storeParentReferenceEntry(parent []byte, child []byte, db DatabaseWriter) error {
+ return db.Put(ParentReferenceIndexKey(parent, child), nil)
+}
diff --git a/trie/index_test.go b/trie/index_test.go
new file mode 100644
index 0000000000..9baa894d3c
--- /dev/null
+++ b/trie/index_test.go
@@ -0,0 +1,58 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+)
+
+// Tests that all index entries are stored in the database after a trie commit.
+func TestTrieIndexDangling(t *testing.T) { testTrieIndex(t, nil) }
+func TestTrieIndexRooted(t *testing.T) { testTrieIndex(t, [][]byte{[]byte{0x00}, []byte{0x00, 0x01}}) }
+
+func testTrieIndex(t *testing.T, referrers [][]byte) {
+ // Create some arbitrary test trie to iterate
+ db, trie, _ := makeTestTrie(referrers)
+
+ // Gather all the indexes that should be present in the database
+ indexes := make(map[string]struct{})
+ for it := NewNodeIterator(trie); it.Next(); {
+ if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+ indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+ }
+ }
+ for _, referrer := range referrers {
+ indexes[string(ParentReferenceIndexKey(referrer, trie.Hash().Bytes()))] = struct{}{}
+ }
+ // Cross check the indexes and the database itself
+ for index, _ := range indexes {
+ if _, err := db.Get([]byte(index)); err != nil {
+ t.Errorf("failed to retrieve reported index %x: %v", index, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if bytes.HasPrefix(key, ParentReferenceIndexPrefix) {
+ if _, ok := indexes[string(key)]; !ok {
+ t.Errorf("index entry not reported %x", key)
+ }
+ }
+ }
+}
diff --git a/trie/iterator.go b/trie/iterator.go
index 5f205e0817..4f7bf924cc 100644
--- a/trie/iterator.go
+++ b/trie/iterator.go
@@ -18,22 +18,27 @@ package trie
import (
"bytes"
+ "fmt"
+ "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
)
+// Iterator is a key-value trie iterator to traverse the data contents.
type Iterator struct {
trie *Trie
- Key []byte
- Value []byte
+ Key []byte // Current data key on which the iterator is positioned on
+ Value []byte // Current data value on which the iterator is positioned on
}
+// NewIterator creates a new key-value iterator.
func NewIterator(trie *Trie) *Iterator {
return &Iterator{trie: trie, Key: nil}
}
+// Next moves the iterator forward with one key-value entry.
func (self *Iterator) Next() bool {
isIterStart := false
if self.Key == nil {
@@ -81,11 +86,11 @@ func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byt
switch bytes.Compare([]byte(k), key) {
case 0:
if isIterStart {
- self.Value = vnode
+ self.Value = vnode.Value
return k
}
case 1:
- self.Value = vnode
+ self.Value = vnode.Value
return k
}
} else {
@@ -120,13 +125,13 @@ func (self *Iterator) key(node interface{}) []byte {
// Leaf node
k := remTerm(node.Key)
if vnode, ok := node.Val.(valueNode); ok {
- self.Value = vnode
+ self.Value = vnode.Value
return k
}
return append(k, self.key(node.Val)...)
case fullNode:
if node[16] != nil {
- self.Value = node[16].(valueNode)
+ self.Value = node[16].(valueNode).Value
return []byte{16}
}
for i := 0; i < 16; i++ {
@@ -142,6 +147,141 @@ func (self *Iterator) key(node interface{}) []byte {
}
return self.key(rn)
}
-
return nil
}
+
+// nodeIteratorState represents the iteration state at one particular node of the
+// trie, which can be resumed at a later invocation.
+type nodeIteratorState struct {
+ hash common.Hash // Hash of the node being iterated (nil if not standalone)
+ node node // Trie node being iterated
+ parent common.Hash // Hash of the first full ancestor node (nil if current is the root)
+ child int // Child to be processed next
+}
+
+// NodeIteratorSubtreeCallback is a callback that is invoked by the trie node
+// iterator whenever it descends into a new subtree, giving the possibility of
+// skipping the branch if it would be non-useful (e.g. already processed before).
+type NodeIteratorSubtreeCallback func(hash, parent common.Hash) bool
+
+// NodeIterator is an iterator to traverse the trie post-order.
+type NodeIterator struct {
+ trie *Trie // Trie being iterated
+ stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state
+
+ // Callback method enabling the user to control subtree descent.
+ //
+ // Note: This hook is invoked pre-order to allow the user to cancel the traversal
+ // of a subtree, but the iterator itself is post-order. This means that the hook
+ // may be invoked multiple times during a single iteration step (when descending)
+ // or never in multiple iteration steps (when ascending).
+ PreOrderHook NodeIteratorSubtreeCallback
+
+ Hash common.Hash // Hash of the current node being iterated (nil if not standalone)
+ Node node // Current node being iterated (internal representation)
+ Parent common.Hash // Hash of the first full ancestor node (nil if current is the root)
+ Leaf bool // Flag whether the current node is a value (data) node
+ LeafBlob []byte // Data blob contained within a leaf (otherwise nil)
+}
+
+// NewNodeIterator creates an post-order trie iterator.
+func NewNodeIterator(trie *Trie) *NodeIterator {
+ if bytes.Compare(trie.Root(), emptyRoot.Bytes()) == 0 {
+ return new(NodeIterator)
+ }
+ return &NodeIterator{trie: trie}
+}
+
+// Next moves the iterator to the next node, returning whether there are any
+// further nodes.
+func (it *NodeIterator) Next() bool {
+ it.step()
+ return it.retrieve()
+}
+
+// step moves the iterator to the next node of the trie.
+func (it *NodeIterator) step() {
+ // Abort if we reached the end of the iteration
+ if it.trie == nil {
+ return
+ }
+ // Initialize the iterator if we've just started, or pop off the old node otherwise
+ if len(it.stack) == 0 {
+ it.stack = append(it.stack, &nodeIteratorState{node: it.trie.root, child: -1})
+ if it.stack[0].node == nil {
+ panic(fmt.Sprintf("root node missing: %x", it.trie.Root()))
+ }
+ } else {
+ it.stack = it.stack[:len(it.stack)-1]
+ if len(it.stack) == 0 {
+ it.trie = nil
+ return
+ }
+ }
+ // Continue iteration to the next child
+ for {
+ parent := it.stack[len(it.stack)-1]
+ ancestor := parent.hash
+ if (ancestor == common.Hash{}) {
+ ancestor = parent.parent
+ }
+ if node, ok := parent.node.(fullNode); ok {
+ // Full node, traverse all children, then the node itself
+ if parent.child >= len(node) {
+ break
+ }
+ for parent.child++; parent.child < len(node); parent.child++ {
+ if current := node[parent.child]; current != nil {
+ it.stack = append(it.stack, &nodeIteratorState{node: current, parent: ancestor, child: -1})
+ break
+ }
+ }
+ } else if node, ok := parent.node.(shortNode); ok {
+ // Short node, traverse the pointer singleton child, then the node itself
+ if parent.child >= 0 {
+ break
+ }
+ parent.child++
+ it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1})
+ } else if hashNode, ok := parent.node.(hashNode); ok {
+ // Hash node, resolve the hash child from the database, then the node itself
+ if parent.child >= 0 {
+ break
+ }
+ parent.child++
+
+ if hash := common.BytesToHash(hashNode); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) {
+ node, err := it.trie.resolveHash(hashNode, nil, nil)
+ if err != nil {
+ panic(err)
+ }
+ it.stack = append(it.stack, &nodeIteratorState{hash: hash, node: node, parent: ancestor, child: -1})
+ }
+ } else {
+ break
+ }
+ }
+}
+
+// retrieve pulls and caches the current trie node the iterator is traversing.
+// In case of a value node, the additional leaf blob is also populated with the
+// data contents for external interpretation.
+//
+// The method returns whether there are any more data left for inspection.
+func (it *NodeIterator) retrieve() bool {
+ // Clear out any previously set values
+ it.Hash, it.Node, it.Parent, it.Leaf, it.LeafBlob = common.Hash{}, nil, common.Hash{}, false, nil
+
+ // If the iteration's done, return no available data
+ if it.trie == nil {
+ return false
+ }
+ // Otherwise retrieve the current node and resolve leaf accessors
+ state := it.stack[len(it.stack)-1]
+
+ it.Hash, it.Node, it.Parent = state.hash, state.node, state.parent
+ if node, ok := it.Node.(valueNode); ok {
+ it.Leaf, it.LeafBlob = true, []byte(node.Value)
+ }
+ return true
+}
diff --git a/trie/iterator_test.go b/trie/iterator_test.go
index fdc60b4122..dadee3fdc3 100644
--- a/trie/iterator_test.go
+++ b/trie/iterator_test.go
@@ -16,7 +16,12 @@
package trie
-import "testing"
+import (
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+)
func TestIterator(t *testing.T) {
trie := newEmpty()
@@ -32,9 +37,9 @@ func TestIterator(t *testing.T) {
v := make(map[string]bool)
for _, val := range vals {
v[val.k] = false
- trie.Update([]byte(val.k), []byte(val.v))
+ trie.UpdateIndexed([]byte(val.k), []byte(val.v), nil)
}
- trie.Commit()
+ trie.CommitIndexed(nil)
it := NewIterator(trie)
for it.Next() {
@@ -47,3 +52,82 @@ func TestIterator(t *testing.T) {
}
}
}
+
+// Tests that the node iterator indeed walks over the entire database contents.
+func TestNodeIteratorCoverage(t *testing.T) {
+ // Create some arbitrary test trie to iterate
+ db, trie, _ := makeTestTrie(nil)
+
+ // Gather all the node hashes found by the iterator
+ hashes := make(map[common.Hash]struct{})
+ for it := NewNodeIterator(trie); it.Next(); {
+ if it.Hash != (common.Hash{}) {
+ hashes[it.Hash] = struct{}{}
+ }
+ }
+ // Cross check the hashes and the database itself
+ for hash, _ := range hashes {
+ if _, err := db.Get(hash.Bytes()); err != nil {
+ t.Errorf("failed to retrieve reported node %x: %v", hash, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if len(key) == common.HashLength {
+ if _, ok := hashes[common.BytesToHash(key)]; !ok {
+ t.Errorf("state entry not reported %x", key)
+ }
+ }
+ }
+}
+
+// Tests that the node iterator hook is invoked for all the nodes of the trie,
+// also testing that the iterator indeed avoids visiting aborted branches.
+func TestNodeIteratorHookCoverageFull(t *testing.T) { testNodeIteratorHookCoverage(t, false) }
+func TestNodeIteratorHookCoverageDedup(t *testing.T) { testNodeIteratorHookCoverage(t, true) }
+
+func testNodeIteratorHookCoverage(t *testing.T, dedup bool) {
+ // Create some arbitrary test trie to iterate
+ db, trie, _ := makeTestTrie(nil)
+
+ // Gather all the node hashes found by the iterator
+ hashes := make(map[common.Hash]int)
+
+ it := NewNodeIterator(trie)
+ it.PreOrderHook = func(hash, parent common.Hash) bool {
+ if !dedup {
+ return true
+ }
+ return hashes[hash] == 0
+ }
+ for it.Next() {
+ if it.Hash != (common.Hash{}) {
+ hashes[it.Hash]++
+ }
+ }
+ // Cross check the hashes and the database itself
+ for hash, _ := range hashes {
+ if _, err := db.Get(hash.Bytes()); err != nil {
+ t.Errorf("failed to retrieve reported node %x: %v", hash, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if len(key) == common.HashLength {
+ if _, ok := hashes[common.BytesToHash(key)]; !ok {
+ t.Errorf("state entry not reported %x", key)
+ }
+ }
+ }
+ // Check whether duplicates were avoided or not
+ duplicates := 0
+ for hash, count := range hashes {
+ if count > 1 {
+ duplicates++
+ if dedup {
+ t.Errorf("duplicate (%d) iteration: %x", count, hash)
+ }
+ }
+ }
+ if !dedup && duplicates == 0 {
+ t.Errorf("iterator didn't traverse common subtrees")
+ }
+}
diff --git a/trie/node.go b/trie/node.go
index 0bfa21dc43..01452a8531 100644
--- a/trie/node.go
+++ b/trie/node.go
@@ -38,9 +38,22 @@ type (
Val node
}
hashNode []byte
- valueNode []byte
+ valueNode struct {
+ Value []byte
+ refs [][]byte
+ }
)
+func (n valueNode) EncodeRLP(w io.Writer) error {
+ return rlp.Encode(w, n.Value)
+}
+
+func (n valueNode) DecodeRLP(s *rlp.Stream) (err error) {
+ fmt.Println("Decoding value node")
+ n.Value, err = s.Bytes()
+ return
+}
+
// Pretty printing.
func (n fullNode) String() string { return n.fstring("") }
func (n shortNode) String() string { return n.fstring("") }
@@ -65,7 +78,7 @@ func (n hashNode) fstring(ind string) string {
return fmt.Sprintf("<%x> ", []byte(n))
}
func (n valueNode) fstring(ind string) string {
- return fmt.Sprintf("%x ", []byte(n))
+ return fmt.Sprintf("%x ", []byte(n.Value))
}
func mustDecodeNode(dbkey, buf []byte) node {
@@ -109,7 +122,7 @@ func decodeShort(buf []byte) (node, error) {
if err != nil {
return nil, fmt.Errorf("invalid value node: %v", err)
}
- return shortNode{key, valueNode(val)}, nil
+ return shortNode{key, valueNode{Value: val}}, nil
}
r, _, err := decodeRef(rest)
if err != nil {
@@ -132,7 +145,7 @@ func decodeFull(buf []byte) (fullNode, error) {
return n, err
}
if len(val) > 0 {
- n[16] = valueNode(val)
+ n[16] = valueNode{Value: val}
}
return n, nil
}
diff --git a/trie/proof.go b/trie/proof.go
index 0f92649424..0cfdc5bc72 100644
--- a/trie/proof.go
+++ b/trie/proof.go
@@ -62,7 +62,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue {
// Don't bother checking for errors here since hasher panics
// if encoding doesn't work and we're not writing to any database.
n, _ = t.hasher.replaceChildren(n, nil)
- hn, _ := t.hasher.store(n, nil, false)
+ hn, _ := t.hasher.store(n, nil, false, nil)
if _, ok := hn.(hashNode); ok || i == 0 {
// If the node's database encoding is a hash (or is the
// root node), it becomes a proof element.
@@ -107,7 +107,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value
if i != len(proof)-1 {
return nil, errors.New("additional nodes at end of proof")
}
- return cld, nil
+ return cld.Value, nil
}
}
return nil, errors.New("unexpected end of proof")
diff --git a/trie/proof_test.go b/trie/proof_test.go
index 6b5bef05c4..0530dff357 100644
--- a/trie/proof_test.go
+++ b/trie/proof_test.go
@@ -119,14 +119,14 @@ func randomTrie(n int) (*Trie, map[string]*kv) {
for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false}
- trie.Update(value.k, value.v)
- trie.Update(value2.k, value2.v)
+ trie.UpdateIndexed(value.k, value.v, nil)
+ trie.UpdateIndexed(value2.k, value2.v, nil)
vals[string(value.k)] = value
vals[string(value2.k)] = value2
}
for i := 0; i < n; i++ {
value := &kv{randBytes(32), randBytes(20), false}
- trie.Update(value.k, value.v)
+ trie.UpdateIndexed(value.k, value.v, nil)
vals[string(value.k)] = value
}
return trie, vals
diff --git a/trie/secure_trie.go b/trie/secure_trie.go
index caeef3c3a3..aabffb40f7 100644
--- a/trie/secure_trie.go
+++ b/trie/secure_trie.go
@@ -79,29 +79,27 @@ func (t *SecureTrie) TryGet(key []byte) ([]byte, error) {
return t.Trie.TryGet(t.hashKey(key))
}
-// Update associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// UpdateIndexed is an extended version of Update, where state trie index entries
+// are also generated for all entities referencing the current node.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
-func (t *SecureTrie) Update(key, value []byte) {
- if err := t.TryUpdate(key, value); err != nil && glog.V(logger.Error) {
+func (t *SecureTrie) UpdateIndexed(key, value []byte, refs [][]byte) {
+ if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled trie error: %v", err)
}
}
-// TryUpdate associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// TryUpdateIndexed is an extended version of Update, where state trie index
+// entries are also generated for all entities referencing the current node.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
//
// If a node was not found in the database, a MissingNodeError is returned.
-func (t *SecureTrie) TryUpdate(key, value []byte) error {
+func (t *SecureTrie) TryUpdateIndexed(key, value []byte, refs [][]byte) error {
hk := t.hashKey(key)
- err := t.Trie.TryUpdate(hk, value)
+ err := t.Trie.TryUpdateIndexed(hk, value, refs)
if err != nil {
return err
}
diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go
index 13c6cd02e9..59ee267efc 100644
--- a/trie/secure_trie_test.go
+++ b/trie/secure_trie_test.go
@@ -45,7 +45,7 @@ func TestSecureDelete(t *testing.T) {
}
for _, val := range vals {
if val.v != "" {
- trie.Update([]byte(val.k), []byte(val.v))
+ trie.UpdateIndexed([]byte(val.k), []byte(val.v), nil)
} else {
trie.Delete([]byte(val.k))
}
@@ -59,7 +59,7 @@ func TestSecureDelete(t *testing.T) {
func TestSecureGetKey(t *testing.T) {
trie := newEmptySecure()
- trie.Update([]byte("foo"), []byte("bar"))
+ trie.UpdateIndexed([]byte("foo"), []byte("bar"), nil)
key := []byte("foo")
value := []byte("bar")
diff --git a/trie/sync.go b/trie/sync.go
index d55399d06b..0daf0ecb92 100644
--- a/trie/sync.go
+++ b/trie/sync.go
@@ -34,9 +34,19 @@ type request struct {
depth int // Depth level within the trie the node is located to prioritise DFS
deps int // Number of dependencies before allowed to commit this node
+ referrers map[common.Hash]struct{} // External parents of this node to index in addition to internal ones
+
callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
}
+// referrer adds a new referring parent to a sync node request.
+func (r *request) referrer(hash common.Hash) {
+ if r.referrers == nil {
+ r.referrers = make(map[common.Hash]struct{})
+ }
+ r.referrers[hash] = struct{}{}
+}
+
// SyncResult is a simple list to return missing nodes along with their request
// hashes.
type SyncResult struct {
@@ -59,25 +69,25 @@ type TrieSync struct {
}
// NewTrieSync creates a new trie data download scheduler.
-func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
+func NewTrieSync(root common.Hash, database ethdb.Database, parent common.Hash, callback TrieSyncLeafCallback) *TrieSync {
ts := &TrieSync{
database: database,
requests: make(map[common.Hash]*request),
queue: prque.New(),
}
- ts.AddSubTrie(root, 0, common.Hash{}, callback)
+ ts.AddSubTrie(root, 0, parent, callback)
return ts
}
// AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
-func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
+func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) error {
// Short circuit if the trie is empty or already known
if root == emptyRoot {
- return
+ return nil
}
blob, _ := s.database.Get(root.Bytes())
if local, err := decodeNode(blob); local != nil && err == nil {
- return
+ return storeParentReferenceEntry(parent.Bytes(), root.Bytes(), s.database)
}
// Assemble the new sub-trie sync request
node := node(hashNode(root.Bytes()))
@@ -89,27 +99,31 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c
}
// If this sub-trie has a designated parent, link them together
if parent != (common.Hash{}) {
- ancestor := s.requests[parent]
- if ancestor == nil {
- panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
+ if depth > 0 {
+ ancestor := s.requests[parent]
+ if ancestor == nil {
+ panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
+ }
+ ancestor.deps++
+ req.parents = append(req.parents, ancestor)
}
- ancestor.deps++
- req.parents = append(req.parents, ancestor)
+ req.referrer(parent)
}
s.schedule(req)
+ return nil
}
// AddRawEntry schedules the direct retrieval of a state entry that should not be
// interpreted as a trie node, but rather accepted and stored into the database
// as is. This method's goal is to support misc state metadata retrievals (e.g.
// contract code).
-func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
+func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) error {
// Short circuit if the entry is empty or already known
if hash == emptyState {
- return
+ return nil
}
if blob, _ := s.database.Get(hash.Bytes()); blob != nil {
- return
+ return storeParentReferenceEntry(parent.Bytes(), hash.Bytes(), s.database)
}
// Assemble the new sub-trie sync request
req := &request{
@@ -118,14 +132,18 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash)
}
// If this sub-trie has a designated parent, link them together
if parent != (common.Hash{}) {
- ancestor := s.requests[parent]
- if ancestor == nil {
- panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
+ if depth > 0 {
+ ancestor := s.requests[parent]
+ if ancestor == nil {
+ panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
+ }
+ ancestor.deps++
+ req.parents = append(req.parents, ancestor)
}
- ancestor.deps++
- req.parents = append(req.parents, ancestor)
+ req.referrer(parent)
}
s.schedule(req)
+ return nil
}
// Missing retrieves the known missing nodes from the trie for retrieval.
@@ -188,6 +206,13 @@ func (s *TrieSync) schedule(req *request) {
// If we're already requesting this node, add a new reference and stop
if old, ok := s.requests[req.hash]; ok {
old.parents = append(old.parents, req.parents...)
+
+ for hash, _ := range req.referrers {
+ old.referrer(hash)
+ }
+ for _, parent := range req.parents {
+ old.referrer(parent.hash)
+ }
return
}
// Schedule the request for future retrieval
@@ -229,7 +254,7 @@ func (s *TrieSync) children(req *request) ([]*request, error) {
// Notify any external watcher of a new key/value node
if req.callback != nil {
if node, ok := (*child.node).(valueNode); ok {
- if err := req.callback(node, req.hash); err != nil {
+ if err := req.callback(node.Value, req.hash); err != nil {
return nil, err
}
}
@@ -266,7 +291,17 @@ func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) {
err = batch.Write()
}()
}
- // Write the node content to disk
+ // Write the node content and indices to disk
+ if req.object != nil {
+ if err := storeParentReferences(req.hash[:], *req.object, batch); err != nil {
+ return err
+ }
+ }
+ for referrer, _ := range req.referrers {
+ if err := storeParentReferenceEntry(referrer[:], req.hash[:], batch); err != nil {
+ return err
+ }
+ }
if err := batch.Put(req.hash[:], req.data); err != nil {
return err
}
diff --git a/trie/sync_test.go b/trie/sync_test.go
index 9c036a3a92..37ccc416a7 100644
--- a/trie/sync_test.go
+++ b/trie/sync_test.go
@@ -18,6 +18,7 @@ package trie
import (
"bytes"
+ "fmt"
"testing"
"github.com/ethereum/go-ethereum/common"
@@ -25,7 +26,7 @@ import (
)
// makeTestTrie create a sample test trie to test node-wise reconstruction.
-func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
+func makeTestTrie(referrers [][]byte) (ethdb.Database, *Trie, map[string][]byte) {
// Create an empty trie
db, _ := ethdb.NewMemDatabase()
trie, _ := New(common.Hash{}, db)
@@ -33,15 +34,26 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
// Fill it with some arbitrary data
content := make(map[string][]byte)
for i := byte(0); i < 255; i++ {
+ // Map the same data under multiple keys
key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i}
content[string(key)] = val
- trie.Update(key, val)
+ trie.UpdateIndexed(key, val, nil)
key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i}
content[string(key)] = val
- trie.Update(key, val)
+ trie.UpdateIndexed(key, val, nil)
+
+ // Add some other data to inflate th trie
+ for j := byte(3); j < 13; j++ {
+ key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i}
+ content[string(key)] = val
+ trie.UpdateIndexed(key, val, nil)
+ }
}
- trie.Commit()
+ trie.CommitIndexed(referrers)
+
+ // Remove any potentially cached data from the test trie creation
+ globalCache.Clear()
// Return the generated trie
return db, trie, content
@@ -49,11 +61,21 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
// checkTrieContents cross references a reconstructed trie with an expected data
// content map.
-func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) {
+func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte, parent common.Hash) {
+ // Remove any potentially cached data from the trie synchronisation
+ globalCache.Clear()
+
+ // Check root availability and trie contents
trie, err := New(common.BytesToHash(root), db)
if err != nil {
t.Fatalf("failed to create trie at %x: %v", root, err)
}
+ if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil {
+ t.Fatalf("inconsistent trie at %x: %v", root, err)
+ }
+ if err := checkTrieIndex(db, common.BytesToHash(root), parent); err != nil {
+ t.Fatalf("index error at %x: %v", root, err)
+ }
for key, val := range content {
if have := trie.Get([]byte(key)); bytes.Compare(have, val) != 0 {
t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val)
@@ -61,14 +83,76 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin
}
}
+// checkTrieConsistency checks that all nodes in a trie and indeed present.
+func checkTrieConsistency(db Database, root common.Hash) (failure error) {
+ // Capture any panics by the iterator
+ defer func() {
+ if r := recover(); r != nil {
+ failure = fmt.Errorf("%v", r)
+ }
+ }()
+ // Remove any potentially cached data from the test trie creation or previous checks
+ globalCache.Clear()
+
+ // Create and iterate a trie rooted in a subnode
+ trie, err := New(root, db)
+ if err != nil {
+ return
+ }
+ for it := NewNodeIterator(trie); it.Next(); {
+ }
+ return nil
+}
+
+// checkTrieIndex iterates over the entire state trie and checks that all required
+// database indexes have been generated by the synchronizer.
+func checkTrieIndex(db Database, root common.Hash, parent common.Hash) error {
+ // Remove any potentially cached data from the test trie creation or previous checks
+ globalCache.Clear()
+
+ trie, err := New(root, db)
+ if err != nil {
+ return err
+ }
+ // Gather all the indexes that should be present in the database
+ indexes := make(map[string]struct{})
+ for it := NewNodeIterator(trie); it.Next(); {
+ if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+ indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+ }
+ }
+ if parent != (common.Hash{}) {
+ indexes[string(ParentReferenceIndexKey(parent.Bytes(), trie.Hash().Bytes()))] = struct{}{}
+ }
+ // Cross check the indexes and the database itself
+ for index, _ := range indexes {
+ if _, err := db.Get([]byte(index)); err != nil {
+ return fmt.Errorf("failed to retrieve reported index %x: %v", index, err)
+ }
+ }
+ for _, key := range db.(*ethdb.MemDatabase).Keys() {
+ if bytes.HasPrefix(key, ParentReferenceIndexPrefix) {
+ if _, ok := indexes[string(key)]; !ok {
+ return fmt.Errorf("index entry not reported %x", key)
+ }
+ }
+ }
+ return nil
+}
+
// Tests that an empty trie is not scheduled for syncing.
-func TestEmptyTrieSync(t *testing.T) {
+func TestEmptyTrieSyncDangling(t *testing.T) { testEmptyTrieSync(t, common.Hash{}) }
+func TestEmptyTrieSyncRooted(t *testing.T) {
+ testEmptyTrieSync(t, common.BytesToHash([]byte{0x01}))
+}
+
+func testEmptyTrieSync(t *testing.T, origin common.Hash) {
emptyA, _ := New(common.Hash{}, nil)
emptyB, _ := New(emptyRoot, nil)
for i, trie := range []*Trie{emptyA, emptyB} {
db, _ := ethdb.NewMemDatabase()
- if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 {
+ if req := NewTrieSync(common.BytesToHash(trie.Root()), db, origin, nil).Missing(1); len(req) != 0 {
t.Errorf("test %d: content requested for empty trie: %v", i, req)
}
}
@@ -76,16 +160,26 @@ func TestEmptyTrieSync(t *testing.T) {
// Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go.
-func TestIterativeTrieSyncIndividual(t *testing.T) { testIterativeTrieSync(t, 1) }
-func TestIterativeTrieSyncBatched(t *testing.T) { testIterativeTrieSync(t, 100) }
+func TestIterativeTrieSyncIndividualDangling(t *testing.T) {
+ testIterativeTrieSync(t, 1, common.Hash{})
+}
+func TestIterativeTrieSyncIndividualRooted(t *testing.T) {
+ testIterativeTrieSync(t, 1, common.BytesToHash([]byte{0x02}))
+}
+func TestIterativeTrieSyncBatchedDangling(t *testing.T) {
+ testIterativeTrieSync(t, 100, common.Hash{})
+}
+func TestIterativeTrieSyncBatchedRooted(t *testing.T) {
+ testIterativeTrieSync(t, 100, common.BytesToHash([]byte{0x03}))
+}
-func testIterativeTrieSync(t *testing.T, batch int) {
+func testIterativeTrieSync(t *testing.T, batch int, origin common.Hash) {
// Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
+ srcDb, srcTrie, srcData := makeTestTrie(nil)
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil)
queue := append([]common.Hash{}, sched.Missing(batch)...)
for len(queue) > 0 {
@@ -102,19 +196,26 @@ func testIterativeTrieSync(t *testing.T, batch int) {
}
queue = append(queue[:0], sched.Missing(batch)...)
}
- // Cross check that the two tries re in sync
- checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+ // Cross check that the two tries are in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned, and the others sent only later.
-func TestIterativeDelayedTrieSync(t *testing.T) {
+func TestIterativeDelayedTrieSyncDangling(t *testing.T) {
+ testIterativeDelayedTrieSync(t, common.Hash{})
+}
+func TestIterativeDelayedTrieSyncRooted(t *testing.T) {
+ testIterativeDelayedTrieSync(t, common.BytesToHash([]byte{0x04}))
+}
+
+func testIterativeDelayedTrieSync(t *testing.T, origin common.Hash) {
// Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
+ srcDb, srcTrie, srcData := makeTestTrie(nil)
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil)
queue := append([]common.Hash{}, sched.Missing(10000)...)
for len(queue) > 0 {
@@ -132,23 +233,33 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
}
queue = append(queue[len(results):], sched.Missing(10000)...)
}
- // Cross check that the two tries re in sync
- checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+ // Cross check that the two tries are in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin)
}
// Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go, however in a
// random order.
-func TestIterativeRandomTrieSyncIndividual(t *testing.T) { testIterativeRandomTrieSync(t, 1) }
-func TestIterativeRandomTrieSyncBatched(t *testing.T) { testIterativeRandomTrieSync(t, 100) }
+func TestIterativeRandomTrieSyncIndividualDangling(t *testing.T) {
+ testIterativeRandomTrieSync(t, 1, common.Hash{})
+}
+func TestIterativeRandomTrieSyncIndividualRooted(t *testing.T) {
+ testIterativeRandomTrieSync(t, 1, common.BytesToHash([]byte{0x05}))
+}
+func TestIterativeRandomTrieSyncBatchedDangling(t *testing.T) {
+ testIterativeRandomTrieSync(t, 100, common.Hash{})
+}
+func TestIterativeRandomTrieSyncBatchedlRooted(t *testing.T) {
+ testIterativeRandomTrieSync(t, 100, common.BytesToHash([]byte{0x06}))
+}
-func testIterativeRandomTrieSync(t *testing.T, batch int) {
+func testIterativeRandomTrieSync(t *testing.T, batch int, origin common.Hash) {
// Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
+ srcDb, srcTrie, srcData := makeTestTrie(nil)
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil)
queue := make(map[common.Hash]struct{})
for _, hash := range sched.Missing(batch) {
@@ -173,19 +284,26 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
queue[hash] = struct{}{}
}
}
- // Cross check that the two tries re in sync
- checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+ // Cross check that the two tries are in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned (Even those randomly), others sent only later.
-func TestIterativeRandomDelayedTrieSync(t *testing.T) {
+func TestIterativeRandomDelayedTrieSyncDangling(t *testing.T) {
+ testIterativeRandomDelayedTrieSync(t, common.Hash{})
+}
+func TestIterativeRandomDelayedTrieSyncRooted(t *testing.T) {
+ testIterativeRandomDelayedTrieSync(t, common.BytesToHash([]byte{0x07}))
+}
+
+func testIterativeRandomDelayedTrieSync(t *testing.T, origin common.Hash) {
// Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
+ srcDb, srcTrie, srcData := makeTestTrie(nil)
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil)
queue := make(map[common.Hash]struct{})
for _, hash := range sched.Missing(10000) {
@@ -216,19 +334,26 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
queue[hash] = struct{}{}
}
}
- // Cross check that the two tries re in sync
- checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+ // Cross check that the two tries are in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin)
}
// Tests that a trie sync will not request nodes multiple times, even if they
// have such references.
-func TestDuplicateAvoidanceTrieSync(t *testing.T) {
+func TestDuplicateAvoidanceTrieSyncDangling(t *testing.T) {
+ testDuplicateAvoidanceTrieSync(t, common.Hash{})
+}
+func TestDuplicateAvoidanceTrieSyncRooted(t *testing.T) {
+ testDuplicateAvoidanceTrieSync(t, common.BytesToHash([]byte{0x08}))
+}
+
+func testDuplicateAvoidanceTrieSync(t *testing.T, origin common.Hash) {
// Create a random trie to copy
- srcDb, srcTrie, srcData := makeTestTrie()
+ srcDb, srcTrie, srcData := makeTestTrie(nil)
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
- sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil)
queue := append([]common.Hash{}, sched.Missing(0)...)
requested := make(map[common.Hash]struct{})
@@ -252,6 +377,57 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
}
queue = append(queue[:0], sched.Missing(0)...)
}
- // Cross check that the two tries re in sync
- checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+ // Cross check that the two tries are in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin)
+}
+
+// Tests that at any point in time during a sync, only complete sub-tries are in
+// the database.
+func TestIncompleteTrieSync(t *testing.T) {
+ // Create a random trie to copy
+ srcDb, srcTrie, _ := makeTestTrie(nil)
+
+ // Create a destination trie and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, common.Hash{}, nil)
+
+ added := []common.Hash{}
+ queue := append([]common.Hash{}, sched.Missing(1)...)
+ for len(queue) > 0 {
+ // Fetch a batch of trie nodes
+ results := make([]SyncResult, len(queue))
+ for i, hash := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results[i] = SyncResult{hash, data}
+ }
+ // Process each of the trie nodes
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ for _, result := range results {
+ added = append(added, result.Hash)
+ }
+ // Check that all known sub-tries in the synced trie is complete
+ for _, root := range added {
+ if err := checkTrieConsistency(dstDb, root); err != nil {
+ t.Fatalf("trie inconsistent: %v", err)
+ }
+ }
+ // Fetch the next batch to retrieve
+ queue = append(queue[:0], sched.Missing(1)...)
+ }
+ // Sanity check that removing any node from the database is detected
+ for _, node := range added[1:] {
+ key := node.Bytes()
+ value, _ := dstDb.Get(key)
+
+ dstDb.Delete(key)
+ if err := checkTrieConsistency(dstDb, added[0]); err == nil {
+ t.Fatalf("trie inconsistency not caught, missing: %x", key)
+ }
+ dstDb.Put(key, value)
+ }
}
diff --git a/trie/trie.go b/trie/trie.go
index 9dfde45294..97cf624075 100644
--- a/trie/trie.go
+++ b/trie/trie.go
@@ -143,33 +143,39 @@ func (t *Trie) TryGet(key []byte) ([]byte, error) {
panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
}
}
- return tn.(valueNode), nil
+ return tn.(valueNode).Value, nil
}
-// Update associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// UpdateIndexed associates key with value in the trie. Subsequent calls to Get
+// will return value. If value has length zero, any existing value is deleted
+// from the trie and calls to Get will return nil. In addition, state trie index
+// entries are also generated for all entities referencing the current node.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
-func (t *Trie) Update(key, value []byte) {
- if err := t.TryUpdate(key, value); err != nil && glog.V(logger.Error) {
+func (t *Trie) UpdateIndexed(key, value []byte, references [][]byte) {
+ if err := t.TryUpdateIndexed(key, value, references); err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled trie error: %v", err)
}
}
-// TryUpdate associates key with value in the trie. Subsequent calls to
-// Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// TryUpdateIndexed associates key with value in the trie. Subsequent calls to
+// Get will return value. If value has length zero, any existing value is deleted
+// from the trie and calls to Get will return nil. In addition, state trie index
+// entries are also generated for all entities referencing the current node.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
//
// If a node was not found in the database, a MissingNodeError is returned.
-func (t *Trie) TryUpdate(key, value []byte) error {
+func (t *Trie) TryUpdateIndexed(key, value []byte, references [][]byte) error {
+ return t.tryUpdateIndexed(key, value, references)
+}
+
+func (t *Trie) tryUpdateIndexed(key, value []byte, refs [][]byte) error {
k := compactHexDecode(key)
if len(value) != 0 {
- n, err := t.insert(t.root, nil, k, valueNode(value))
+ n, err := t.insert(t.root, nil, k, valueNode{Value: value, refs: refs})
if err != nil {
return err
}
@@ -413,31 +419,32 @@ func (t *Trie) Root() []byte { return t.Hash().Bytes() }
// Hash returns the root hash of the trie. It does not write to the
// database and can be used even if the trie doesn't have one.
func (t *Trie) Hash() common.Hash {
- root, _ := t.hashRoot(nil)
+ root, _ := t.hashRoot(nil, nil)
return common.BytesToHash(root.(hashNode))
}
-// Commit writes all nodes to the trie's database.
-// Nodes are stored with their sha3 hash as the key.
+// CommitIndexed writes all nodes to the trie's database, and if any references
+// were specified, adds those to the trie index as well. Nodes are stored with
+// their sha3 hash as the key.
//
-// Committing flushes nodes from memory.
-// Subsequent Get calls will load nodes from the database.
-func (t *Trie) Commit() (root common.Hash, err error) {
+// Committing flushes nodes from memory. Subsequent Get calls will load nodes
+// from the database.
+func (t *Trie) CommitIndexed(referrers [][]byte) (root common.Hash, err error) {
if t.db == nil {
panic("Commit called on trie with nil database")
}
- return t.CommitTo(t.db)
+ return t.CommitToIndexed(t.db, referrers)
}
-// CommitTo writes all nodes to the given database.
-// Nodes are stored with their sha3 hash as the key.
+// CommitToIndexed writes all nodes to the given database, and if any references
+// were specified, adds those to the trie index as well. Nodes are stored with
+// their sha3 hash as the key.
//
-// Committing flushes nodes from memory. Subsequent Get calls will
-// load nodes from the trie's database. Calling code must ensure that
-// the changes made to db are written back to the trie's attached
-// database before using the trie.
-func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
- n, err := t.hashRoot(db)
+// Committing flushes nodes from memory. Subsequent Get calls will load nodes
+// from the trie's database. Calling code must ensure that the changes made to
+// db are written back to the trie's attached database before using the trie.
+func (t *Trie) CommitToIndexed(db DatabaseWriter, referrers [][]byte) (root common.Hash, err error) {
+ n, err := t.hashRoot(db, referrers)
if err != nil {
return (common.Hash{}), err
}
@@ -445,14 +452,14 @@ func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
return common.BytesToHash(n.(hashNode)), nil
}
-func (t *Trie) hashRoot(db DatabaseWriter) (node, error) {
+func (t *Trie) hashRoot(db DatabaseWriter, referrers [][]byte) (node, error) {
if t.root == nil {
return hashNode(emptyRoot.Bytes()), nil
}
if t.hasher == nil {
t.hasher = newHasher()
}
- return t.hasher.hash(t.root, db, true)
+ return t.hasher.hash(t.root, db, true, referrers)
}
type hasher struct {
@@ -464,12 +471,12 @@ func newHasher() *hasher {
return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()}
}
-func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, error) {
+func (h *hasher) hash(n node, db DatabaseWriter, force bool, referrers [][]byte) (node, error) {
hashed, err := h.replaceChildren(n, db)
if err != nil {
return hashNode{}, err
}
- if n, err = h.store(hashed, db, force); err != nil {
+ if n, err = h.store(hashed, db, force, referrers); err != nil {
return hashNode{}, err
}
return n, nil
@@ -483,28 +490,28 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) {
case shortNode:
n.Key = compactEncode(n.Key)
if _, ok := n.Val.(valueNode); !ok {
- if n.Val, err = h.hash(n.Val, db, false); err != nil {
+ if n.Val, err = h.hash(n.Val, db, false, nil); err != nil {
return n, err
}
}
if n.Val == nil {
// Ensure that nil children are encoded as empty strings.
- n.Val = valueNode(nil)
+ n.Val = valueNode{Value: nil}
}
return n, nil
case fullNode:
for i := 0; i < 16; i++ {
if n[i] != nil {
- if n[i], err = h.hash(n[i], db, false); err != nil {
+ if n[i], err = h.hash(n[i], db, false, nil); err != nil {
return n, err
}
} else {
// Ensure that nil children are encoded as empty strings.
- n[i] = valueNode(nil)
+ n[i] = valueNode{Value: nil}
}
}
if n[16] == nil {
- n[16] = valueNode(nil)
+ n[16] = valueNode{Value: nil}
}
return n, nil
default:
@@ -512,7 +519,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) {
}
}
-func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
+func (h *hasher) store(n node, db DatabaseWriter, force bool, referrers [][]byte) (node, error) {
// Don't store hashes or empty nodes.
if _, isHash := n.(hashNode); n == nil || isHash {
return n, nil
@@ -530,8 +537,18 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
h.sha.Write(h.tmp.Bytes())
key := hashNode(h.sha.Sum(nil))
if db != nil {
- err := db.Put(key, h.tmp.Bytes())
- return key, err
+ if err := storeParentReferences(key, n, db); err != nil {
+ return key, err
+ }
+ for _, referrer := range referrers {
+ if err := storeParentReferenceEntry(referrer, key, db); err != nil {
+ return key, err
+ }
+ }
+ if err := db.Put(key, h.tmp.Bytes()); err != nil {
+ return key, err
+ }
+ return key, nil
}
return key, nil
}
diff --git a/trie/trie_test.go b/trie/trie_test.go
index 35d043cdf1..244f5d28d5 100644
--- a/trie/trie_test.go
+++ b/trie/trie_test.go
@@ -54,7 +54,7 @@ func TestNull(t *testing.T) {
var trie Trie
key := make([]byte, 32)
value := common.FromHex("0x823140710bf13990e4500136726d8b55")
- trie.Update(key, value)
+ trie.UpdateIndexed(key, value, nil)
value = trie.Get(key)
}
@@ -74,7 +74,7 @@ func TestMissingNode(t *testing.T) {
trie, _ := New(common.Hash{}, db)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
- root, _ := trie.Commit()
+ root, _ := trie.CommitIndexed(nil)
ClearGlobalCache()
@@ -97,7 +97,7 @@ func TestMissingNode(t *testing.T) {
}
trie, _ = New(root, db)
- err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
+ err = trie.TryUpdateIndexed([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"), nil)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
@@ -130,7 +130,7 @@ func TestMissingNode(t *testing.T) {
}
trie, _ = New(root, db)
- err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
+ err = trie.TryUpdateIndexed([]byte("120099"), []byte("zxcv"), nil)
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
@@ -159,7 +159,7 @@ func TestInsert(t *testing.T) {
updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
- root, err := trie.Commit()
+ root, err := trie.CommitIndexed(nil)
if err != nil {
t.Fatalf("commit error: %v", err)
}
@@ -188,7 +188,7 @@ func TestGet(t *testing.T) {
if i == 1 {
return
}
- trie.Commit()
+ trie.CommitIndexed(nil)
}
}
@@ -257,7 +257,7 @@ func TestReplication(t *testing.T) {
for _, val := range vals {
updateString(trie, val.k, val.v)
}
- exp, err := trie.Commit()
+ exp, err := trie.CommitIndexed(nil)
if err != nil {
t.Fatalf("commit error: %v", err)
}
@@ -272,7 +272,7 @@ func TestReplication(t *testing.T) {
t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
}
}
- hash, err := trie2.Commit()
+ hash, err := trie2.CommitIndexed(nil)
if err != nil {
t.Fatalf("commit error: %v", err)
}
@@ -304,7 +304,7 @@ func paranoiaCheck(t1 *Trie) (bool, *Trie) {
t2 := new(Trie)
it := NewIterator(t1)
for it.Next() {
- t2.Update(it.Key, it.Value)
+ t2.UpdateIndexed(it.Key, it.Value, nil)
}
return t2.Hash() == t1.Hash(), t2
}
@@ -327,7 +327,7 @@ func TestParanoia(t *testing.T) {
for _, val := range vals {
updateString(trie, val.k, val.v)
}
- trie.Commit()
+ trie.CommitIndexed(nil)
ok, t2 := paranoiaCheck(trie)
if !ok {
@@ -347,7 +347,7 @@ func TestOutput(t *testing.T) {
fmt.Println("############################## FULL ################################")
fmt.Println(trie.root)
- trie.Commit()
+ trie.CommitIndexed(nil)
fmt.Println("############################## SMALL ################################")
trie2, _ := New(trie.Hash(), trie.db)
getString(trie2, base+"20")
@@ -356,8 +356,8 @@ func TestOutput(t *testing.T) {
func TestLargeValue(t *testing.T) {
trie := newEmpty()
- trie.Update([]byte("key1"), []byte{99, 99, 99, 99})
- trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32))
+ trie.UpdateIndexed([]byte("key1"), []byte{99, 99, 99, 99}, nil)
+ trie.UpdateIndexed([]byte("key2"), bytes.Repeat([]byte{1}, 32), nil)
trie.Hash()
}
@@ -374,8 +374,8 @@ func TestLargeData(t *testing.T) {
for i := byte(0); i < 255; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false}
- trie.Update(value.k, value.v)
- trie.Update(value2.k, value2.v)
+ trie.UpdateIndexed(value.k, value.v, nil)
+ trie.UpdateIndexed(value2.k, value2.v, nil)
vals[string(value.k)] = value
vals[string(value2.k)] = value2
}
@@ -419,11 +419,11 @@ func benchGet(b *testing.B, commit bool) {
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
binary.LittleEndian.PutUint64(k, uint64(i))
- trie.Update(k, k)
+ trie.UpdateIndexed(k, k, nil)
}
binary.LittleEndian.PutUint64(k, benchElemCount/2)
if commit {
- trie.Commit()
+ trie.CommitIndexed(nil)
}
b.ResetTimer()
@@ -437,7 +437,7 @@ func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
k := make([]byte, 32)
for i := 0; i < b.N; i++ {
e.PutUint64(k, uint64(i))
- trie.Update(k, k)
+ trie.UpdateIndexed(k, k, nil)
}
return trie
}
@@ -447,7 +447,7 @@ func benchHash(b *testing.B, e binary.ByteOrder) {
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
e.PutUint64(k, uint64(i))
- trie.Update(k, k)
+ trie.UpdateIndexed(k, k, nil)
}
b.ResetTimer()
@@ -473,7 +473,7 @@ func getString(trie *Trie, k string) []byte {
}
func updateString(trie *Trie, k, v string) {
- trie.Update([]byte(k), []byte(v))
+ trie.UpdateIndexed([]byte(k), []byte(v), nil)
}
func deleteString(trie *Trie, k string) {