From 1181d93030479b6a05939b65726b5cc94ef4bc50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 28 Dec 2015 15:20:37 +0200 Subject: [PATCH 1/8] core/state, trie: add node iterator, test state/trie sync consistency --- core/state/iterator.go | 133 ++++++++++++++++++++++++++++++++++++++++ core/state/sync_test.go | 107 +++++++++++++++++++++++++++++++- trie/iterator.go | 120 +++++++++++++++++++++++++++++++++++- trie/sync_test.go | 102 ++++++++++++++++++++++++++++-- 4 files changed, 451 insertions(+), 11 deletions(-) create mode 100644 core/state/iterator.go diff --git a/core/state/iterator.go b/core/state/iterator.go new file mode 100644 index 0000000000..a0b71f3eec --- /dev/null +++ b/core/state/iterator.go @@ -0,0 +1,133 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// NodeIterator is an iterator to traverse the entire state trie post-order, +// including all of the contract code and contract state tries. +type NodeIterator struct { + state *StateDB // State being iterated + + stateIt *trie.NodeIterator // Primary iterator for the global state trie + dataIt *trie.NodeIterator // Secondary iterator for the data trie of a contract + code []byte // Source code associated with a contract + + Entry interface{} // Current state entry being iterated (internal representation) +} + +// NewNodeIterator creates an post-order state node iterator. +func NewNodeIterator(state *StateDB) *NodeIterator { + return &NodeIterator{ + state: state, + } +} + +// Next moves the iterator to the next node, returning whether there are any +// further nodes. +func (it *NodeIterator) Next() bool { + it.step() + return it.retrieve() +} + +// step moves the iterator to the next entry of the state trie. +func (it *NodeIterator) step() { + // Abort if we reached the end of the iteration + if it.state == nil { + return + } + // Initialize the iterator if we've just started + if it.stateIt == nil { + it.stateIt = trie.NewNodeIterator(it.state.trie.Trie) + } + // If we had data nodes previously, we surely have at least state nodes + if it.dataIt != nil { + if cont := it.dataIt.Next(); !cont { + it.dataIt = nil + } + return + } + // If we had source code previously, discard that + if it.code != nil { + it.code = nil + return + } + // Step to the next state trie node, terminating if we're out of nodes + if cont := it.stateIt.Next(); !cont { + it.state, it.stateIt = nil, nil + return + } + // If the state trie node is an internal entry, leave as is + if !it.stateIt.Leaf { + return + } + // Otherwise we've reached an account node, initiate data iteration + var account struct { + Nonce uint64 + Balance *big.Int + Root common.Hash + CodeHash []byte + } + err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob), &account) + if err != nil { + panic(err) + } + dataTrie, err := trie.New(account.Root, it.state.db) + if err != nil { + panic(err) + } + it.dataIt = trie.NewNodeIterator(dataTrie) + if !it.dataIt.Next() { + it.dataIt = nil + } + if bytes.Compare(account.CodeHash, emptyCodeHash) != 0 { + it.code, err = it.state.db.Get(account.CodeHash) + if err != nil { + panic(fmt.Sprintf("code %x: %v", account.CodeHash, err)) + } + } +} + +// retrieve pulls and caches the current state entry the iterator is traversing. +// The method returns whether there are any more data left for inspection. +func (it *NodeIterator) retrieve() bool { + // Clear out any previously set values + it.Entry = nil + + // If the iteration's done, return no available data + if it.state == nil { + return false + } + // Otherwise retrieve the current entry + switch { + case it.dataIt != nil: + it.Entry = it.dataIt.Node + case it.code != nil: + it.Entry = it.code + case it.stateIt != nil: + it.Entry = it.stateIt.Node + } + return true +} diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 0dab372ba0..5d6d90d5d2 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -18,10 +18,12 @@ package state import ( "bytes" + "fmt" "math/big" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" ) @@ -42,7 +44,7 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { // Fill it with some arbitrary data accounts := []*testAccount{} - for i := byte(0); i < 255; i++ { + for i := byte(0); i < 96; i++ { obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i})) acc := &testAccount{address: common.BytesToAddress([]byte{i})} @@ -61,6 +63,9 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { } root, _ := state.Commit() + // Remove any potentially cached data from the test state creation + trie.ClearGlobalCache() + // Return the generated state return db, root, accounts } @@ -68,9 +73,18 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { // checkStateAccounts cross references a reconstructed state with an expected // account array. func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { - state, _ := New(root, db) - for i, acc := range accounts { + // Remove any potentially cached data from the state synchronisation + trie.ClearGlobalCache() + // Check root availability and state contents + state, err := New(root, db) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", root, err) + } + if err := checkStateConsistency(db, root); err != nil { + t.Fatalf("inconsistent state trie at %x: %v", root, err) + } + for i, acc := range accounts { if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 { t.Errorf("account %d: balance mismatch: have %v, want %v", i, balance, acc.balance) } @@ -83,6 +97,31 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou } } +// checkStateConsistency checks that all nodes in a state trie and indeed present. +func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error) { + // Capture any panics by the iterator + defer func() { + if r := recover(); r != nil { + failure = fmt.Errorf("%v", r) + } + }() + // Remove any potentially cached data from the test state creation or previous checks + trie.ClearGlobalCache() + + // Create and iterate a state trie rooted in a sub-node + if _, err := db.Get(root.Bytes()); err != nil { + return + } + state, err := New(root, db) + if err != nil { + return + } + it := NewNodeIterator(state) + for it.Next() { + } + return nil +} + // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") @@ -236,3 +275,65 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) } + +// Tests that at any point in time during a sync, only complete sub-tries are in +// the database. +func TestIncompleteStateSync(t *testing.T) { + // Create a random state to copy + srcDb, srcRoot, srcAccounts := makeTestState() + + // Create a destination state and sync with the scheduler + dstDb, _ := ethdb.NewMemDatabase() + sched := NewStateSync(srcRoot, dstDb) + + added := []common.Hash{} + queue := append([]common.Hash{}, sched.Missing(1)...) + for len(queue) > 0 { + // Fetch a batch of state nodes + results := make([]trie.SyncResult, len(queue)) + for i, hash := range queue { + data, err := srcDb.Get(hash.Bytes()) + if err != nil { + t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + } + results[i] = trie.SyncResult{hash, data} + } + // Process each of the state nodes + if index, err := sched.Process(results); err != nil { + t.Fatalf("failed to process result #%d: %v", index, err) + } + for _, result := range results { + added = append(added, result.Hash) + } + // Check that all known sub-tries in the synced state is complete + for _, root := range added { + // Skim through the accounts and make sure the root hash is not a code node + codeHash := false + for _, acc := range srcAccounts { + if bytes.Compare(root.Bytes(), crypto.Sha3(acc.code)) == 0 { + codeHash = true + break + } + } + // If the root is a real trie node, check consistency + if !codeHash { + if err := checkStateConsistency(dstDb, root); err != nil { + t.Fatalf("state inconsistent: %v", err) + } + } + } + // Fetch the next batch to retrieve + queue = append(queue[:0], sched.Missing(1)...) + } + // Sanity check that removing any node from the database is detected + for _, node := range added[1:] { + key := node.Bytes() + value, _ := dstDb.Get(key) + + dstDb.Delete(key) + if err := checkStateConsistency(dstDb, added[0]); err == nil { + t.Fatalf("trie inconsistency not caught, missing: %x", key) + } + dstDb.Put(key, value) + } +} diff --git a/trie/iterator.go b/trie/iterator.go index 5f205e0817..e79de2e4e7 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -18,22 +18,26 @@ package trie import ( "bytes" + "fmt" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" ) +// Iterator is a key-value trie iterator to traverse the data contents. type Iterator struct { trie *Trie - Key []byte - Value []byte + Key []byte // Current data key on which the iterator is positioned on + Value []byte // Current data value on which the iterator is positioned on } +// NewIterator creates a new key-value iterator. func NewIterator(trie *Trie) *Iterator { return &Iterator{trie: trie, Key: nil} } +// Next moves the iterator forward with one key-value entry. func (self *Iterator) Next() bool { isIterStart := false if self.Key == nil { @@ -142,6 +146,116 @@ func (self *Iterator) key(node interface{}) []byte { } return self.key(rn) } - return nil } + +// nodeIteratorState represents the iteration state at one particular node of the +// trie, which can be resumed at a later invocation. +type nodeIteratorState struct { + node node // Trie node being iterated + child int // Child to be processed next +} + +// NodeIterator is an iterator to traverse the trie post-order. +type NodeIterator struct { + trie *Trie // Trie being iterated + stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state + + Node node // Current node being iterated (internal representation) + Leaf bool // Flag whether the current node is a value (data) node + LeafBlob []byte // Data blob contained within a leaf (otherwise nil) +} + +// NewNodeIterator creates an post-order trie iterator. +func NewNodeIterator(trie *Trie) *NodeIterator { + if bytes.Compare(trie.Root(), emptyRoot.Bytes()) == 0 { + return new(NodeIterator) + } + return &NodeIterator{trie: trie} +} + +// Next moves the iterator to the next node, returning whether there are any +// further nodes. +func (it *NodeIterator) Next() bool { + it.step() + return it.retrieve() +} + +// step moves the iterator to the next node of the trie. +func (it *NodeIterator) step() { + // Abort if we reached the end of the iteration + if it.trie == nil { + return + } + // Initialize the iterator if we've just started, or pop off the old node otherwise + if len(it.stack) == 0 { + it.stack = append(it.stack, &nodeIteratorState{node: it.trie.root, child: -1}) + if it.stack[0].node == nil { + panic(fmt.Sprintf("root node missing: %x", it.trie.Root())) + } + } else { + it.stack = it.stack[:len(it.stack)-1] + if len(it.stack) == 0 { + it.trie = nil + return + } + } + // Continue iteration to the next child + for { + parent := it.stack[len(it.stack)-1] + if node, ok := parent.node.(fullNode); ok { + // Full node, traverse all children, then the node itself + if parent.child >= len(node) { + break + } + for parent.child++; parent.child < len(node); parent.child++ { + if current := node[parent.child]; current != nil { + it.stack = append(it.stack, &nodeIteratorState{node: current, child: -1}) + break + } + } + } else if node, ok := parent.node.(shortNode); ok { + // Short node, traverse the pointer singleton child, then the node itself + if parent.child >= 0 { + break + } + parent.child++ + it.stack = append(it.stack, &nodeIteratorState{node: node.Val, child: -1}) + } else if node, ok := parent.node.(hashNode); ok { + // Hash node, resolve the hash child from the database, then the node itself + if parent.child >= 0 { + break + } + parent.child++ + + node, err := it.trie.resolveHash(node, nil, nil) + if err != nil { + panic(err) + } + it.stack = append(it.stack, &nodeIteratorState{node: node, child: -1}) + } else { + break + } + } +} + +// retrieve pulls and caches the current trie node the iterator is traversing. +// In case of a value node, the additional leaf blob is also populated with the +// data contents for external interpretation. +// +// The method returns whether there are any more data left for inspection. +func (it *NodeIterator) retrieve() bool { + // Clear out any previously set values + it.Node, it.Leaf, it.LeafBlob = nil, false, nil + + // If the iteration's done, return no available data + if it.trie == nil { + return false + } + // Otherwise retrieve the current node and resolve leaf accessors + it.Node = it.stack[len(it.stack)-1].node + if value, ok := it.Node.(valueNode); ok { + it.Leaf, it.LeafBlob = true, []byte(value) + } + return true +} diff --git a/trie/sync_test.go b/trie/sync_test.go index 9c036a3a92..94d6edd689 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -33,6 +34,7 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { // Fill it with some arbitrary data content := make(map[string][]byte) for i := byte(0); i < 255; i++ { + // Map the same data under multiple keys key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i} content[string(key)] = val trie.Update(key, val) @@ -40,9 +42,19 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i} content[string(key)] = val trie.Update(key, val) + + // Add some other data to inflate th trie + for j := byte(3); j < 13; j++ { + key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i} + content[string(key)] = val + trie.Update(key, val) + } } trie.Commit() + // Remove any potentially cached data from the test trie creation + globalCache.Clear() + // Return the generated trie return db, trie, content } @@ -50,10 +62,17 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { // checkTrieContents cross references a reconstructed trie with an expected data // content map. func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) { + // Remove any potentially cached data from the trie synchronisation + globalCache.Clear() + + // Check root availability and trie contents trie, err := New(common.BytesToHash(root), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } + if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil { + t.Fatalf("inconsistent trie at %x: %v", root, err) + } for key, val := range content { if have := trie.Get([]byte(key)); bytes.Compare(have, val) != 0 { t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val) @@ -61,6 +80,28 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin } } +// checkTrieConsistency checks that all nodes in a trie and indeed present. +func checkTrieConsistency(db Database, root common.Hash) (failure error) { + // Capture any panics by the iterator + defer func() { + if r := recover(); r != nil { + failure = fmt.Errorf("%v", r) + } + }() + // Remove any potentially cached data from the test trie creation or previous checks + globalCache.Clear() + + // Create and iterate a trie rooted in a subnode + trie, err := New(root, db) + if err != nil { + return + } + it := NewNodeIterator(trie) + for it.Next() { + } + return nil +} + // Tests that an empty trie is not scheduled for syncing. func TestEmptyTrieSync(t *testing.T) { emptyA, _ := New(common.Hash{}, nil) @@ -102,7 +143,7 @@ func testIterativeTrieSync(t *testing.T, batch int) { } queue = append(queue[:0], sched.Missing(batch)...) } - // Cross check that the two tries re in sync + // Cross check that the two tries are in sync checkTrieContents(t, dstDb, srcTrie.Root(), srcData) } @@ -132,7 +173,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) { } queue = append(queue[len(results):], sched.Missing(10000)...) } - // Cross check that the two tries re in sync + // Cross check that the two tries are in sync checkTrieContents(t, dstDb, srcTrie.Root(), srcData) } @@ -173,7 +214,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { queue[hash] = struct{}{} } } - // Cross check that the two tries re in sync + // Cross check that the two tries are in sync checkTrieContents(t, dstDb, srcTrie.Root(), srcData) } @@ -216,7 +257,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { queue[hash] = struct{}{} } } - // Cross check that the two tries re in sync + // Cross check that the two tries are in sync checkTrieContents(t, dstDb, srcTrie.Root(), srcData) } @@ -252,6 +293,57 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { } queue = append(queue[:0], sched.Missing(0)...) } - // Cross check that the two tries re in sync + // Cross check that the two tries are in sync checkTrieContents(t, dstDb, srcTrie.Root(), srcData) } + +// Tests that at any point in time during a sync, only complete sub-tries are in +// the database. +func TestIncompleteTrieSync(t *testing.T) { + // Create a random trie to copy + srcDb, srcTrie, _ := makeTestTrie() + + // Create a destination trie and sync with the scheduler + dstDb, _ := ethdb.NewMemDatabase() + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + + added := []common.Hash{} + queue := append([]common.Hash{}, sched.Missing(1)...) + for len(queue) > 0 { + // Fetch a batch of trie nodes + results := make([]SyncResult, len(queue)) + for i, hash := range queue { + data, err := srcDb.Get(hash.Bytes()) + if err != nil { + t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + } + results[i] = SyncResult{hash, data} + } + // Process each of the trie nodes + if index, err := sched.Process(results); err != nil { + t.Fatalf("failed to process result #%d: %v", index, err) + } + for _, result := range results { + added = append(added, result.Hash) + } + // Check that all known sub-tries in the synced trie is complete + for _, root := range added { + if err := checkTrieConsistency(dstDb, root); err != nil { + t.Fatalf("trie inconsistent: %v", err) + } + } + // Fetch the next batch to retrieve + queue = append(queue[:0], sched.Missing(1)...) + } + // Sanity check that removing any node from the database is detected + for _, node := range added[1:] { + key := node.Bytes() + value, _ := dstDb.Get(key) + + dstDb.Delete(key) + if err := checkTrieConsistency(dstDb, added[0]); err == nil { + t.Fatalf("trie inconsistency not caught, missing: %x", key) + } + dstDb.Put(key, value) + } +} From 3848e94ce771b9459b78988ca169f6d88ee8fd06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 6 Jan 2016 12:11:56 +0200 Subject: [PATCH 2/8] core/state, trie: surface iterator entry hashes --- core/state/iterator.go | 14 +++++---- core/state/iterator_test.go | 57 +++++++++++++++++++++++++++++++++++++ core/state/sync_test.go | 3 +- trie/iterator.go | 25 +++++++++------- trie/iterator_test.go | 32 ++++++++++++++++++++- trie/sync_test.go | 3 +- 6 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 core/state/iterator_test.go diff --git a/core/state/iterator.go b/core/state/iterator.go index a0b71f3eec..59a8e02424 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -33,8 +33,11 @@ type NodeIterator struct { stateIt *trie.NodeIterator // Primary iterator for the global state trie dataIt *trie.NodeIterator // Secondary iterator for the data trie of a contract - code []byte // Source code associated with a contract + codeHash common.Hash // Hash of the contract source code + code []byte // Source code associated with a contract + + Hash common.Hash // Hash of the current entry being iterated (nil if not standalone) Entry interface{} // Current state entry being iterated (internal representation) } @@ -103,6 +106,7 @@ func (it *NodeIterator) step() { it.dataIt = nil } if bytes.Compare(account.CodeHash, emptyCodeHash) != 0 { + it.codeHash = common.BytesToHash(account.CodeHash) it.code, err = it.state.db.Get(account.CodeHash) if err != nil { panic(fmt.Sprintf("code %x: %v", account.CodeHash, err)) @@ -114,7 +118,7 @@ func (it *NodeIterator) step() { // The method returns whether there are any more data left for inspection. func (it *NodeIterator) retrieve() bool { // Clear out any previously set values - it.Entry = nil + it.Hash, it.Entry = common.Hash{}, nil // If the iteration's done, return no available data if it.state == nil { @@ -123,11 +127,11 @@ func (it *NodeIterator) retrieve() bool { // Otherwise retrieve the current entry switch { case it.dataIt != nil: - it.Entry = it.dataIt.Node + it.Hash, it.Entry = it.dataIt.Hash, it.dataIt.Node case it.code != nil: - it.Entry = it.code + it.Hash, it.Entry = it.codeHash, it.code case it.stateIt != nil: - it.Entry = it.stateIt.Node + it.Hash, it.Entry = it.stateIt.Hash, it.stateIt.Node } return true } diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go new file mode 100644 index 0000000000..8b68870c6e --- /dev/null +++ b/core/state/iterator_test.go @@ -0,0 +1,57 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +// Tests that the node iterator indeed walks over the entire database contents. +func TestNodeIteratorCoverage(t *testing.T) { + // Create some arbitrary test state to iterate + db, root, _ := makeTestState() + + state, err := New(root, db) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", root, err) + } + // Gather all the node hashes found by the iterator + hashes := make(map[common.Hash]struct{}) + for it := NewNodeIterator(state); it.Next(); { + if it.Hash != (common.Hash{}) { + hashes[it.Hash] = struct{}{} + } + } + // Cross check the hashes and the database itself + for hash, _ := range hashes { + if _, err := db.Get(hash.Bytes()); err != nil { + t.Errorf("failed to retrieve reported node %x: %v", hash, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, []byte("secure-key-")) { + continue + } + if _, ok := hashes[common.BytesToHash(key)]; !ok { + t.Errorf("state entry not reported %x", key) + } + } +} diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 5d6d90d5d2..727a276a50 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -116,8 +116,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error) if err != nil { return } - it := NewNodeIterator(state) - for it.Next() { + for it := NewNodeIterator(state); it.Next(); { } return nil } diff --git a/trie/iterator.go b/trie/iterator.go index e79de2e4e7..12ca96bd13 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" ) @@ -152,8 +153,9 @@ func (self *Iterator) key(node interface{}) []byte { // nodeIteratorState represents the iteration state at one particular node of the // trie, which can be resumed at a later invocation. type nodeIteratorState struct { - node node // Trie node being iterated - child int // Child to be processed next + hash common.Hash // Hash of the node being iterated (nil if not standalone) + node node // Trie node being iterated + child int // Child to be processed next } // NodeIterator is an iterator to traverse the trie post-order. @@ -161,9 +163,10 @@ type NodeIterator struct { trie *Trie // Trie being iterated stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state - Node node // Current node being iterated (internal representation) - Leaf bool // Flag whether the current node is a value (data) node - LeafBlob []byte // Data blob contained within a leaf (otherwise nil) + Hash common.Hash // Hash of the current node being iterated (nil if not standalone) + Node node // Current node being iterated (internal representation) + Leaf bool // Flag whether the current node is a value (data) node + LeafBlob []byte // Data blob contained within a leaf (otherwise nil) } // NewNodeIterator creates an post-order trie iterator. @@ -221,18 +224,18 @@ func (it *NodeIterator) step() { } parent.child++ it.stack = append(it.stack, &nodeIteratorState{node: node.Val, child: -1}) - } else if node, ok := parent.node.(hashNode); ok { + } else if hash, ok := parent.node.(hashNode); ok { // Hash node, resolve the hash child from the database, then the node itself if parent.child >= 0 { break } parent.child++ - node, err := it.trie.resolveHash(node, nil, nil) + node, err := it.trie.resolveHash(hash, nil, nil) if err != nil { panic(err) } - it.stack = append(it.stack, &nodeIteratorState{node: node, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{hash: common.BytesToHash(hash), node: node, child: -1}) } else { break } @@ -246,14 +249,16 @@ func (it *NodeIterator) step() { // The method returns whether there are any more data left for inspection. func (it *NodeIterator) retrieve() bool { // Clear out any previously set values - it.Node, it.Leaf, it.LeafBlob = nil, false, nil + it.Hash, it.Node, it.Leaf, it.LeafBlob = common.Hash{}, nil, false, nil // If the iteration's done, return no available data if it.trie == nil { return false } // Otherwise retrieve the current node and resolve leaf accessors - it.Node = it.stack[len(it.stack)-1].node + state := it.stack[len(it.stack)-1] + + it.Hash, it.Node = state.hash, state.node if value, ok := it.Node.(valueNode); ok { it.Leaf, it.LeafBlob = true, []byte(value) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index fdc60b4122..dc82761165 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -16,7 +16,12 @@ package trie -import "testing" +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) func TestIterator(t *testing.T) { trie := newEmpty() @@ -47,3 +52,28 @@ func TestIterator(t *testing.T) { } } } + +// Tests that the node iterator indeed walks over the entire database contents. +func TestNodeIteratorCoverage(t *testing.T) { + // Create some arbitrary test trie to iterate + db, trie, _ := makeTestTrie() + + // Gather all the node hashes found by the iterator + hashes := make(map[common.Hash]struct{}) + for it := NewNodeIterator(trie); it.Next(); { + if it.Hash != (common.Hash{}) { + hashes[it.Hash] = struct{}{} + } + } + // Cross check the hashes and the database itself + for hash, _ := range hashes { + if _, err := db.Get(hash.Bytes()); err != nil { + t.Errorf("failed to retrieve reported node %x: %v", hash, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if _, ok := hashes[common.BytesToHash(key)]; !ok { + t.Errorf("state entry not reported %x", key) + } + } +} diff --git a/trie/sync_test.go b/trie/sync_test.go index 94d6edd689..ce098d5a0d 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -96,8 +96,7 @@ func checkTrieConsistency(db Database, root common.Hash) (failure error) { if err != nil { return } - it := NewNodeIterator(trie) - for it.Next() { + for it := NewNodeIterator(trie); it.Next(); { } return nil } From 40228cb20f8da68d6d686884f76ae662c55de35a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 8 Jan 2016 13:46:45 +0200 Subject: [PATCH 3/8] core/state, trie: node iterator reports parent hashes too --- core/state/iterator.go | 20 +++++++++++++------- trie/iterator.go | 22 ++++++++++++++-------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/core/state/iterator.go b/core/state/iterator.go index 59a8e02424..a051e5e01a 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -34,11 +34,13 @@ type NodeIterator struct { stateIt *trie.NodeIterator // Primary iterator for the global state trie dataIt *trie.NodeIterator // Secondary iterator for the data trie of a contract - codeHash common.Hash // Hash of the contract source code - code []byte // Source code associated with a contract + accountHash common.Hash // Hash of the node containing the account + codeHash common.Hash // Hash of the contract source code + code []byte // Source code associated with a contract - Hash common.Hash // Hash of the current entry being iterated (nil if not standalone) - Entry interface{} // Current state entry being iterated (internal representation) + Hash common.Hash // Hash of the current entry being iterated (nil if not standalone) + Entry interface{} // Current state entry being iterated (internal representation) + Parent common.Hash // Hash of the first full ancestor node (nil if current is the root) } // NewNodeIterator creates an post-order state node iterator. @@ -112,6 +114,7 @@ func (it *NodeIterator) step() { panic(fmt.Sprintf("code %x: %v", account.CodeHash, err)) } } + it.accountHash = it.stateIt.Parent } // retrieve pulls and caches the current state entry the iterator is traversing. @@ -127,11 +130,14 @@ func (it *NodeIterator) retrieve() bool { // Otherwise retrieve the current entry switch { case it.dataIt != nil: - it.Hash, it.Entry = it.dataIt.Hash, it.dataIt.Node + it.Hash, it.Entry, it.Parent = it.dataIt.Hash, it.dataIt.Node, it.dataIt.Parent + if it.Parent == (common.Hash{}) { + it.Parent = it.accountHash + } case it.code != nil: - it.Hash, it.Entry = it.codeHash, it.code + it.Hash, it.Entry, it.Parent = it.codeHash, it.code, it.accountHash case it.stateIt != nil: - it.Hash, it.Entry = it.stateIt.Hash, it.stateIt.Node + it.Hash, it.Entry, it.Parent = it.stateIt.Hash, it.stateIt.Node, it.stateIt.Parent } return true } diff --git a/trie/iterator.go b/trie/iterator.go index 12ca96bd13..d43b77698c 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -153,9 +153,10 @@ func (self *Iterator) key(node interface{}) []byte { // nodeIteratorState represents the iteration state at one particular node of the // trie, which can be resumed at a later invocation. type nodeIteratorState struct { - hash common.Hash // Hash of the node being iterated (nil if not standalone) - node node // Trie node being iterated - child int // Child to be processed next + hash common.Hash // Hash of the node being iterated (nil if not standalone) + node node // Trie node being iterated + parent common.Hash // Hash of the first full ancestor node (nil if current is the root) + child int // Child to be processed next } // NodeIterator is an iterator to traverse the trie post-order. @@ -165,6 +166,7 @@ type NodeIterator struct { Hash common.Hash // Hash of the current node being iterated (nil if not standalone) Node node // Current node being iterated (internal representation) + Parent common.Hash // Hash of the first full ancestor node (nil if current is the root) Leaf bool // Flag whether the current node is a value (data) node LeafBlob []byte // Data blob contained within a leaf (otherwise nil) } @@ -206,6 +208,10 @@ func (it *NodeIterator) step() { // Continue iteration to the next child for { parent := it.stack[len(it.stack)-1] + ancestor := parent.hash + if (ancestor == common.Hash{}) { + ancestor = parent.parent + } if node, ok := parent.node.(fullNode); ok { // Full node, traverse all children, then the node itself if parent.child >= len(node) { @@ -213,7 +219,7 @@ func (it *NodeIterator) step() { } for parent.child++; parent.child < len(node); parent.child++ { if current := node[parent.child]; current != nil { - it.stack = append(it.stack, &nodeIteratorState{node: current, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{node: current, parent: ancestor, child: -1}) break } } @@ -223,7 +229,7 @@ func (it *NodeIterator) step() { break } parent.child++ - it.stack = append(it.stack, &nodeIteratorState{node: node.Val, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1}) } else if hash, ok := parent.node.(hashNode); ok { // Hash node, resolve the hash child from the database, then the node itself if parent.child >= 0 { @@ -235,7 +241,7 @@ func (it *NodeIterator) step() { if err != nil { panic(err) } - it.stack = append(it.stack, &nodeIteratorState{hash: common.BytesToHash(hash), node: node, child: -1}) + it.stack = append(it.stack, &nodeIteratorState{hash: common.BytesToHash(hash), node: node, parent: ancestor, child: -1}) } else { break } @@ -249,7 +255,7 @@ func (it *NodeIterator) step() { // The method returns whether there are any more data left for inspection. func (it *NodeIterator) retrieve() bool { // Clear out any previously set values - it.Hash, it.Node, it.Leaf, it.LeafBlob = common.Hash{}, nil, false, nil + it.Hash, it.Node, it.Parent, it.Leaf, it.LeafBlob = common.Hash{}, nil, common.Hash{}, false, nil // If the iteration's done, return no available data if it.trie == nil { @@ -258,7 +264,7 @@ func (it *NodeIterator) retrieve() bool { // Otherwise retrieve the current node and resolve leaf accessors state := it.stack[len(it.stack)-1] - it.Hash, it.Node = state.hash, state.node + it.Hash, it.Node, it.Parent = state.hash, state.node, state.parent if value, ok := it.Node.(valueNode); ok { it.Leaf, it.LeafBlob = true, []byte(value) } From a7f54f1b9500b0d098981519255ec2c9ad33f999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 8 Jan 2016 19:25:23 +0200 Subject: [PATCH 4/8] core/state, trie: generate parent reference database indexes --- core/state/index_test.go | 57 +++++++++++++++++++++++++++++++ core/state/iterator_test.go | 4 +++ core/state/statedb.go | 4 ++- core/state/sync_test.go | 28 +++++++++++++++ trie/index.go | 68 +++++++++++++++++++++++++++++++++++++ trie/index_test.go | 52 ++++++++++++++++++++++++++++ trie/iterator.go | 12 +++---- trie/iterator_test.go | 6 ++-- trie/node.go | 21 +++++++++--- trie/proof.go | 2 +- trie/secure_trie.go | 28 +++++++++++++++ trie/sync.go | 18 ++++++++-- trie/sync_test.go | 25 ++++++++++++++ trie/trie.go | 45 ++++++++++++++++++++---- 14 files changed, 347 insertions(+), 23 deletions(-) create mode 100644 core/state/index_test.go create mode 100644 trie/index.go create mode 100644 trie/index_test.go diff --git a/core/state/index_test.go b/core/state/index_test.go new file mode 100644 index 0000000000..25f1f853f2 --- /dev/null +++ b/core/state/index_test.go @@ -0,0 +1,57 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" +) + +// Tests that all index entries are stored in the database after a state commit. +func TestStateIndex(t *testing.T) { + // Create some arbitrary test state to iterate + db, root, _ := makeTestState() + + state, err := New(root, db) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", root, err) + } + // Gather all the indexes that should be present in the database + indexes := make(map[string]struct{}) + for it := NewNodeIterator(state); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + t.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + t.Errorf("index entry not reported %x", key) + } + } + } +} diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 8b68870c6e..df0135296a 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) // Tests that the node iterator indeed walks over the entire database contents. @@ -50,6 +51,9 @@ func TestNodeIteratorCoverage(t *testing.T) { if bytes.HasPrefix(key, []byte("secure-key-")) { continue } + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + continue + } if _, ok := hashes[common.BytesToHash(key)]; !ok { t.Errorf("state entry not reported %x", key) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 8093472b59..e8e1e269e2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -206,15 +206,17 @@ func (self *StateDB) Delete(addr common.Address) bool { // Update the given state object and apply it to state trie func (self *StateDB) UpdateStateObject(stateObject *StateObject) { + refs := [][]byte{stateObject.Root()} if len(stateObject.code) > 0 { self.db.Put(stateObject.codeHash, stateObject.code) + refs = append(refs, stateObject.codeHash) } addr := stateObject.Address() data, err := rlp.EncodeToBytes(stateObject) if err != nil { panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err)) } - self.trie.Update(addr[:], data) + self.trie.UpdateIndexed(addr[:], data, refs) } // Delete the given state object and delete it from the state trie diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 727a276a50..5e59c8f9cd 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -84,6 +84,9 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou if err := checkStateConsistency(db, root); err != nil { t.Fatalf("inconsistent state trie at %x: %v", root, err) } + if err := checkStateIndex(db, root); err != nil { + t.Fatalf("index error at %x: %v", root, err) + } for i, acc := range accounts { if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 { t.Errorf("account %d: balance mismatch: have %v, want %v", i, balance, acc.balance) @@ -121,6 +124,31 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error) return nil } +// checkStateIndex iterates over the entire state trie and checks that all required +// database indexes have been generated by the synchronizer. +func checkStateIndex(db ethdb.Database, root common.Hash) error { + // Remove any potentially cached data from the test state creation or previous checks + trie.ClearGlobalCache() + + // Create and iterate a state trie rooted in a sub-node + if _, err := db.Get(root.Bytes()); err != nil { + return err + } + state, err := New(root, db) + if err != nil { + return err + } + for it := NewNodeIterator(state); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + parentRef := trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()) + if _, err := db.Get(parentRef); err != nil { + return fmt.Errorf("parent reference lookup %x: %v", parentRef, err) + } + } + } + return nil +} + // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") diff --git a/trie/index.go b/trie/index.go new file mode 100644 index 0000000000..705e605ea5 --- /dev/null +++ b/trie/index.go @@ -0,0 +1,68 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import "bytes" + +var ( + // ParentReferenceIndexPrefix is the database key prefix storing parent references index entries + ParentReferenceIndexPrefix = []byte("ref-") +) + +// ParentReferenceIndexKey constructs a child->parent database index key. +func ParentReferenceIndexKey(parent []byte, child []byte) []byte { + return append(append(ParentReferenceIndexPrefix, child...), parent...) +} + +// storeParentReferences expands a trie node to find all its stored children and +// adds a database reference pointing to the parent to permit reference tracking. +func storeParentReferences(key []byte, node node, db DatabaseWriter) error { + switch node := node.(type) { + case fullNode: + for _, child := range node { + if child != nil { + if err := storeParentReferences(key, child, db); err != nil { + return err + } + } + } + case shortNode: + if child := node.Val; child != nil { + return storeParentReferences(key, child, db) + } + case hashNode: + return db.Put(ParentReferenceIndexKey(key, node), nil) + + case valueNode: + for _, child := range node.refs { + if bytes.Compare(child, emptyRoot.Bytes()) == 0 { + continue // don't index an empty trie + } + if err := db.Put(ParentReferenceIndexKey(key, child), nil); err != nil { + return err + } + } + } + return nil +} + +// storeParentReferenceEntry manually inserts a parent->child reference entry into +// the database index without requiring direct, trie-internal descendancy. This is +// useful to store references to external entities, like accounts or blocks. +func storeParentReferenceEntry(parent []byte, child []byte, db DatabaseWriter) error { + return db.Put(ParentReferenceIndexKey(parent, child), nil) +} diff --git a/trie/index_test.go b/trie/index_test.go new file mode 100644 index 0000000000..1ecf8dd7bf --- /dev/null +++ b/trie/index_test.go @@ -0,0 +1,52 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +// Tests that all index entries are stored in the database after a trie commit. +func TestTrieIndex(t *testing.T) { + // Create some arbitrary test trie to iterate + db, trie, _ := makeTestTrie() + + // Gather all the indexes that should be present in the database + indexes := make(map[string]struct{}) + for it := NewNodeIterator(trie); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + t.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + t.Errorf("index entry not reported %x", key) + } + } + } +} diff --git a/trie/iterator.go b/trie/iterator.go index d43b77698c..59ac3c5182 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -86,11 +86,11 @@ func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byt switch bytes.Compare([]byte(k), key) { case 0: if isIterStart { - self.Value = vnode + self.Value = vnode.Value return k } case 1: - self.Value = vnode + self.Value = vnode.Value return k } } else { @@ -125,13 +125,13 @@ func (self *Iterator) key(node interface{}) []byte { // Leaf node k := remTerm(node.Key) if vnode, ok := node.Val.(valueNode); ok { - self.Value = vnode + self.Value = vnode.Value return k } return append(k, self.key(node.Val)...) case fullNode: if node[16] != nil { - self.Value = node[16].(valueNode) + self.Value = node[16].(valueNode).Value return []byte{16} } for i := 0; i < 16; i++ { @@ -265,8 +265,8 @@ func (it *NodeIterator) retrieve() bool { state := it.stack[len(it.stack)-1] it.Hash, it.Node, it.Parent = state.hash, state.node, state.parent - if value, ok := it.Node.(valueNode); ok { - it.Leaf, it.LeafBlob = true, []byte(value) + if node, ok := it.Node.(valueNode); ok { + it.Leaf, it.LeafBlob = true, []byte(node.Value) } return true } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index dc82761165..815743bfdc 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -72,8 +72,10 @@ func TestNodeIteratorCoverage(t *testing.T) { } } for _, key := range db.(*ethdb.MemDatabase).Keys() { - if _, ok := hashes[common.BytesToHash(key)]; !ok { - t.Errorf("state entry not reported %x", key) + if len(key) == common.HashLength { + if _, ok := hashes[common.BytesToHash(key)]; !ok { + t.Errorf("state entry not reported %x", key) + } } } } diff --git a/trie/node.go b/trie/node.go index 0bfa21dc43..01452a8531 100644 --- a/trie/node.go +++ b/trie/node.go @@ -38,9 +38,22 @@ type ( Val node } hashNode []byte - valueNode []byte + valueNode struct { + Value []byte + refs [][]byte + } ) +func (n valueNode) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, n.Value) +} + +func (n valueNode) DecodeRLP(s *rlp.Stream) (err error) { + fmt.Println("Decoding value node") + n.Value, err = s.Bytes() + return +} + // Pretty printing. func (n fullNode) String() string { return n.fstring("") } func (n shortNode) String() string { return n.fstring("") } @@ -65,7 +78,7 @@ func (n hashNode) fstring(ind string) string { return fmt.Sprintf("<%x> ", []byte(n)) } func (n valueNode) fstring(ind string) string { - return fmt.Sprintf("%x ", []byte(n)) + return fmt.Sprintf("%x ", []byte(n.Value)) } func mustDecodeNode(dbkey, buf []byte) node { @@ -109,7 +122,7 @@ func decodeShort(buf []byte) (node, error) { if err != nil { return nil, fmt.Errorf("invalid value node: %v", err) } - return shortNode{key, valueNode(val)}, nil + return shortNode{key, valueNode{Value: val}}, nil } r, _, err := decodeRef(rest) if err != nil { @@ -132,7 +145,7 @@ func decodeFull(buf []byte) (fullNode, error) { return n, err } if len(val) > 0 { - n[16] = valueNode(val) + n[16] = valueNode{Value: val} } return n, nil } diff --git a/trie/proof.go b/trie/proof.go index 0f92649424..3c49309026 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -107,7 +107,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value if i != len(proof)-1 { return nil, errors.New("additional nodes at end of proof") } - return cld, nil + return cld.Value, nil } } return nil, errors.New("unexpected end of proof") diff --git a/trie/secure_trie.go b/trie/secure_trie.go index caeef3c3a3..fb5f694d8e 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -109,6 +109,34 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error { return nil } +// UpdateIndexed is an extended version of Update, where state trie index entries +// are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +func (t *SecureTrie) UpdateIndexed(key, value []byte, refs [][]byte) { + if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) { + glog.Errorf("Unhandled trie error: %v", err) + } +} + +// TryUpdateIndexed is an extended version of Update, where state trie index +// entries are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +// +// If a node was not found in the database, a MissingNodeError is returned. +func (t *SecureTrie) TryUpdateIndexed(key, value []byte, refs [][]byte) error { + hk := t.hashKey(key) + err := t.Trie.TryUpdateIndexed(hk, value, refs) + if err != nil { + return err + } + t.Trie.db.Put(t.secKey(hk), key) + return nil +} + // Delete removes any existing value for key from the trie. func (t *SecureTrie) Delete(key []byte) { if err := t.TryDelete(key); err != nil && glog.V(logger.Error) { diff --git a/trie/sync.go b/trie/sync.go index d55399d06b..c3cd36cf79 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -34,6 +34,8 @@ type request struct { depth int // Depth level within the trie the node is located to prioritise DFS deps int // Number of dependencies before allowed to commit this node + externals []common.Hash // External children of this node to index in addition to internal ones + callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch } @@ -94,6 +96,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) } ancestor.deps++ + ancestor.externals = append(ancestor.externals, root) req.parents = append(req.parents, ancestor) } s.schedule(req) @@ -123,6 +126,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) } ancestor.deps++ + ancestor.externals = append(ancestor.externals, hash) req.parents = append(req.parents, ancestor) } s.schedule(req) @@ -229,7 +233,7 @@ func (s *TrieSync) children(req *request) ([]*request, error) { // Notify any external watcher of a new key/value node if req.callback != nil { if node, ok := (*child.node).(valueNode); ok { - if err := req.callback(node, req.hash); err != nil { + if err := req.callback(node.Value, req.hash); err != nil { return nil, err } } @@ -266,7 +270,17 @@ func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) { err = batch.Write() }() } - // Write the node content to disk + // Write the node content and indices to disk + if req.object != nil { + if err := storeParentReferences(req.hash[:], *req.object, batch); err != nil { + return err + } + } + for _, ext := range req.externals { + if err := storeParentReferenceEntry(req.hash[:], ext[:], batch); err != nil { + return err + } + } if err := batch.Put(req.hash[:], req.data); err != nil { return err } diff --git a/trie/sync_test.go b/trie/sync_test.go index ce098d5a0d..6738427bfb 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -73,6 +73,9 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil { t.Fatalf("inconsistent trie at %x: %v", root, err) } + if err := checkTrieIndex(db, common.BytesToHash(root)); err != nil { + t.Fatalf("index error at %x: %v", root, err) + } for key, val := range content { if have := trie.Get([]byte(key)); bytes.Compare(have, val) != 0 { t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val) @@ -101,6 +104,28 @@ func checkTrieConsistency(db Database, root common.Hash) (failure error) { return nil } +// checkTrieIndex iterates over the entire state trie and checks that all required +// database indexes have been generated by the synchronizer. +func checkTrieIndex(db Database, root common.Hash) error { + // Remove any potentially cached data from the test trie creation or previous checks + globalCache.Clear() + + // Create and iterate a trie + trie, err := New(root, db) + if err != nil { + return err + } + for it := NewNodeIterator(trie); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + parentRef := ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()) + if _, err := db.Get(parentRef); err != nil { + return fmt.Errorf("parent reference lookup %x: %v", parentRef, err) + } + } + } + return nil +} + // Tests that an empty trie is not scheduled for syncing. func TestEmptyTrieSync(t *testing.T) { emptyA, _ := New(common.Hash{}, nil) diff --git a/trie/trie.go b/trie/trie.go index 9dfde45294..88b32aa45a 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -143,7 +143,7 @@ func (t *Trie) TryGet(key []byte) ([]byte, error) { panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - return tn.(valueNode), nil + return tn.(valueNode).Value, nil } // Update associates key with value in the trie. Subsequent calls to @@ -167,9 +167,35 @@ func (t *Trie) Update(key, value []byte) { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + return t.tryUpdateIndexed(key, value, nil) +} + +// UpdateIndexed is an extended version of Update, where state trie index entries +// are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +func (t *Trie) UpdateIndexed(key, value []byte, refs [][]byte) { + if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) { + glog.Errorf("Unhandled trie error: %v", err) + } +} + +// TryUpdateIndexed is an extended version of Update, where state trie index +// entries are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +// +// If a node was not found in the database, a MissingNodeError is returned. +func (t *Trie) TryUpdateIndexed(key, value []byte, refs [][]byte) error { + return t.tryUpdateIndexed(key, value, refs) +} + +func (t *Trie) tryUpdateIndexed(key, value []byte, refs [][]byte) error { k := compactHexDecode(key) if len(value) != 0 { - n, err := t.insert(t.root, nil, k, valueNode(value)) + n, err := t.insert(t.root, nil, k, valueNode{Value: value, refs: refs}) if err != nil { return err } @@ -489,7 +515,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { } if n.Val == nil { // Ensure that nil children are encoded as empty strings. - n.Val = valueNode(nil) + n.Val = valueNode{Value: nil} } return n, nil case fullNode: @@ -500,11 +526,11 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { } } else { // Ensure that nil children are encoded as empty strings. - n[i] = valueNode(nil) + n[i] = valueNode{Value: nil} } } if n[16] == nil { - n[16] = valueNode(nil) + n[16] = valueNode{Value: nil} } return n, nil default: @@ -530,8 +556,13 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { h.sha.Write(h.tmp.Bytes()) key := hashNode(h.sha.Sum(nil)) if db != nil { - err := db.Put(key, h.tmp.Bytes()) - return key, err + if err := storeParentReferences(key, n, db); err != nil { + return key, err + } + if err := db.Put(key, h.tmp.Bytes()); err != nil { + return key, err + } + return key, nil } return key, nil } From 371be370504f61d55d243cac6afc4914733cc1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 11 Jan 2016 18:20:31 +0200 Subject: [PATCH 5/8] core/vm: resolve circular dependency to debug vm storage --- core/vm/contract.go | 7 +++++++ core/vm/environment.go | 1 + core/vm/jit_test.go | 17 +++++++++-------- core/vm/vm.go | 9 +++------ 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/core/vm/contract.go b/core/vm/contract.go index 95417e7471..5981dcca0d 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -27,6 +27,7 @@ type ContractRef interface { ReturnGas(*big.Int, *big.Int) Address() common.Address SetCode([]byte) + EachStorage(cb func(key, value []byte)) } // Contract represents an ethereum contract in the state database. It contains @@ -124,3 +125,9 @@ func (self *Contract) SetCallCode(addr *common.Address, code []byte) { self.Code = code self.CodeAddr = addr } + +// EachStorage iterates the contract's storage and calls a method for every key +// value pair. +func (self *Contract) EachStorage(cb func(key, value []byte)) { + self.caller.EachStorage(cb) +} diff --git a/core/vm/environment.go b/core/vm/environment.go index 299d126747..4fee583bfb 100644 --- a/core/vm/environment.go +++ b/core/vm/environment.go @@ -121,4 +121,5 @@ type Account interface { Address() common.Address ReturnGas(*big.Int, *big.Int) SetCode([]byte) + EachStorage(cb func(key, value []byte)) } diff --git a/core/vm/jit_test.go b/core/vm/jit_test.go index aa97e51844..8c50ed0f5c 100644 --- a/core/vm/jit_test.go +++ b/core/vm/jit_test.go @@ -125,14 +125,15 @@ type vmBench struct { type account struct{} -func (account) SubBalance(amount *big.Int) {} -func (account) AddBalance(amount *big.Int) {} -func (account) SetBalance(*big.Int) {} -func (account) SetNonce(uint64) {} -func (account) Balance() *big.Int { return nil } -func (account) Address() common.Address { return common.Address{} } -func (account) ReturnGas(*big.Int, *big.Int) {} -func (account) SetCode([]byte) {} +func (account) SubBalance(amount *big.Int) {} +func (account) AddBalance(amount *big.Int) {} +func (account) SetBalance(*big.Int) {} +func (account) SetNonce(uint64) {} +func (account) Balance() *big.Int { return nil } +func (account) Address() common.Address { return common.Address{} } +func (account) ReturnGas(*big.Int, *big.Int) {} +func (account) SetCode([]byte) {} +func (account) EachStorage(cb func(key, value []byte)) {} func runVmBench(test vmBench, b *testing.B) { var sender account diff --git a/core/vm/vm.go b/core/vm/vm.go index 4b03e55f04..8e07aaa899 100644 --- a/core/vm/vm.go +++ b/core/vm/vm.go @@ -376,12 +376,9 @@ func (self *Vm) log(pc uint64, op OpCode, gas, cost *big.Int, memory *Memory, st stck[i] = new(big.Int).Set(item) } storage := make(map[common.Hash][]byte) - /* - object := contract.self.(*state.StateObject) - object.EachStorage(func(k, v []byte) { - storage[common.BytesToHash(k)] = v - }) - */ + contract.self.EachStorage(func(k, v []byte) { + storage[common.BytesToHash(k)] = v + }) self.env.AddStructLog(StructLog{pc, op, new(big.Int).Set(gas), cost, mem, stck, storage, err}) } } From c485ac81bee8f06582390e381827cb9b893fa639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 12 Jan 2016 11:38:36 +0200 Subject: [PATCH 6/8] core, eth, light, miner, tests, trie: cross trie indexing --- core/blockchain.go | 4 +- core/blockchain_test.go | 2 +- core/chain_makers.go | 10 ++- core/genesis.go | 18 ++--- core/index_test.go | 155 ++++++++++++++++++++++++++++++++++++ core/state/index_test.go | 14 +++- core/state/iterator_test.go | 2 +- core/state/state_object.go | 2 +- core/state/state_test.go | 2 +- core/state/statedb.go | 22 ++--- core/state/sync.go | 4 +- core/state/sync_test.go | 119 +++++++++++++++++++-------- core/types/derive_sha.go | 2 +- eth/downloader/queue.go | 2 +- light/state_test.go | 2 +- light/trie.go | 11 +-- miner/worker.go | 2 +- tests/block_test_util.go | 2 +- tests/state_test_util.go | 4 +- trie/index_test.go | 10 ++- trie/iterator_test.go | 6 +- trie/proof.go | 2 +- trie/proof_test.go | 6 +- trie/secure_trie.go | 30 ------- trie/secure_trie_test.go | 4 +- trie/sync.go | 38 +++++---- trie/sync_test.go | 140 ++++++++++++++++++++++---------- trie/trie.go | 92 +++++++++------------ trie/trie_test.go | 40 +++++----- 29 files changed, 496 insertions(+), 251 deletions(-) create mode 100644 core/index_test.go diff --git a/core/blockchain.go b/core/blockchain.go index 95ed06d8d6..0878597b36 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1195,11 +1195,9 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { return i, err } // Write state changes to database - _, err = statedb.Commit() - if err != nil { + if _, err = statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil { return i, err } - // coalesce logs for later processing coalescedLogs = append(coalescedLogs, logs...) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b4ac1696a3..7afc829efd 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -153,7 +153,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { blockchain.mu.Lock() WriteTd(blockchain.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), blockchain.GetTd(block.ParentHash()))) WriteBlock(blockchain.chainDb, block) - statedb.Commit() + statedb.CommitIndexed([]common.Hash{block.Hash()}) blockchain.mu.Unlock() } return nil diff --git a/core/chain_makers.go b/core/chain_makers.go index c62618e6cb..761d61901d 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -171,12 +171,14 @@ func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, gen(i, b) } AccumulateRewards(statedb, h, b.uncles) - root, err := statedb.Commit() - if err != nil { + + h.Root = statedb.IntermediateRoot() + block := types.NewBlock(h, b.txs, b.uncles, b.receipts) + + if _, err := statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil { panic(fmt.Sprintf("state write error: %v", err)) } - h.Root = root - return types.NewBlock(h, b.txs, b.uncles, b.receipts), b.receipts + return block, b.receipts } for i := 0; i < n; i++ { statedb, err := state.New(parent.Root(), db) diff --git a/core/genesis.go b/core/genesis.go index d8c6e9cea3..ed4949194c 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -72,8 +72,6 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, statedb.SetState(address, common.HexToHash(key), common.HexToHash(value)) } } - root, stateBatch := statedb.CommitBatch() - difficulty := common.String2Big(genesis.Difficulty) block := types.NewBlock(&types.Header{ Nonce: types.EncodeNonce(common.String2Big(genesis.Nonce).Uint64()), @@ -84,7 +82,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, Difficulty: difficulty, MixDigest: common.HexToHash(genesis.Mixhash), Coinbase: common.HexToAddress(genesis.Coinbase), - Root: root, + Root: statedb.IntermediateRoot(), }, nil, nil, nil) if block := GetBlock(chainDb, block.Hash()); block != nil { @@ -95,8 +93,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, } return block, nil } - - if err := stateBatch.Write(); err != nil { + if _, err := statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil { return nil, fmt.Errorf("cannot write state: %v", err) } if err := WriteTd(chainDb, block.Hash(), difficulty); err != nil { @@ -123,15 +120,16 @@ func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big statedb, _ := state.New(common.Hash{}, db) obj := statedb.GetOrNewStateObject(addr) obj.SetBalance(balance) - root, err := statedb.Commit() - if err != nil { - panic(fmt.Sprintf("cannot write state: %v", err)) - } + block := types.NewBlock(&types.Header{ Difficulty: params.GenesisDifficulty, GasLimit: params.GenesisGasLimit, - Root: root, + Root: statedb.IntermediateRoot(), }, nil, nil, nil) + + if _, err := statedb.CommitIndexed([]common.Hash{block.Hash()}); err != nil { + panic(fmt.Sprintf("cannot write state: %v", err)) + } return block } diff --git a/core/index_test.go b/core/index_test.go new file mode 100644 index 0000000000..2d4a8635b3 --- /dev/null +++ b/core/index_test.go @@ -0,0 +1,155 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "bytes" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +// blackHoleContract is a trivial Ethereum contract that can just store some data +// points in its state trie. It's goal is to have a means to test functionality +// depending on evolving state roots. +// +// contract BlackHole { +// mapping (int256 => uint) public data; +// +// function set(int256 key, uint value) { +// data[key] = value; +// } +// } +var blackHoleContract = common.Hex2Bytes("6060604052605f8060106000396000f3606060405260e060020a60003504639398e0cd81146024578063a22c554014603b575b005b605560043560006020819052908152604090205481565b600435600090815260208190526040902060243590556022565b6060908152602090f3") +var blackHoleSetter = common.Hex2Bytes("a22c5540") + +// Tests that state trie indexes are properly constructed for an entire chain of +// imported blocks, cross referencing between various state roots, as well as +// making sure that state updates do not lose reference entries. +func TestChainIndex(t *testing.T) { + // Configure the test chain and ensure we have enough funds to play with + var ( + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + key3, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + addr3 = crypto.PubkeyToAddress(key3.PublicKey) + db, _ = ethdb.NewMemDatabase() + ) + genesis := WriteGenesisBlockForTesting(db, GenesisAccount{addr1, big.NewInt(10000000000)}) + + // Generate a chain will all kinds of events happening in it + var contract common.Address + + chain, _ := GenerateChain(genesis, db, 8, func(i int, gen *BlockGen) { + switch i { + case 0: + // In block 1, addr1 sends addr2 some ether. + tx, _ := types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil).SignECDSA(key1) + gen.AddTx(tx) + case 1: + // In block 2, addr1 sends some more ether to addr2. + // addr2 passes it on to addr3. + tx1, _ := types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key1) + tx2, _ := types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key2) + gen.AddTx(tx1) + gen.AddTx(tx2) + case 2: + // Block 3 is empty but was mined by addr3. + gen.SetCoinbase(addr3) + gen.SetExtra([]byte("yeehaw")) + case 3: + // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). + b2 := gen.PrevBlock(1).Header() + b2.Extra = []byte("foo") + gen.AddUncle(b2) + b3 := gen.PrevBlock(2).Header() + b3.Extra = []byte("foo") + gen.AddUncle(b3) + case 4: + // In block 5, we create a simple storage contract to add data entries to + tx, _ := types.NewContractCreation(gen.TxNonce(addr1), big.NewInt(31415), params.GenesisGasLimit, big.NewInt(1), blackHoleContract).SignECDSA(key1) + contract = crypto.CreateAddress(addr1, tx.Nonce()) + gen.AddTx(tx) + case 5: + // In block 6, we store a single entry into the storage to check single update indexing + key := common.Hex2BytesFixed("01", 32) + val := common.Hex2BytesFixed("02", 32) + + tx, _ := types.NewTransaction(gen.TxNonce(addr1), contract, nil, params.GenesisGasLimit, nil, append(append(blackHoleSetter, key...), val...)).SignECDSA(key1) + gen.AddTx(tx) + case 6: + // In block 7, we store a lot of entries into the storage to check multi update indexing + for i := int64(0); i < 50; i++ { + key := common.Hex2BytesFixed(common.Bytes2Hex(big.NewInt(i).Bytes()), 32) + val := common.Hex2BytesFixed(common.Bytes2Hex(big.NewInt(i+2).Bytes()), 32) + + tx, _ := types.NewTransaction(gen.TxNonce(addr1), contract, nil, big.NewInt(45000), nil, append(append(blackHoleSetter, key...), val...)).SignECDSA(key1) + gen.AddTx(tx) + } + } + }) + // Import the chain. This runs all block validation rules. + evmux := &event.TypeMux{} + + blockchain, _ := NewBlockChain(db, FakePow{}, evmux) + if i, err := blockchain.InsertChain(chain); err != nil { + t.Fatalf("block %d: insert failed: %v\n", i, err) + return + } + // Iterate over all the blocks and check the state trie indexes + indexes := make(map[string]struct{}) + + for num := uint64(0); num <= blockchain.CurrentBlock().NumberU64(); num++ { + // Gather all the indexes that should be present in the state trie + block := blockchain.GetBlockByNumber(num) + + stateDb, err := state.New(block.Root(), db) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", block.Root(), err) + } + for it := state.NewNodeIterator(stateDb); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + // Add the block->state index entries + indexes[string(trie.ParentReferenceIndexKey(block.Hash().Bytes(), block.Root().Bytes()))] = struct{}{} + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + t.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.Keys() { + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + t.Errorf("index entry not reported %x", key) + } + } + } +} diff --git a/core/state/index_test.go b/core/state/index_test.go index 25f1f853f2..8b349857eb 100644 --- a/core/state/index_test.go +++ b/core/state/index_test.go @@ -26,9 +26,16 @@ import ( ) // Tests that all index entries are stored in the database after a state commit. -func TestStateIndex(t *testing.T) { +func TestStateIndexDangling(t *testing.T) { + testStateIndex(t, nil) +} +func TestStateIndexRooted(t *testing.T) { + testStateIndex(t, []common.Hash{common.BytesToHash([]byte{0x01}), common.BytesToHash([]byte{0x02, 0x03})}) +} + +func testStateIndex(t *testing.T, referrers []common.Hash) { // Create some arbitrary test state to iterate - db, root, _ := makeTestState() + db, root, _ := makeTestState(referrers) state, err := New(root, db) if err != nil { @@ -41,6 +48,9 @@ func TestStateIndex(t *testing.T) { indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} } } + for _, referrer := range referrers { + indexes[string(trie.ParentReferenceIndexKey(referrer.Bytes(), root.Bytes()))] = struct{}{} + } // Cross check the indexes and the database itself for index, _ := range indexes { if _, err := db.Get([]byte(index)); err != nil { diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index df0135296a..d44e15be7d 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -28,7 +28,7 @@ import ( // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { // Create some arbitrary test state to iterate - db, root, _ := makeTestState() + db, root, _ := makeTestState(nil) state, err := New(root, db) if err != nil { diff --git a/core/state/state_object.go b/core/state/state_object.go index 47546112f4..292f32f301 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -118,7 +118,7 @@ func (c *StateObject) setAddr(addr []byte, value common.Hash) { // if RLPing failed we better panic and not fail silently. This would be considered a consensus issue panic(err) } - c.trie.Update(addr, v) + c.trie.UpdateIndexed(addr, v, nil) } func (self *StateObject) Storage() Storage { diff --git a/core/state/state_test.go b/core/state/state_test.go index 7ce341c361..040683ae9d 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -89,7 +89,7 @@ func TestNull(t *testing.T) { //value := common.FromHex("0x823140710bf13990e4500136726d8b55") var value common.Hash state.SetState(address, common.Hash{}, value) - state.Commit() + state.CommitIndexed(nil) value = state.GetState(address, common.Hash{}) if !common.EmptyHash(value) { t.Errorf("expected empty hash. got %x", value) diff --git a/core/state/statedb.go b/core/state/statedb.go index e8e1e269e2..1077e3ccd8 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -353,22 +353,22 @@ func (s *StateDB) IntermediateRoot() common.Hash { return s.trie.Hash() } -// Commit commits all state changes to the database. -func (s *StateDB) Commit() (root common.Hash, err error) { - root, batch := s.CommitBatch() +// CommitIndexed commits all state changes to the database. +func (s *StateDB) CommitIndexed(refs []common.Hash) (root common.Hash, err error) { + root, batch := s.CommitBatchIndexed(refs) return root, batch.Write() } -// CommitBatch commits all state changes to a write batch but does not +// CommitBatchIndexed commits all state changes to a write batch but does not // execute the batch. It is used to validate state changes against // the root hash stored in a block. -func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) { +func (s *StateDB) CommitBatchIndexed(refs []common.Hash) (root common.Hash, batch ethdb.Batch) { batch = s.db.NewBatch() - root, _ = s.commit(batch) + root, _ = s.commit(batch, refs) return root, batch } -func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) { +func (s *StateDB) commit(db trie.DatabaseWriter, refs []common.Hash) (common.Hash, error) { s.refund = new(big.Int) for _, stateObject := range s.stateObjects { @@ -383,7 +383,7 @@ func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) { // This updates the trie root internally, so // getting the root hash of the storage trie // through UpdateStateObject is fast. - if _, err := stateObject.trie.CommitTo(db); err != nil { + if _, err := stateObject.trie.CommitToIndexed(db, nil); err != nil { return common.Hash{}, err } // Update the object in the account trie. @@ -391,7 +391,11 @@ func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) { } stateObject.dirty = false } - return s.trie.CommitTo(db) + referrers := make([][]byte, len(refs)) + for i, ref := range refs { + referrers[i] = ref.Bytes() + } + return s.trie.CommitToIndexed(db, referrers) } func (self *StateDB) Refunds() *big.Int { diff --git a/core/state/sync.go b/core/state/sync.go index ef2b4b84c5..5d77517e7e 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -32,7 +32,7 @@ import ( type StateSync trie.TrieSync // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database ethdb.Database) *StateSync { +func NewStateSync(root common.Hash, database ethdb.Database, parent common.Hash) *StateSync { var syncer *trie.TrieSync callback := func(leaf []byte, parent common.Hash) error { @@ -50,7 +50,7 @@ func NewStateSync(root common.Hash, database ethdb.Database) *StateSync { return nil } - syncer = trie.NewTrieSync(root, database, callback) + syncer = trie.NewTrieSync(root, database, parent, callback) return (*StateSync)(syncer) } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 5e59c8f9cd..30cd70eec6 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -37,7 +37,7 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { +func makeTestState(referrers []common.Hash) (ethdb.Database, common.Hash, []*testAccount) { // Create an empty state db, _ := ethdb.NewMemDatabase() state, _ := New(common.Hash{}, db) @@ -61,7 +61,7 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { state.UpdateStateObject(obj) accounts = append(accounts, acc) } - root, _ := state.Commit() + root, _ := state.CommitIndexed(referrers) // Remove any potentially cached data from the test state creation trie.ClearGlobalCache() @@ -72,7 +72,7 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { // checkStateAccounts cross references a reconstructed state with an expected // account array. -func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { +func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount, parent common.Hash) { // Remove any potentially cached data from the state synchronisation trie.ClearGlobalCache() @@ -84,7 +84,7 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou if err := checkStateConsistency(db, root); err != nil { t.Fatalf("inconsistent state trie at %x: %v", root, err) } - if err := checkStateIndex(db, root); err != nil { + if err := checkStateIndex(db, root, parent); err != nil { t.Fatalf("index error at %x: %v", root, err) } for i, acc := range accounts { @@ -126,23 +126,37 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error) // checkStateIndex iterates over the entire state trie and checks that all required // database indexes have been generated by the synchronizer. -func checkStateIndex(db ethdb.Database, root common.Hash) error { +func checkStateIndex(db ethdb.Database, root common.Hash, parent common.Hash) error { // Remove any potentially cached data from the test state creation or previous checks trie.ClearGlobalCache() - // Create and iterate a state trie rooted in a sub-node if _, err := db.Get(root.Bytes()); err != nil { return err } state, err := New(root, db) if err != nil { - return err + return fmt.Errorf("failed to create state trie at %x: %v", root, err) } + // Gather all the indexes that should be present in the database + indexes := make(map[string]struct{}) for it := NewNodeIterator(state); it.Next(); { if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { - parentRef := trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()) - if _, err := db.Get(parentRef); err != nil { - return fmt.Errorf("parent reference lookup %x: %v", parentRef, err) + indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + if parent != (common.Hash{}) { + indexes[string(trie.ParentReferenceIndexKey(parent.Bytes(), root.Bytes()))] = struct{}{} + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + return fmt.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + return fmt.Errorf("index entry not reported %x", key) } } } @@ -150,26 +164,39 @@ func checkStateIndex(db ethdb.Database, root common.Hash) error { } // Tests that an empty state is not scheduled for syncing. -func TestEmptyStateSync(t *testing.T) { +func TestEmptyStateSyncDangling(t *testing.T) { testEmptyStateSync(t, common.Hash{}) } +func TestEmptyStateSyncRooted(t *testing.T) { testEmptyStateSync(t, common.BytesToHash([]byte{0x01})) } + +func testEmptyStateSync(t *testing.T, origin common.Hash) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") db, _ := ethdb.NewMemDatabase() - if req := NewStateSync(empty, db).Missing(1); len(req) != 0 { + if req := NewStateSync(empty, db, origin).Missing(1); len(req) != 0 { t.Errorf("content requested for empty state: %v", req) } } // Tests that given a root hash, a state can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go. -func TestIterativeStateSyncIndividual(t *testing.T) { testIterativeStateSync(t, 1) } -func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, 100) } +func TestIterativeStateSyncIndividualDangling(t *testing.T) { + testIterativeStateSync(t, 1, common.Hash{}) +} +func TestIterativeStateSyncIndividualRooted(t *testing.T) { + testIterativeStateSync(t, 1, common.BytesToHash([]byte{0x02})) +} +func TestIterativeStateSyncBatchedDangling(t *testing.T) { + testIterativeStateSync(t, 100, common.Hash{}) +} +func TestIterativeStateSyncBatchedRooted(t *testing.T) { + testIterativeStateSync(t, 100, common.BytesToHash([]byte{0x03})) +} -func testIterativeStateSync(t *testing.T, batch int) { +func testIterativeStateSync(t *testing.T, batch int, origin common.Hash) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState(nil) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewStateSync(srcRoot, dstDb) + sched := NewStateSync(srcRoot, dstDb, origin) queue := append([]common.Hash{}, sched.Missing(batch)...) for len(queue) > 0 { @@ -187,18 +214,25 @@ func testIterativeStateSync(t *testing.T, batch int) { queue = append(queue[:0], sched.Missing(batch)...) } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned, and the others sent only later. -func TestIterativeDelayedStateSync(t *testing.T) { +func TestIterativeDelayedStateSyncDangling(t *testing.T) { + testIterativeDelayedStateSync(t, common.Hash{}) +} +func TestIterativeDelayedStateSyncRooted(t *testing.T) { + testIterativeDelayedStateSync(t, common.BytesToHash([]byte{0x04})) +} + +func testIterativeDelayedStateSync(t *testing.T, origin common.Hash) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState(nil) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewStateSync(srcRoot, dstDb) + sched := NewStateSync(srcRoot, dstDb, origin) queue := append([]common.Hash{}, sched.Missing(0)...) for len(queue) > 0 { @@ -217,22 +251,32 @@ func TestIterativeDelayedStateSync(t *testing.T) { queue = append(queue[len(results):], sched.Missing(0)...) } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin) } // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go, however in a // random order. -func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) } -func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) } +func TestIterativeRandomStateSyncIndividualDangling(t *testing.T) { + testIterativeRandomStateSync(t, 1, common.Hash{}) +} +func TestIterativeRandomStateSyncIndividualRooted(t *testing.T) { + testIterativeRandomStateSync(t, 1, common.BytesToHash([]byte{0x05})) +} +func TestIterativeRandomStateSyncBatchedDangling(t *testing.T) { + testIterativeRandomStateSync(t, 100, common.Hash{}) +} +func TestIterativeRandomStateSyncBatchedRooted(t *testing.T) { + testIterativeRandomStateSync(t, 100, common.BytesToHash([]byte{0x06})) +} -func testIterativeRandomStateSync(t *testing.T, batch int) { +func testIterativeRandomStateSync(t *testing.T, batch int, origin common.Hash) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState(nil) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewStateSync(srcRoot, dstDb) + sched := NewStateSync(srcRoot, dstDb, origin) queue := make(map[common.Hash]struct{}) for _, hash := range sched.Missing(batch) { @@ -258,18 +302,25 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { } } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned (Even those randomly), others sent only later. -func TestIterativeRandomDelayedStateSync(t *testing.T) { +func TestIterativeRandomDelayedStateSyncDangling(t *testing.T) { + testIterativeRandomDelayedStateSync(t, common.Hash{}) +} +func TestIterativeRandomDelayedStateSyncRooted(t *testing.T) { + testIterativeRandomDelayedStateSync(t, common.BytesToHash([]byte{0x07})) +} + +func testIterativeRandomDelayedStateSync(t *testing.T, origin common.Hash) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState(nil) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewStateSync(srcRoot, dstDb) + sched := NewStateSync(srcRoot, dstDb, origin) queue := make(map[common.Hash]struct{}) for _, hash := range sched.Missing(0) { @@ -300,18 +351,18 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { } } // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, srcRoot, srcAccounts, origin) } // Tests that at any point in time during a sync, only complete sub-tries are in // the database. func TestIncompleteStateSync(t *testing.T) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + srcDb, srcRoot, srcAccounts := makeTestState(nil) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewStateSync(srcRoot, dstDb) + sched := NewStateSync(srcRoot, dstDb, common.Hash{}) added := []common.Hash{} queue := append([]common.Hash{}, sched.Missing(1)...) diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go index 00c42c5bc6..68d803e448 100644 --- a/core/types/derive_sha.go +++ b/core/types/derive_sha.go @@ -35,7 +35,7 @@ func DeriveSha(list DerivableList) common.Hash { for i := 0; i < list.Len(); i++ { keybuf.Reset() rlp.Encode(keybuf, uint(i)) - trie.Update(keybuf.Bytes(), list.GetRlp(i)) + trie.UpdateIndexed(keybuf.Bytes(), list.GetRlp(i), nil) } return trie.Hash() } diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 1e55560dbf..1f37ab6c00 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -356,7 +356,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { if q.mode == FastSync && header.Number.Uint64() == q.fastSyncPivot { // Pivoting point of the fast sync, retrieve the state tries q.stateSchedLock.Lock() - q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase) + q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase, header.Hash()) q.stateSchedLock.Unlock() } inserts = append(inserts, header) diff --git a/light/state_test.go b/light/state_test.go index 2c2e6daea1..418dee83b5 100644 --- a/light/state_test.go +++ b/light/state_test.go @@ -65,7 +65,7 @@ func makeTestState() (common.Hash, ethdb.Database) { so.Update() st.UpdateStateObject(so) } - root, _ := st.Commit() + root, _ := st.CommitIndexed(nil) return root, sdb } diff --git a/light/trie.go b/light/trie.go index e9c96ea48e..d99fe3ca4c 100644 --- a/light/trie.go +++ b/light/trie.go @@ -89,19 +89,20 @@ func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error) return } -// Update associates key with value in the trie. Subsequent calls to -// Get will return value. If value has length zero, any existing value -// is deleted from the trie and calls to Get will return nil. +// UpdateIndexed associates key with value in the trie. Subsequent calls to Get +// will return value. If value has length zero, any existing value is deleted +// from the trie and calls to Get will return nil. In addition, state trie index +// entries are also generated for all entities referencing the current node. // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) { +func (t *LightTrie) UpdateIndexed(ctx context.Context, key, value []byte, references [][]byte) (err error) { err = t.do(ctx, key, func() (err error) { if t.trie == nil { t.trie, err = trie.NewSecure(t.originalRoot, t.db) } if err == nil { - err = t.trie.TryUpdate(key, value) + err = t.trie.TryUpdateIndexed(key, value, references) } return }) diff --git a/miner/worker.go b/miner/worker.go index 1a411ae20b..648cbb57fd 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -284,7 +284,7 @@ func (self *worker) wait() { } go self.mux.Post(core.NewMinedBlockEvent{block}) } else { - work.state.Commit() + work.state.CommitIndexed([]common.Hash{block.Hash()}) parent := self.chain.GetBlock(block.ParentHash()) if parent == nil { glog.V(logger.Error).Infoln("Invalid block found during mining") diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 473bc3419f..9503497380 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -250,7 +250,7 @@ func (t *BlockTest) InsertPreState(db ethdb.Database, am *accounts.Manager) (*st } } - root, err := statedb.Commit() + root, err := statedb.CommitIndexed(nil) if err != nil { return nil, fmt.Errorf("error writing state: %v", err) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 352fe3570b..1e71fa93a5 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -201,7 +201,7 @@ func runStateTest(test VmTest) error { } } - root, _ := statedb.Commit() + root, _ := statedb.CommitIndexed(nil) if common.HexToHash(test.PostStateRoot) != root { return fmt.Errorf("Post state root error. Expected %s, got %x", test.PostStateRoot, root) } @@ -245,7 +245,7 @@ func RunState(statedb *state.StateDB, env, tx map[string]string) ([]byte, vm.Log if core.IsNonceErr(err) || core.IsInvalidTxErr(err) || core.IsGasLimitErr(err) { statedb.Set(snapshot) } - statedb.Commit() + statedb.CommitIndexed(nil) return ret, vmenv.state.Logs(), vmenv.Gas, err } diff --git a/trie/index_test.go b/trie/index_test.go index 1ecf8dd7bf..9baa894d3c 100644 --- a/trie/index_test.go +++ b/trie/index_test.go @@ -25,9 +25,12 @@ import ( ) // Tests that all index entries are stored in the database after a trie commit. -func TestTrieIndex(t *testing.T) { +func TestTrieIndexDangling(t *testing.T) { testTrieIndex(t, nil) } +func TestTrieIndexRooted(t *testing.T) { testTrieIndex(t, [][]byte{[]byte{0x00}, []byte{0x00, 0x01}}) } + +func testTrieIndex(t *testing.T, referrers [][]byte) { // Create some arbitrary test trie to iterate - db, trie, _ := makeTestTrie() + db, trie, _ := makeTestTrie(referrers) // Gather all the indexes that should be present in the database indexes := make(map[string]struct{}) @@ -36,6 +39,9 @@ func TestTrieIndex(t *testing.T) { indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} } } + for _, referrer := range referrers { + indexes[string(ParentReferenceIndexKey(referrer, trie.Hash().Bytes()))] = struct{}{} + } // Cross check the indexes and the database itself for index, _ := range indexes { if _, err := db.Get([]byte(index)); err != nil { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 815743bfdc..b095de855e 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -37,9 +37,9 @@ func TestIterator(t *testing.T) { v := make(map[string]bool) for _, val := range vals { v[val.k] = false - trie.Update([]byte(val.k), []byte(val.v)) + trie.UpdateIndexed([]byte(val.k), []byte(val.v), nil) } - trie.Commit() + trie.CommitIndexed(nil) it := NewIterator(trie) for it.Next() { @@ -56,7 +56,7 @@ func TestIterator(t *testing.T) { // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { // Create some arbitrary test trie to iterate - db, trie, _ := makeTestTrie() + db, trie, _ := makeTestTrie(nil) // Gather all the node hashes found by the iterator hashes := make(map[common.Hash]struct{}) diff --git a/trie/proof.go b/trie/proof.go index 3c49309026..0cfdc5bc72 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -62,7 +62,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. n, _ = t.hasher.replaceChildren(n, nil) - hn, _ := t.hasher.store(n, nil, false) + hn, _ := t.hasher.store(n, nil, false, nil) if _, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. diff --git a/trie/proof_test.go b/trie/proof_test.go index 6b5bef05c4..0530dff357 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -119,14 +119,14 @@ func randomTrie(n int) (*Trie, map[string]*kv) { for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false} - trie.Update(value.k, value.v) - trie.Update(value2.k, value2.v) + trie.UpdateIndexed(value.k, value.v, nil) + trie.UpdateIndexed(value2.k, value2.v, nil) vals[string(value.k)] = value vals[string(value2.k)] = value2 } for i := 0; i < n; i++ { value := &kv{randBytes(32), randBytes(20), false} - trie.Update(value.k, value.v) + trie.UpdateIndexed(value.k, value.v, nil) vals[string(value.k)] = value } return trie, vals diff --git a/trie/secure_trie.go b/trie/secure_trie.go index fb5f694d8e..aabffb40f7 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -79,36 +79,6 @@ func (t *SecureTrie) TryGet(key []byte) ([]byte, error) { return t.Trie.TryGet(t.hashKey(key)) } -// Update associates key with value in the trie. Subsequent calls to -// Get will return value. If value has length zero, any existing value -// is deleted from the trie and calls to Get will return nil. -// -// The value bytes must not be modified by the caller while they are -// stored in the trie. -func (t *SecureTrie) Update(key, value []byte) { - if err := t.TryUpdate(key, value); err != nil && glog.V(logger.Error) { - glog.Errorf("Unhandled trie error: %v", err) - } -} - -// TryUpdate associates key with value in the trie. Subsequent calls to -// Get will return value. If value has length zero, any existing value -// is deleted from the trie and calls to Get will return nil. -// -// The value bytes must not be modified by the caller while they are -// stored in the trie. -// -// If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryUpdate(key, value []byte) error { - hk := t.hashKey(key) - err := t.Trie.TryUpdate(hk, value) - if err != nil { - return err - } - t.Trie.db.Put(t.secKey(hk), key) - return nil -} - // UpdateIndexed is an extended version of Update, where state trie index entries // are also generated for all entities referencing the current node. // diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 13c6cd02e9..59ee267efc 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -45,7 +45,7 @@ func TestSecureDelete(t *testing.T) { } for _, val := range vals { if val.v != "" { - trie.Update([]byte(val.k), []byte(val.v)) + trie.UpdateIndexed([]byte(val.k), []byte(val.v), nil) } else { trie.Delete([]byte(val.k)) } @@ -59,7 +59,7 @@ func TestSecureDelete(t *testing.T) { func TestSecureGetKey(t *testing.T) { trie := newEmptySecure() - trie.Update([]byte("foo"), []byte("bar")) + trie.UpdateIndexed([]byte("foo"), []byte("bar"), nil) key := []byte("foo") value := []byte("bar") diff --git a/trie/sync.go b/trie/sync.go index c3cd36cf79..e9955cfb74 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -34,7 +34,7 @@ type request struct { depth int // Depth level within the trie the node is located to prioritise DFS deps int // Number of dependencies before allowed to commit this node - externals []common.Hash // External children of this node to index in addition to internal ones + referrers []common.Hash // External parents of this node to index in addition to internal ones callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch } @@ -61,13 +61,13 @@ type TrieSync struct { } // NewTrieSync creates a new trie data download scheduler. -func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync { +func NewTrieSync(root common.Hash, database ethdb.Database, parent common.Hash, callback TrieSyncLeafCallback) *TrieSync { ts := &TrieSync{ database: database, requests: make(map[common.Hash]*request), queue: prque.New(), } - ts.AddSubTrie(root, 0, common.Hash{}, callback) + ts.AddSubTrie(root, 0, parent, callback) return ts } @@ -91,13 +91,15 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.requests[parent] - if ancestor == nil { - panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) + if depth > 0 { + ancestor := s.requests[parent] + if ancestor == nil { + panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) + } + ancestor.deps++ + req.parents = append(req.parents, ancestor) } - ancestor.deps++ - ancestor.externals = append(ancestor.externals, root) - req.parents = append(req.parents, ancestor) + req.referrers = append(req.referrers, parent) } s.schedule(req) } @@ -121,13 +123,15 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.requests[parent] - if ancestor == nil { - panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) + if depth > 0 { + ancestor := s.requests[parent] + if ancestor == nil { + panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) + } + ancestor.deps++ + req.parents = append(req.parents, ancestor) } - ancestor.deps++ - ancestor.externals = append(ancestor.externals, hash) - req.parents = append(req.parents, ancestor) + req.referrers = append(req.referrers, parent) } s.schedule(req) } @@ -276,8 +280,8 @@ func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) { return err } } - for _, ext := range req.externals { - if err := storeParentReferenceEntry(req.hash[:], ext[:], batch); err != nil { + for _, referrer := range req.referrers { + if err := storeParentReferenceEntry(referrer[:], req.hash[:], batch); err != nil { return err } } diff --git a/trie/sync_test.go b/trie/sync_test.go index 6738427bfb..37ccc416a7 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -26,7 +26,7 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { +func makeTestTrie(referrers [][]byte) (ethdb.Database, *Trie, map[string][]byte) { // Create an empty trie db, _ := ethdb.NewMemDatabase() trie, _ := New(common.Hash{}, db) @@ -37,20 +37,20 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { // Map the same data under multiple keys key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i} content[string(key)] = val - trie.Update(key, val) + trie.UpdateIndexed(key, val, nil) key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i} content[string(key)] = val - trie.Update(key, val) + trie.UpdateIndexed(key, val, nil) // Add some other data to inflate th trie for j := byte(3); j < 13; j++ { key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i} content[string(key)] = val - trie.Update(key, val) + trie.UpdateIndexed(key, val, nil) } } - trie.Commit() + trie.CommitIndexed(referrers) // Remove any potentially cached data from the test trie creation globalCache.Clear() @@ -61,7 +61,7 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) { // checkTrieContents cross references a reconstructed trie with an expected data // content map. -func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) { +func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte, parent common.Hash) { // Remove any potentially cached data from the trie synchronisation globalCache.Clear() @@ -73,7 +73,7 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil { t.Fatalf("inconsistent trie at %x: %v", root, err) } - if err := checkTrieIndex(db, common.BytesToHash(root)); err != nil { + if err := checkTrieIndex(db, common.BytesToHash(root), parent); err != nil { t.Fatalf("index error at %x: %v", root, err) } for key, val := range content { @@ -106,20 +106,34 @@ func checkTrieConsistency(db Database, root common.Hash) (failure error) { // checkTrieIndex iterates over the entire state trie and checks that all required // database indexes have been generated by the synchronizer. -func checkTrieIndex(db Database, root common.Hash) error { +func checkTrieIndex(db Database, root common.Hash, parent common.Hash) error { // Remove any potentially cached data from the test trie creation or previous checks globalCache.Clear() - // Create and iterate a trie trie, err := New(root, db) if err != nil { return err } + // Gather all the indexes that should be present in the database + indexes := make(map[string]struct{}) for it := NewNodeIterator(trie); it.Next(); { if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { - parentRef := ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()) - if _, err := db.Get(parentRef); err != nil { - return fmt.Errorf("parent reference lookup %x: %v", parentRef, err) + indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + if parent != (common.Hash{}) { + indexes[string(ParentReferenceIndexKey(parent.Bytes(), trie.Hash().Bytes()))] = struct{}{} + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + return fmt.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + return fmt.Errorf("index entry not reported %x", key) } } } @@ -127,13 +141,18 @@ func checkTrieIndex(db Database, root common.Hash) error { } // Tests that an empty trie is not scheduled for syncing. -func TestEmptyTrieSync(t *testing.T) { +func TestEmptyTrieSyncDangling(t *testing.T) { testEmptyTrieSync(t, common.Hash{}) } +func TestEmptyTrieSyncRooted(t *testing.T) { + testEmptyTrieSync(t, common.BytesToHash([]byte{0x01})) +} + +func testEmptyTrieSync(t *testing.T, origin common.Hash) { emptyA, _ := New(common.Hash{}, nil) emptyB, _ := New(emptyRoot, nil) for i, trie := range []*Trie{emptyA, emptyB} { db, _ := ethdb.NewMemDatabase() - if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 { + if req := NewTrieSync(common.BytesToHash(trie.Root()), db, origin, nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -141,16 +160,26 @@ func TestEmptyTrieSync(t *testing.T) { // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go. -func TestIterativeTrieSyncIndividual(t *testing.T) { testIterativeTrieSync(t, 1) } -func TestIterativeTrieSyncBatched(t *testing.T) { testIterativeTrieSync(t, 100) } +func TestIterativeTrieSyncIndividualDangling(t *testing.T) { + testIterativeTrieSync(t, 1, common.Hash{}) +} +func TestIterativeTrieSyncIndividualRooted(t *testing.T) { + testIterativeTrieSync(t, 1, common.BytesToHash([]byte{0x02})) +} +func TestIterativeTrieSyncBatchedDangling(t *testing.T) { + testIterativeTrieSync(t, 100, common.Hash{}) +} +func TestIterativeTrieSyncBatchedRooted(t *testing.T) { + testIterativeTrieSync(t, 100, common.BytesToHash([]byte{0x03})) +} -func testIterativeTrieSync(t *testing.T, batch int) { +func testIterativeTrieSync(t *testing.T, batch int, origin common.Hash) { // Create a random trie to copy - srcDb, srcTrie, srcData := makeTestTrie() + srcDb, srcTrie, srcData := makeTestTrie(nil) // Create a destination trie and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil) queue := append([]common.Hash{}, sched.Missing(batch)...) for len(queue) > 0 { @@ -168,18 +197,25 @@ func testIterativeTrieSync(t *testing.T, batch int) { queue = append(queue[:0], sched.Missing(batch)...) } // Cross check that the two tries are in sync - checkTrieContents(t, dstDb, srcTrie.Root(), srcData) + checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned, and the others sent only later. -func TestIterativeDelayedTrieSync(t *testing.T) { +func TestIterativeDelayedTrieSyncDangling(t *testing.T) { + testIterativeDelayedTrieSync(t, common.Hash{}) +} +func TestIterativeDelayedTrieSyncRooted(t *testing.T) { + testIterativeDelayedTrieSync(t, common.BytesToHash([]byte{0x04})) +} + +func testIterativeDelayedTrieSync(t *testing.T, origin common.Hash) { // Create a random trie to copy - srcDb, srcTrie, srcData := makeTestTrie() + srcDb, srcTrie, srcData := makeTestTrie(nil) // Create a destination trie and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil) queue := append([]common.Hash{}, sched.Missing(10000)...) for len(queue) > 0 { @@ -198,22 +234,32 @@ func TestIterativeDelayedTrieSync(t *testing.T) { queue = append(queue[len(results):], sched.Missing(10000)...) } // Cross check that the two tries are in sync - checkTrieContents(t, dstDb, srcTrie.Root(), srcData) + checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin) } // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go, however in a // random order. -func TestIterativeRandomTrieSyncIndividual(t *testing.T) { testIterativeRandomTrieSync(t, 1) } -func TestIterativeRandomTrieSyncBatched(t *testing.T) { testIterativeRandomTrieSync(t, 100) } +func TestIterativeRandomTrieSyncIndividualDangling(t *testing.T) { + testIterativeRandomTrieSync(t, 1, common.Hash{}) +} +func TestIterativeRandomTrieSyncIndividualRooted(t *testing.T) { + testIterativeRandomTrieSync(t, 1, common.BytesToHash([]byte{0x05})) +} +func TestIterativeRandomTrieSyncBatchedDangling(t *testing.T) { + testIterativeRandomTrieSync(t, 100, common.Hash{}) +} +func TestIterativeRandomTrieSyncBatchedlRooted(t *testing.T) { + testIterativeRandomTrieSync(t, 100, common.BytesToHash([]byte{0x06})) +} -func testIterativeRandomTrieSync(t *testing.T, batch int) { +func testIterativeRandomTrieSync(t *testing.T, batch int, origin common.Hash) { // Create a random trie to copy - srcDb, srcTrie, srcData := makeTestTrie() + srcDb, srcTrie, srcData := makeTestTrie(nil) // Create a destination trie and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil) queue := make(map[common.Hash]struct{}) for _, hash := range sched.Missing(batch) { @@ -239,18 +285,25 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { } } // Cross check that the two tries are in sync - checkTrieContents(t, dstDb, srcTrie.Root(), srcData) + checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned (Even those randomly), others sent only later. -func TestIterativeRandomDelayedTrieSync(t *testing.T) { +func TestIterativeRandomDelayedTrieSyncDangling(t *testing.T) { + testIterativeRandomDelayedTrieSync(t, common.Hash{}) +} +func TestIterativeRandomDelayedTrieSyncRooted(t *testing.T) { + testIterativeRandomDelayedTrieSync(t, common.BytesToHash([]byte{0x07})) +} + +func testIterativeRandomDelayedTrieSync(t *testing.T, origin common.Hash) { // Create a random trie to copy - srcDb, srcTrie, srcData := makeTestTrie() + srcDb, srcTrie, srcData := makeTestTrie(nil) // Create a destination trie and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil) queue := make(map[common.Hash]struct{}) for _, hash := range sched.Missing(10000) { @@ -282,18 +335,25 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { } } // Cross check that the two tries are in sync - checkTrieContents(t, dstDb, srcTrie.Root(), srcData) + checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin) } // Tests that a trie sync will not request nodes multiple times, even if they // have such references. -func TestDuplicateAvoidanceTrieSync(t *testing.T) { +func TestDuplicateAvoidanceTrieSyncDangling(t *testing.T) { + testDuplicateAvoidanceTrieSync(t, common.Hash{}) +} +func TestDuplicateAvoidanceTrieSyncRooted(t *testing.T) { + testDuplicateAvoidanceTrieSync(t, common.BytesToHash([]byte{0x08})) +} + +func testDuplicateAvoidanceTrieSync(t *testing.T, origin common.Hash) { // Create a random trie to copy - srcDb, srcTrie, srcData := makeTestTrie() + srcDb, srcTrie, srcData := makeTestTrie(nil) // Create a destination trie and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, origin, nil) queue := append([]common.Hash{}, sched.Missing(0)...) requested := make(map[common.Hash]struct{}) @@ -318,18 +378,18 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { queue = append(queue[:0], sched.Missing(0)...) } // Cross check that the two tries are in sync - checkTrieContents(t, dstDb, srcTrie.Root(), srcData) + checkTrieContents(t, dstDb, srcTrie.Root(), srcData, origin) } // Tests that at any point in time during a sync, only complete sub-tries are in // the database. func TestIncompleteTrieSync(t *testing.T) { // Create a random trie to copy - srcDb, srcTrie, _ := makeTestTrie() + srcDb, srcTrie, _ := makeTestTrie(nil) // Create a destination trie and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() - sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil) + sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, common.Hash{}, nil) added := []common.Hash{} queue := append([]common.Hash{}, sched.Missing(1)...) diff --git a/trie/trie.go b/trie/trie.go index 88b32aa45a..97cf624075 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -146,50 +146,30 @@ func (t *Trie) TryGet(key []byte) ([]byte, error) { return tn.(valueNode).Value, nil } -// Update associates key with value in the trie. Subsequent calls to -// Get will return value. If value has length zero, any existing value -// is deleted from the trie and calls to Get will return nil. +// UpdateIndexed associates key with value in the trie. Subsequent calls to Get +// will return value. If value has length zero, any existing value is deleted +// from the trie and calls to Get will return nil. In addition, state trie index +// entries are also generated for all entities referencing the current node. // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *Trie) Update(key, value []byte) { - if err := t.TryUpdate(key, value); err != nil && glog.V(logger.Error) { +func (t *Trie) UpdateIndexed(key, value []byte, references [][]byte) { + if err := t.TryUpdateIndexed(key, value, references); err != nil && glog.V(logger.Error) { glog.Errorf("Unhandled trie error: %v", err) } } -// TryUpdate associates key with value in the trie. Subsequent calls to -// Get will return value. If value has length zero, any existing value -// is deleted from the trie and calls to Get will return nil. -// -// The value bytes must not be modified by the caller while they are -// stored in the trie. -// -// If a node was not found in the database, a MissingNodeError is returned. -func (t *Trie) TryUpdate(key, value []byte) error { - return t.tryUpdateIndexed(key, value, nil) -} - -// UpdateIndexed is an extended version of Update, where state trie index entries -// are also generated for all entities referencing the current node. -// -// The value bytes must not be modified by the caller while they are -// stored in the trie. -func (t *Trie) UpdateIndexed(key, value []byte, refs [][]byte) { - if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) { - glog.Errorf("Unhandled trie error: %v", err) - } -} - -// TryUpdateIndexed is an extended version of Update, where state trie index +// TryUpdateIndexed associates key with value in the trie. Subsequent calls to +// Get will return value. If value has length zero, any existing value is deleted +// from the trie and calls to Get will return nil. In addition, state trie index // entries are also generated for all entities referencing the current node. // // The value bytes must not be modified by the caller while they are // stored in the trie. // // If a node was not found in the database, a MissingNodeError is returned. -func (t *Trie) TryUpdateIndexed(key, value []byte, refs [][]byte) error { - return t.tryUpdateIndexed(key, value, refs) +func (t *Trie) TryUpdateIndexed(key, value []byte, references [][]byte) error { + return t.tryUpdateIndexed(key, value, references) } func (t *Trie) tryUpdateIndexed(key, value []byte, refs [][]byte) error { @@ -439,31 +419,32 @@ func (t *Trie) Root() []byte { return t.Hash().Bytes() } // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { - root, _ := t.hashRoot(nil) + root, _ := t.hashRoot(nil, nil) return common.BytesToHash(root.(hashNode)) } -// Commit writes all nodes to the trie's database. -// Nodes are stored with their sha3 hash as the key. +// CommitIndexed writes all nodes to the trie's database, and if any references +// were specified, adds those to the trie index as well. Nodes are stored with +// their sha3 hash as the key. // -// Committing flushes nodes from memory. -// Subsequent Get calls will load nodes from the database. -func (t *Trie) Commit() (root common.Hash, err error) { +// Committing flushes nodes from memory. Subsequent Get calls will load nodes +// from the database. +func (t *Trie) CommitIndexed(referrers [][]byte) (root common.Hash, err error) { if t.db == nil { panic("Commit called on trie with nil database") } - return t.CommitTo(t.db) + return t.CommitToIndexed(t.db, referrers) } -// CommitTo writes all nodes to the given database. -// Nodes are stored with their sha3 hash as the key. +// CommitToIndexed writes all nodes to the given database, and if any references +// were specified, adds those to the trie index as well. Nodes are stored with +// their sha3 hash as the key. // -// Committing flushes nodes from memory. Subsequent Get calls will -// load nodes from the trie's database. Calling code must ensure that -// the changes made to db are written back to the trie's attached -// database before using the trie. -func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) { - n, err := t.hashRoot(db) +// Committing flushes nodes from memory. Subsequent Get calls will load nodes +// from the trie's database. Calling code must ensure that the changes made to +// db are written back to the trie's attached database before using the trie. +func (t *Trie) CommitToIndexed(db DatabaseWriter, referrers [][]byte) (root common.Hash, err error) { + n, err := t.hashRoot(db, referrers) if err != nil { return (common.Hash{}), err } @@ -471,14 +452,14 @@ func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) { return common.BytesToHash(n.(hashNode)), nil } -func (t *Trie) hashRoot(db DatabaseWriter) (node, error) { +func (t *Trie) hashRoot(db DatabaseWriter, referrers [][]byte) (node, error) { if t.root == nil { return hashNode(emptyRoot.Bytes()), nil } if t.hasher == nil { t.hasher = newHasher() } - return t.hasher.hash(t.root, db, true) + return t.hasher.hash(t.root, db, true, referrers) } type hasher struct { @@ -490,12 +471,12 @@ func newHasher() *hasher { return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()} } -func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, error) { +func (h *hasher) hash(n node, db DatabaseWriter, force bool, referrers [][]byte) (node, error) { hashed, err := h.replaceChildren(n, db) if err != nil { return hashNode{}, err } - if n, err = h.store(hashed, db, force); err != nil { + if n, err = h.store(hashed, db, force, referrers); err != nil { return hashNode{}, err } return n, nil @@ -509,7 +490,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { case shortNode: n.Key = compactEncode(n.Key) if _, ok := n.Val.(valueNode); !ok { - if n.Val, err = h.hash(n.Val, db, false); err != nil { + if n.Val, err = h.hash(n.Val, db, false, nil); err != nil { return n, err } } @@ -521,7 +502,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { case fullNode: for i := 0; i < 16; i++ { if n[i] != nil { - if n[i], err = h.hash(n[i], db, false); err != nil { + if n[i], err = h.hash(n[i], db, false, nil); err != nil { return n, err } } else { @@ -538,7 +519,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { } } -func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { +func (h *hasher) store(n node, db DatabaseWriter, force bool, referrers [][]byte) (node, error) { // Don't store hashes or empty nodes. if _, isHash := n.(hashNode); n == nil || isHash { return n, nil @@ -559,6 +540,11 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { if err := storeParentReferences(key, n, db); err != nil { return key, err } + for _, referrer := range referrers { + if err := storeParentReferenceEntry(referrer, key, db); err != nil { + return key, err + } + } if err := db.Put(key, h.tmp.Bytes()); err != nil { return key, err } diff --git a/trie/trie_test.go b/trie/trie_test.go index 35d043cdf1..244f5d28d5 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -54,7 +54,7 @@ func TestNull(t *testing.T) { var trie Trie key := make([]byte, 32) value := common.FromHex("0x823140710bf13990e4500136726d8b55") - trie.Update(key, value) + trie.UpdateIndexed(key, value, nil) value = trie.Get(key) } @@ -74,7 +74,7 @@ func TestMissingNode(t *testing.T) { trie, _ := New(common.Hash{}, db) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _ := trie.Commit() + root, _ := trie.CommitIndexed(nil) ClearGlobalCache() @@ -97,7 +97,7 @@ func TestMissingNode(t *testing.T) { } trie, _ = New(root, db) - err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) + err = trie.TryUpdateIndexed([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"), nil) if err != nil { t.Errorf("Unexpected error: %v", err) } @@ -130,7 +130,7 @@ func TestMissingNode(t *testing.T) { } trie, _ = New(root, db) - err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) + err = trie.TryUpdateIndexed([]byte("120099"), []byte("zxcv"), nil) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } @@ -159,7 +159,7 @@ func TestInsert(t *testing.T) { updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, err := trie.Commit() + root, err := trie.CommitIndexed(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -188,7 +188,7 @@ func TestGet(t *testing.T) { if i == 1 { return } - trie.Commit() + trie.CommitIndexed(nil) } } @@ -257,7 +257,7 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, err := trie.Commit() + exp, err := trie.CommitIndexed(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -272,7 +272,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, err := trie2.Commit() + hash, err := trie2.CommitIndexed(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -304,7 +304,7 @@ func paranoiaCheck(t1 *Trie) (bool, *Trie) { t2 := new(Trie) it := NewIterator(t1) for it.Next() { - t2.Update(it.Key, it.Value) + t2.UpdateIndexed(it.Key, it.Value, nil) } return t2.Hash() == t1.Hash(), t2 } @@ -327,7 +327,7 @@ func TestParanoia(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - trie.Commit() + trie.CommitIndexed(nil) ok, t2 := paranoiaCheck(trie) if !ok { @@ -347,7 +347,7 @@ func TestOutput(t *testing.T) { fmt.Println("############################## FULL ################################") fmt.Println(trie.root) - trie.Commit() + trie.CommitIndexed(nil) fmt.Println("############################## SMALL ################################") trie2, _ := New(trie.Hash(), trie.db) getString(trie2, base+"20") @@ -356,8 +356,8 @@ func TestOutput(t *testing.T) { func TestLargeValue(t *testing.T) { trie := newEmpty() - trie.Update([]byte("key1"), []byte{99, 99, 99, 99}) - trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32)) + trie.UpdateIndexed([]byte("key1"), []byte{99, 99, 99, 99}, nil) + trie.UpdateIndexed([]byte("key2"), bytes.Repeat([]byte{1}, 32), nil) trie.Hash() } @@ -374,8 +374,8 @@ func TestLargeData(t *testing.T) { for i := byte(0); i < 255; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false} - trie.Update(value.k, value.v) - trie.Update(value2.k, value2.v) + trie.UpdateIndexed(value.k, value.v, nil) + trie.UpdateIndexed(value2.k, value2.v, nil) vals[string(value.k)] = value vals[string(value2.k)] = value2 } @@ -419,11 +419,11 @@ func benchGet(b *testing.B, commit bool) { k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) - trie.Update(k, k) + trie.UpdateIndexed(k, k, nil) } binary.LittleEndian.PutUint64(k, benchElemCount/2) if commit { - trie.Commit() + trie.CommitIndexed(nil) } b.ResetTimer() @@ -437,7 +437,7 @@ func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { k := make([]byte, 32) for i := 0; i < b.N; i++ { e.PutUint64(k, uint64(i)) - trie.Update(k, k) + trie.UpdateIndexed(k, k, nil) } return trie } @@ -447,7 +447,7 @@ func benchHash(b *testing.B, e binary.ByteOrder) { k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { e.PutUint64(k, uint64(i)) - trie.Update(k, k) + trie.UpdateIndexed(k, k, nil) } b.ResetTimer() @@ -473,7 +473,7 @@ func getString(trie *Trie, k string) []byte { } func updateString(trie *Trie, k, v string) { - trie.Update([]byte(k), []byte(v)) + trie.UpdateIndexed([]byte(k), []byte(v), nil) } func deleteString(trie *Trie, k string) { From 70978907bcefd7a35b8a2d212329b1e06c53f522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 13 Jan 2016 16:57:00 +0200 Subject: [PATCH 7/8] core/state, trie: add pre-order iterator hook, fix missing indices --- core/state/iterator.go | 33 +++++++++++++++----- core/state/iterator_test.go | 60 +++++++++++++++++++++++++++++++++++++ core/state/sync_test.go | 26 ++++++++-------- trie/iterator.go | 17 ++++++++++- trie/iterator_test.go | 52 ++++++++++++++++++++++++++++++++ trie/sync.go | 37 ++++++++++++++++------- 6 files changed, 194 insertions(+), 31 deletions(-) diff --git a/core/state/iterator.go b/core/state/iterator.go index a051e5e01a..84259ac512 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -26,6 +26,11 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// NodeIteratorSubtreeCallback is a callback that is invoked by the trie node +// iterator whenever it descends into a new subtree, giving the possibility of +// skipping the branch if it would be non-useful (e.g. already processed before). +type NodeIteratorSubtreeCallback func(hash, parent common.Hash) bool + // NodeIterator is an iterator to traverse the entire state trie post-order, // including all of the contract code and contract state tries. type NodeIterator struct { @@ -38,6 +43,14 @@ type NodeIterator struct { codeHash common.Hash // Hash of the contract source code code []byte // Source code associated with a contract + // Callback method enabling the user to control subtree descent. + // + // Note: This hook is invoked pre-order to allow the user to cancel the traversal + // of a subtree, but the iterator itself is post-order. This means that the hook + // may be invoked multiple times during a single iteration step (when descending) + // or never in multiple iteration steps (when ascending). + PreOrderHook NodeIteratorSubtreeCallback + Hash common.Hash // Hash of the current entry being iterated (nil if not standalone) Entry interface{} // Current state entry being iterated (internal representation) Parent common.Hash // Hash of the first full ancestor node (nil if current is the root) @@ -66,6 +79,7 @@ func (it *NodeIterator) step() { // Initialize the iterator if we've just started if it.stateIt == nil { it.stateIt = trie.NewNodeIterator(it.state.trie.Trie) + it.stateIt.PreOrderHook = trie.NodeIteratorSubtreeCallback(it.PreOrderHook) } // If we had data nodes previously, we surely have at least state nodes if it.dataIt != nil { @@ -99,22 +113,25 @@ func (it *NodeIterator) step() { if err != nil { panic(err) } + it.accountHash = it.stateIt.Parent + + if bytes.Compare(account.CodeHash, emptyCodeHash) != 0 { + if hash := common.BytesToHash(account.CodeHash); it.PreOrderHook == nil || it.PreOrderHook(hash, it.accountHash) { + it.codeHash = hash + if it.code, err = it.state.db.Get(account.CodeHash); err != nil { + panic(fmt.Sprintf("code %x: %v", account.CodeHash, err)) + } + } + } dataTrie, err := trie.New(account.Root, it.state.db) if err != nil { panic(err) } it.dataIt = trie.NewNodeIterator(dataTrie) + it.dataIt.PreOrderHook = trie.NodeIteratorSubtreeCallback(it.PreOrderHook) if !it.dataIt.Next() { it.dataIt = nil } - if bytes.Compare(account.CodeHash, emptyCodeHash) != 0 { - it.codeHash = common.BytesToHash(account.CodeHash) - it.code, err = it.state.db.Get(account.CodeHash) - if err != nil { - panic(fmt.Sprintf("code %x: %v", account.CodeHash, err)) - } - } - it.accountHash = it.stateIt.Parent } // retrieve pulls and caches the current state entry the iterator is traversing. diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index d44e15be7d..0e7aba78c6 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -59,3 +59,63 @@ func TestNodeIteratorCoverage(t *testing.T) { } } } + +// Tests that the node iterator hook is invoked for all the nodes of the state, +// also testing that the iterator indeed avoids visiting aborted branches. +func TestNodeIteratorHookCoverageFull(t *testing.T) { testNodeIteratorHookCoverage(t, false) } +func TestNodeIteratorHookCoverageDedup(t *testing.T) { testNodeIteratorHookCoverage(t, true) } + +func testNodeIteratorHookCoverage(t *testing.T, dedup bool) { + // Create some arbitrary test state to iterate + db, root, _ := makeTestState(nil) + + state, err := New(root, db) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", root, err) + } + // Gather all the node hashes found by the iterator + hashes := make(map[common.Hash]int) + + it := NewNodeIterator(state) + it.PreOrderHook = func(hash, parent common.Hash) bool { + if !dedup { + return true + } + return hashes[hash] == 0 + } + for it.Next() { + if it.Hash != (common.Hash{}) { + hashes[it.Hash]++ + } + } + // Cross check the hashes and the database itself + for hash, _ := range hashes { + if _, err := db.Get(hash.Bytes()); err != nil { + t.Errorf("failed to retrieve reported node %x: %v", hash, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, []byte("secure-key-")) { + continue + } + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + continue + } + if _, ok := hashes[common.BytesToHash(key)]; !ok { + t.Errorf("state entry not reported %x", key) + } + } + // Check whether duplicates were avoided or not + duplicates := 0 + for hash, count := range hashes { + if count > 1 { + duplicates++ + if dedup { + t.Errorf("duplicate (%d) iteration: %x", count, hash) + } + } + } + if !dedup && duplicates == 0 { + t.Errorf("iterator didn't traverse common subtrees") + } +} diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 30cd70eec6..e975a317ec 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -44,22 +44,24 @@ func makeTestState(referrers []common.Hash) (ethdb.Database, common.Hash, []*tes // Fill it with some arbitrary data accounts := []*testAccount{} - for i := byte(0); i < 96; i++ { - obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i})) - acc := &testAccount{address: common.BytesToAddress([]byte{i})} + for i := byte(0); i < 1; i++ { + for j := byte(0); j < 2; j++ { + obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i, j})) + acc := &testAccount{address: common.BytesToAddress([]byte{i, j})} - obj.AddBalance(big.NewInt(int64(11 * i))) - acc.balance = big.NewInt(int64(11 * i)) + obj.AddBalance(big.NewInt(int64(11 * i))) + acc.balance = big.NewInt(int64(11 * i)) - obj.SetNonce(uint64(42 * i)) - acc.nonce = uint64(42 * i) + obj.SetNonce(uint64(42 * i)) + acc.nonce = uint64(42 * i) - if i%3 == 0 { - obj.SetCode([]byte{i, i, i, i, i}) - acc.code = []byte{i, i, i, i, i} + if i%3 == 0 { + obj.SetCode([]byte{i, i, i, i, i}) + acc.code = []byte{i, i, i, i, i} + } + state.UpdateStateObject(obj) + accounts = append(accounts, acc) } - state.UpdateStateObject(obj) - accounts = append(accounts, acc) } root, _ := state.CommitIndexed(referrers) diff --git a/trie/iterator.go b/trie/iterator.go index 59ac3c5182..df9efece7f 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -159,11 +159,24 @@ type nodeIteratorState struct { child int // Child to be processed next } +// NodeIteratorSubtreeCallback is a callback that is invoked by the trie node +// iterator whenever it descends into a new subtree, giving the possibility of +// skipping the branch if it would be non-useful (e.g. already processed before). +type NodeIteratorSubtreeCallback func(hash, parent common.Hash) bool + // NodeIterator is an iterator to traverse the trie post-order. type NodeIterator struct { trie *Trie // Trie being iterated stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state + // Callback method enabling the user to control subtree descent. + // + // Note: This hook is invoked pre-order to allow the user to cancel the traversal + // of a subtree, but the iterator itself is post-order. This means that the hook + // may be invoked multiple times during a single iteration step (when descending) + // or never in multiple iteration steps (when ascending). + PreOrderHook NodeIteratorSubtreeCallback + Hash common.Hash // Hash of the current node being iterated (nil if not standalone) Node node // Current node being iterated (internal representation) Parent common.Hash // Hash of the first full ancestor node (nil if current is the root) @@ -241,7 +254,9 @@ func (it *NodeIterator) step() { if err != nil { panic(err) } - it.stack = append(it.stack, &nodeIteratorState{hash: common.BytesToHash(hash), node: node, parent: ancestor, child: -1}) + if hash := common.BytesToHash(hash); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) { + it.stack = append(it.stack, &nodeIteratorState{hash: hash, node: node, parent: ancestor, child: -1}) + } } else { break } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index b095de855e..dadee3fdc3 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -79,3 +79,55 @@ func TestNodeIteratorCoverage(t *testing.T) { } } } + +// Tests that the node iterator hook is invoked for all the nodes of the trie, +// also testing that the iterator indeed avoids visiting aborted branches. +func TestNodeIteratorHookCoverageFull(t *testing.T) { testNodeIteratorHookCoverage(t, false) } +func TestNodeIteratorHookCoverageDedup(t *testing.T) { testNodeIteratorHookCoverage(t, true) } + +func testNodeIteratorHookCoverage(t *testing.T, dedup bool) { + // Create some arbitrary test trie to iterate + db, trie, _ := makeTestTrie(nil) + + // Gather all the node hashes found by the iterator + hashes := make(map[common.Hash]int) + + it := NewNodeIterator(trie) + it.PreOrderHook = func(hash, parent common.Hash) bool { + if !dedup { + return true + } + return hashes[hash] == 0 + } + for it.Next() { + if it.Hash != (common.Hash{}) { + hashes[it.Hash]++ + } + } + // Cross check the hashes and the database itself + for hash, _ := range hashes { + if _, err := db.Get(hash.Bytes()); err != nil { + t.Errorf("failed to retrieve reported node %x: %v", hash, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if len(key) == common.HashLength { + if _, ok := hashes[common.BytesToHash(key)]; !ok { + t.Errorf("state entry not reported %x", key) + } + } + } + // Check whether duplicates were avoided or not + duplicates := 0 + for hash, count := range hashes { + if count > 1 { + duplicates++ + if dedup { + t.Errorf("duplicate (%d) iteration: %x", count, hash) + } + } + } + if !dedup && duplicates == 0 { + t.Errorf("iterator didn't traverse common subtrees") + } +} diff --git a/trie/sync.go b/trie/sync.go index e9955cfb74..0daf0ecb92 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -34,11 +34,19 @@ type request struct { depth int // Depth level within the trie the node is located to prioritise DFS deps int // Number of dependencies before allowed to commit this node - referrers []common.Hash // External parents of this node to index in addition to internal ones + referrers map[common.Hash]struct{} // External parents of this node to index in addition to internal ones callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch } +// referrer adds a new referring parent to a sync node request. +func (r *request) referrer(hash common.Hash) { + if r.referrers == nil { + r.referrers = make(map[common.Hash]struct{}) + } + r.referrers[hash] = struct{}{} +} + // SyncResult is a simple list to return missing nodes along with their request // hashes. type SyncResult struct { @@ -72,14 +80,14 @@ func NewTrieSync(root common.Hash, database ethdb.Database, parent common.Hash, } // AddSubTrie registers a new trie to the sync code, rooted at the designated parent. -func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) { +func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) error { // Short circuit if the trie is empty or already known if root == emptyRoot { - return + return nil } blob, _ := s.database.Get(root.Bytes()) if local, err := decodeNode(blob); local != nil && err == nil { - return + return storeParentReferenceEntry(parent.Bytes(), root.Bytes(), s.database) } // Assemble the new sub-trie sync request node := node(hashNode(root.Bytes())) @@ -99,22 +107,23 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c ancestor.deps++ req.parents = append(req.parents, ancestor) } - req.referrers = append(req.referrers, parent) + req.referrer(parent) } s.schedule(req) + return nil } // AddRawEntry schedules the direct retrieval of a state entry that should not be // interpreted as a trie node, but rather accepted and stored into the database // as is. This method's goal is to support misc state metadata retrievals (e.g. // contract code). -func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) { +func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) error { // Short circuit if the entry is empty or already known if hash == emptyState { - return + return nil } if blob, _ := s.database.Get(hash.Bytes()); blob != nil { - return + return storeParentReferenceEntry(parent.Bytes(), hash.Bytes(), s.database) } // Assemble the new sub-trie sync request req := &request{ @@ -131,9 +140,10 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) ancestor.deps++ req.parents = append(req.parents, ancestor) } - req.referrers = append(req.referrers, parent) + req.referrer(parent) } s.schedule(req) + return nil } // Missing retrieves the known missing nodes from the trie for retrieval. @@ -196,6 +206,13 @@ func (s *TrieSync) schedule(req *request) { // If we're already requesting this node, add a new reference and stop if old, ok := s.requests[req.hash]; ok { old.parents = append(old.parents, req.parents...) + + for hash, _ := range req.referrers { + old.referrer(hash) + } + for _, parent := range req.parents { + old.referrer(parent.hash) + } return } // Schedule the request for future retrieval @@ -280,7 +297,7 @@ func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) { return err } } - for _, referrer := range req.referrers { + for referrer, _ := range req.referrers { if err := storeParentReferenceEntry(referrer[:], req.hash[:], batch); err != nil { return err } From 1d1040a4a19b4892fda91452596616b918249356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 14 Jan 2016 16:54:50 +0200 Subject: [PATCH 8/8] cmd, eth, trie: implement state database upgrade-indexing --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 6 +++ eth/backend.go | 77 ++++++++++++++++++++++++++++++++++-- eth/downloader/downloader.go | 2 + trie/iterator.go | 12 +++--- 6 files changed, 89 insertions(+), 10 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index f2bb275520..abf0303484 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -291,6 +291,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.FastSyncFlag, utils.CacheFlag, utils.LightKDFFlag, + utils.StateRetentionFlag, utils.JSpathFlag, utils.ListenPortFlag, utils.MaxPeersFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 7a6ff704c4..7c02544f41 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -69,6 +69,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.GenesisFileFlag, utils.IdentityFlag, utils.FastSyncFlag, + utils.StateRetentionFlag, utils.LightKDFFlag, utils.CacheFlag, utils.BlockchainVersionFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 63efa08ee3..6246d35b99 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -163,6 +163,11 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } + StateRetentionFlag = cli.IntFlag{ + Name: "retain", + Usage: "Number of recent state tries to retain (0 = keep all history)", + Value: 5000, + } // Miner settings // TODO: refactor CPU vs GPU mining flags MiningEnabledFlag = cli.BoolFlag{ @@ -613,6 +618,7 @@ func MakeSystemNode(name, version string, extra []byte, ctx *cli.Context) *node. ethConf := ð.Config{ Genesis: MakeGenesisBlock(ctx), FastSync: ctx.GlobalBool(FastSyncFlag.Name), + StateRetention: ctx.GlobalInt(StateRetentionFlag.Name), BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name), DatabaseCache: ctx.GlobalInt(CacheFlag.Name), NetworkId: ctx.GlobalInt(NetworkIdFlag.Name), diff --git a/eth/backend.go b/eth/backend.go index abd1214caa..6e130a9c89 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/compiler" "github.com/ethereum/go-ethereum/common/httpclient" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" @@ -45,6 +46,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" rpc "github.com/ethereum/go-ethereum/rpc/v2" + "github.com/ethereum/go-ethereum/trie" ) const ( @@ -61,9 +63,10 @@ var ( ) type Config struct { - NetworkId int // Network ID to use for selecting peers to connect to - Genesis string // Genesis JSON to seed the chain database with - FastSync bool // Enables the state download based fast synchronisation algorithm + NetworkId int // Network ID to use for selecting peers to connect to + Genesis string // Genesis JSON to seed the chain database with + FastSync bool // Enables the state download based fast synchronisation algorithm + StateRetention int // Number of recent state tries to retain pruning BlockChainVersion int SkipBcVersionCheck bool // e.g. blockchain export @@ -136,6 +139,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { const dbCount = 3 ethdb.OpenFileLimit = 128 / (dbCount + 1) + // Sanity check that the user doesn't throw away more data than what would hurt the network + if config.StateRetention != 0 && config.StateRetention < downloader.FsStateRetention { + return nil, fmt.Errorf("configured state retention (%d) too low (< %d) for network security", config.StateRetention, downloader.FsStateRetention) + } // Open the chain database and perform any upgrades needed chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache) if err != nil { @@ -150,7 +157,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err := addMipmapBloomBins(chainDb); err != nil { return nil, err } - + if err := indexStateDatabase(chainDb, uint64(config.StateRetention)); err != nil { + return nil, err + } dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache) if err != nil { return nil, err @@ -578,3 +587,63 @@ func addMipmapBloomBins(db ethdb.Database) (err error) { glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) return nil } + +// indexStateDatabase iterates over the state tries of the topmost blocks in the +// chain and inserts all child-parent references into the trie index to facilitate +// state trie pruning afterwards. +func indexStateDatabase(db ethdb.Database, retain uint64) error { + // Short circuit if the head block is already indexed + head := core.GetBlock(db, core.GetHeadBlockHash(db)) + if head == nil { + return nil // Empty database, don't die on it + } + if _, err := db.Get(trie.ParentReferenceIndexKey(head.Hash().Bytes(), head.Root().Bytes())); err == nil { + return nil // Head already indexed, assume up to date database + } + // Iterate over the top blocks that are being retained and index all the state tries + glog.V(logger.Info).Infoln("Indexing database for state pruning (first block takes a few minutes, please be patient)...") + + from, till := uint64(0), head.NumberU64() + if from+retain < till { + from = till - retain + } + for num := from; num <= till; num++ { + glog.V(logger.Info).Infof("%.2f%%: Indexing state of block #%d...", 100*float64(num-from)/float64(till-from+1), num) + + // Load the block and ensure any database corruption is signaled + header := core.GetHeader(db, core.GetCanonicalHash(db, num)) + if header == nil { + return fmt.Errorf("block #%d missing", num) + } + // Short circuit if the block has already been indexed previously + if _, err := db.Get(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes())); err == nil { + continue + } + // Otherwise iterate the entire state trie and insert the indices + stateDb, err := state.New(header.Root, db) + if err != nil { + return err + } + it := state.NewNodeIterator(stateDb) + it.PreOrderHook = func(hash, parent common.Hash) bool { + // If we've already indexed this branch, skip + _, err := db.Get(trie.ParentReferenceIndexKey(parent.Bytes(), hash.Bytes())) + if err == nil { + return false + } + return true + } + for it.Next() { + if it.Hash != (common.Hash{}) && it.Parent != (common.Hash{}) { + if err := db.Put(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()), nil); err != nil { + return err + } + } + } + if err := db.Put(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes()), nil); err != nil { + return err + } + } + glog.V(logger.Info).Infof("Database successfully indexed") + return nil +} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1639947304..4ddd128777 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -67,6 +67,8 @@ var ( fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync + + FsStateRetention = fsMinFullBlocks + fsPivotInterval + 1 // Number of state tries needed to reliably fast sync ) var ( diff --git a/trie/iterator.go b/trie/iterator.go index df9efece7f..4f7bf924cc 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -243,18 +243,18 @@ func (it *NodeIterator) step() { } parent.child++ it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1}) - } else if hash, ok := parent.node.(hashNode); ok { + } else if hashNode, ok := parent.node.(hashNode); ok { // Hash node, resolve the hash child from the database, then the node itself if parent.child >= 0 { break } parent.child++ - node, err := it.trie.resolveHash(hash, nil, nil) - if err != nil { - panic(err) - } - if hash := common.BytesToHash(hash); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) { + if hash := common.BytesToHash(hashNode); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) { + node, err := it.trie.resolveHash(hashNode, nil, nil) + if err != nil { + panic(err) + } it.stack = append(it.stack, &nodeIteratorState{hash: hash, node: node, parent: ancestor, child: -1}) } } else {