diff --git a/core/state/index_test.go b/core/state/index_test.go new file mode 100644 index 0000000000..25f1f853f2 --- /dev/null +++ b/core/state/index_test.go @@ -0,0 +1,57 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" +) + +// Tests that all index entries are stored in the database after a state commit. +func TestStateIndex(t *testing.T) { + // Create some arbitrary test state to iterate + db, root, _ := makeTestState() + + state, err := New(root, db) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", root, err) + } + // Gather all the indexes that should be present in the database + indexes := make(map[string]struct{}) + for it := NewNodeIterator(state); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + t.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + t.Errorf("index entry not reported %x", key) + } + } + } +} diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 8b68870c6e..df0135296a 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) // Tests that the node iterator indeed walks over the entire database contents. @@ -50,6 +51,9 @@ func TestNodeIteratorCoverage(t *testing.T) { if bytes.HasPrefix(key, []byte("secure-key-")) { continue } + if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) { + continue + } if _, ok := hashes[common.BytesToHash(key)]; !ok { t.Errorf("state entry not reported %x", key) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 8093472b59..e8e1e269e2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -206,15 +206,17 @@ func (self *StateDB) Delete(addr common.Address) bool { // Update the given state object and apply it to state trie func (self *StateDB) UpdateStateObject(stateObject *StateObject) { + refs := [][]byte{stateObject.Root()} if len(stateObject.code) > 0 { self.db.Put(stateObject.codeHash, stateObject.code) + refs = append(refs, stateObject.codeHash) } addr := stateObject.Address() data, err := rlp.EncodeToBytes(stateObject) if err != nil { panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err)) } - self.trie.Update(addr[:], data) + self.trie.UpdateIndexed(addr[:], data, refs) } // Delete the given state object and delete it from the state trie diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 727a276a50..5e59c8f9cd 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -84,6 +84,9 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou if err := checkStateConsistency(db, root); err != nil { t.Fatalf("inconsistent state trie at %x: %v", root, err) } + if err := checkStateIndex(db, root); err != nil { + t.Fatalf("index error at %x: %v", root, err) + } for i, acc := range accounts { if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 { t.Errorf("account %d: balance mismatch: have %v, want %v", i, balance, acc.balance) @@ -121,6 +124,31 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error) return nil } +// checkStateIndex iterates over the entire state trie and checks that all required +// database indexes have been generated by the synchronizer. +func checkStateIndex(db ethdb.Database, root common.Hash) error { + // Remove any potentially cached data from the test state creation or previous checks + trie.ClearGlobalCache() + + // Create and iterate a state trie rooted in a sub-node + if _, err := db.Get(root.Bytes()); err != nil { + return err + } + state, err := New(root, db) + if err != nil { + return err + } + for it := NewNodeIterator(state); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + parentRef := trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()) + if _, err := db.Get(parentRef); err != nil { + return fmt.Errorf("parent reference lookup %x: %v", parentRef, err) + } + } + } + return nil +} + // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") diff --git a/trie/index.go b/trie/index.go new file mode 100644 index 0000000000..705e605ea5 --- /dev/null +++ b/trie/index.go @@ -0,0 +1,68 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import "bytes" + +var ( + // ParentReferenceIndexPrefix is the database key prefix storing parent references index entries + ParentReferenceIndexPrefix = []byte("ref-") +) + +// ParentReferenceIndexKey constructs a child->parent database index key. +func ParentReferenceIndexKey(parent []byte, child []byte) []byte { + return append(append(ParentReferenceIndexPrefix, child...), parent...) +} + +// storeParentReferences expands a trie node to find all its stored children and +// adds a database reference pointing to the parent to permit reference tracking. +func storeParentReferences(key []byte, node node, db DatabaseWriter) error { + switch node := node.(type) { + case fullNode: + for _, child := range node { + if child != nil { + if err := storeParentReferences(key, child, db); err != nil { + return err + } + } + } + case shortNode: + if child := node.Val; child != nil { + return storeParentReferences(key, child, db) + } + case hashNode: + return db.Put(ParentReferenceIndexKey(key, node), nil) + + case valueNode: + for _, child := range node.refs { + if bytes.Compare(child, emptyRoot.Bytes()) == 0 { + continue // don't index an empty trie + } + if err := db.Put(ParentReferenceIndexKey(key, child), nil); err != nil { + return err + } + } + } + return nil +} + +// storeParentReferenceEntry manually inserts a parent->child reference entry into +// the database index without requiring direct, trie-internal descendancy. This is +// useful to store references to external entities, like accounts or blocks. +func storeParentReferenceEntry(parent []byte, child []byte, db DatabaseWriter) error { + return db.Put(ParentReferenceIndexKey(parent, child), nil) +} diff --git a/trie/index_test.go b/trie/index_test.go new file mode 100644 index 0000000000..1ecf8dd7bf --- /dev/null +++ b/trie/index_test.go @@ -0,0 +1,52 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +// Tests that all index entries are stored in the database after a trie commit. +func TestTrieIndex(t *testing.T) { + // Create some arbitrary test trie to iterate + db, trie, _ := makeTestTrie() + + // Gather all the indexes that should be present in the database + indexes := make(map[string]struct{}) + for it := NewNodeIterator(trie); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{} + } + } + // Cross check the indexes and the database itself + for index, _ := range indexes { + if _, err := db.Get([]byte(index)); err != nil { + t.Errorf("failed to retrieve reported index %x: %v", index, err) + } + } + for _, key := range db.(*ethdb.MemDatabase).Keys() { + if bytes.HasPrefix(key, ParentReferenceIndexPrefix) { + if _, ok := indexes[string(key)]; !ok { + t.Errorf("index entry not reported %x", key) + } + } + } +} diff --git a/trie/iterator.go b/trie/iterator.go index d43b77698c..59ac3c5182 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -86,11 +86,11 @@ func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byt switch bytes.Compare([]byte(k), key) { case 0: if isIterStart { - self.Value = vnode + self.Value = vnode.Value return k } case 1: - self.Value = vnode + self.Value = vnode.Value return k } } else { @@ -125,13 +125,13 @@ func (self *Iterator) key(node interface{}) []byte { // Leaf node k := remTerm(node.Key) if vnode, ok := node.Val.(valueNode); ok { - self.Value = vnode + self.Value = vnode.Value return k } return append(k, self.key(node.Val)...) case fullNode: if node[16] != nil { - self.Value = node[16].(valueNode) + self.Value = node[16].(valueNode).Value return []byte{16} } for i := 0; i < 16; i++ { @@ -265,8 +265,8 @@ func (it *NodeIterator) retrieve() bool { state := it.stack[len(it.stack)-1] it.Hash, it.Node, it.Parent = state.hash, state.node, state.parent - if value, ok := it.Node.(valueNode); ok { - it.Leaf, it.LeafBlob = true, []byte(value) + if node, ok := it.Node.(valueNode); ok { + it.Leaf, it.LeafBlob = true, []byte(node.Value) } return true } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index dc82761165..815743bfdc 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -72,8 +72,10 @@ func TestNodeIteratorCoverage(t *testing.T) { } } for _, key := range db.(*ethdb.MemDatabase).Keys() { - if _, ok := hashes[common.BytesToHash(key)]; !ok { - t.Errorf("state entry not reported %x", key) + if len(key) == common.HashLength { + if _, ok := hashes[common.BytesToHash(key)]; !ok { + t.Errorf("state entry not reported %x", key) + } } } } diff --git a/trie/node.go b/trie/node.go index 0bfa21dc43..01452a8531 100644 --- a/trie/node.go +++ b/trie/node.go @@ -38,9 +38,22 @@ type ( Val node } hashNode []byte - valueNode []byte + valueNode struct { + Value []byte + refs [][]byte + } ) +func (n valueNode) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, n.Value) +} + +func (n valueNode) DecodeRLP(s *rlp.Stream) (err error) { + fmt.Println("Decoding value node") + n.Value, err = s.Bytes() + return +} + // Pretty printing. func (n fullNode) String() string { return n.fstring("") } func (n shortNode) String() string { return n.fstring("") } @@ -65,7 +78,7 @@ func (n hashNode) fstring(ind string) string { return fmt.Sprintf("<%x> ", []byte(n)) } func (n valueNode) fstring(ind string) string { - return fmt.Sprintf("%x ", []byte(n)) + return fmt.Sprintf("%x ", []byte(n.Value)) } func mustDecodeNode(dbkey, buf []byte) node { @@ -109,7 +122,7 @@ func decodeShort(buf []byte) (node, error) { if err != nil { return nil, fmt.Errorf("invalid value node: %v", err) } - return shortNode{key, valueNode(val)}, nil + return shortNode{key, valueNode{Value: val}}, nil } r, _, err := decodeRef(rest) if err != nil { @@ -132,7 +145,7 @@ func decodeFull(buf []byte) (fullNode, error) { return n, err } if len(val) > 0 { - n[16] = valueNode(val) + n[16] = valueNode{Value: val} } return n, nil } diff --git a/trie/proof.go b/trie/proof.go index 0f92649424..3c49309026 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -107,7 +107,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value if i != len(proof)-1 { return nil, errors.New("additional nodes at end of proof") } - return cld, nil + return cld.Value, nil } } return nil, errors.New("unexpected end of proof") diff --git a/trie/secure_trie.go b/trie/secure_trie.go index caeef3c3a3..fb5f694d8e 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -109,6 +109,34 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error { return nil } +// UpdateIndexed is an extended version of Update, where state trie index entries +// are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +func (t *SecureTrie) UpdateIndexed(key, value []byte, refs [][]byte) { + if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) { + glog.Errorf("Unhandled trie error: %v", err) + } +} + +// TryUpdateIndexed is an extended version of Update, where state trie index +// entries are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +// +// If a node was not found in the database, a MissingNodeError is returned. +func (t *SecureTrie) TryUpdateIndexed(key, value []byte, refs [][]byte) error { + hk := t.hashKey(key) + err := t.Trie.TryUpdateIndexed(hk, value, refs) + if err != nil { + return err + } + t.Trie.db.Put(t.secKey(hk), key) + return nil +} + // Delete removes any existing value for key from the trie. func (t *SecureTrie) Delete(key []byte) { if err := t.TryDelete(key); err != nil && glog.V(logger.Error) { diff --git a/trie/sync.go b/trie/sync.go index d55399d06b..c3cd36cf79 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -34,6 +34,8 @@ type request struct { depth int // Depth level within the trie the node is located to prioritise DFS deps int // Number of dependencies before allowed to commit this node + externals []common.Hash // External children of this node to index in addition to internal ones + callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch } @@ -94,6 +96,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) } ancestor.deps++ + ancestor.externals = append(ancestor.externals, root) req.parents = append(req.parents, ancestor) } s.schedule(req) @@ -123,6 +126,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) } ancestor.deps++ + ancestor.externals = append(ancestor.externals, hash) req.parents = append(req.parents, ancestor) } s.schedule(req) @@ -229,7 +233,7 @@ func (s *TrieSync) children(req *request) ([]*request, error) { // Notify any external watcher of a new key/value node if req.callback != nil { if node, ok := (*child.node).(valueNode); ok { - if err := req.callback(node, req.hash); err != nil { + if err := req.callback(node.Value, req.hash); err != nil { return nil, err } } @@ -266,7 +270,17 @@ func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) { err = batch.Write() }() } - // Write the node content to disk + // Write the node content and indices to disk + if req.object != nil { + if err := storeParentReferences(req.hash[:], *req.object, batch); err != nil { + return err + } + } + for _, ext := range req.externals { + if err := storeParentReferenceEntry(req.hash[:], ext[:], batch); err != nil { + return err + } + } if err := batch.Put(req.hash[:], req.data); err != nil { return err } diff --git a/trie/sync_test.go b/trie/sync_test.go index ce098d5a0d..6738427bfb 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -73,6 +73,9 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil { t.Fatalf("inconsistent trie at %x: %v", root, err) } + if err := checkTrieIndex(db, common.BytesToHash(root)); err != nil { + t.Fatalf("index error at %x: %v", root, err) + } for key, val := range content { if have := trie.Get([]byte(key)); bytes.Compare(have, val) != 0 { t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val) @@ -101,6 +104,28 @@ func checkTrieConsistency(db Database, root common.Hash) (failure error) { return nil } +// checkTrieIndex iterates over the entire state trie and checks that all required +// database indexes have been generated by the synchronizer. +func checkTrieIndex(db Database, root common.Hash) error { + // Remove any potentially cached data from the test trie creation or previous checks + globalCache.Clear() + + // Create and iterate a trie + trie, err := New(root, db) + if err != nil { + return err + } + for it := NewNodeIterator(trie); it.Next(); { + if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) { + parentRef := ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()) + if _, err := db.Get(parentRef); err != nil { + return fmt.Errorf("parent reference lookup %x: %v", parentRef, err) + } + } + } + return nil +} + // Tests that an empty trie is not scheduled for syncing. func TestEmptyTrieSync(t *testing.T) { emptyA, _ := New(common.Hash{}, nil) diff --git a/trie/trie.go b/trie/trie.go index 9dfde45294..88b32aa45a 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -143,7 +143,7 @@ func (t *Trie) TryGet(key []byte) ([]byte, error) { panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - return tn.(valueNode), nil + return tn.(valueNode).Value, nil } // Update associates key with value in the trie. Subsequent calls to @@ -167,9 +167,35 @@ func (t *Trie) Update(key, value []byte) { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + return t.tryUpdateIndexed(key, value, nil) +} + +// UpdateIndexed is an extended version of Update, where state trie index entries +// are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +func (t *Trie) UpdateIndexed(key, value []byte, refs [][]byte) { + if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) { + glog.Errorf("Unhandled trie error: %v", err) + } +} + +// TryUpdateIndexed is an extended version of Update, where state trie index +// entries are also generated for all entities referencing the current node. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +// +// If a node was not found in the database, a MissingNodeError is returned. +func (t *Trie) TryUpdateIndexed(key, value []byte, refs [][]byte) error { + return t.tryUpdateIndexed(key, value, refs) +} + +func (t *Trie) tryUpdateIndexed(key, value []byte, refs [][]byte) error { k := compactHexDecode(key) if len(value) != 0 { - n, err := t.insert(t.root, nil, k, valueNode(value)) + n, err := t.insert(t.root, nil, k, valueNode{Value: value, refs: refs}) if err != nil { return err } @@ -489,7 +515,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { } if n.Val == nil { // Ensure that nil children are encoded as empty strings. - n.Val = valueNode(nil) + n.Val = valueNode{Value: nil} } return n, nil case fullNode: @@ -500,11 +526,11 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) { } } else { // Ensure that nil children are encoded as empty strings. - n[i] = valueNode(nil) + n[i] = valueNode{Value: nil} } } if n[16] == nil { - n[16] = valueNode(nil) + n[16] = valueNode{Value: nil} } return n, nil default: @@ -530,8 +556,13 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { h.sha.Write(h.tmp.Bytes()) key := hashNode(h.sha.Sum(nil)) if db != nil { - err := db.Put(key, h.tmp.Bytes()) - return key, err + if err := storeParentReferences(key, n, db); err != nil { + return key, err + } + if err := db.Put(key, h.tmp.Bytes()); err != nil { + return key, err + } + return key, nil } return key, nil }