core/state, trie: generate parent reference database indexes

core/state/index_test.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package state
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+// Tests that all index entries are stored in the database after a state commit.
+func TestStateIndex(t *testing.T) {
+	// Create some arbitrary test state to iterate
+	db, root, _ := makeTestState()
+
+	state, err := New(root, db)
+	if err != nil {
+		t.Fatalf("failed to create state trie at %x: %v", root, err)
+	}
+	// Gather all the indexes that should be present in the database
+	indexes := make(map[string]struct{})
+	for it := NewNodeIterator(state); it.Next(); {
+		if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+			indexes[string(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+		}
+	}
+	// Cross check the indexes and the database itself
+	for index, _ := range indexes {
+		if _, err := db.Get([]byte(index)); err != nil {
+			t.Errorf("failed to retrieve reported index %x: %v", index, err)
+		}
+	}
+	for _, key := range db.(*ethdb.MemDatabase).Keys() {
+		if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+			if _, ok := indexes[string(key)]; !ok {
+				t.Errorf("index entry not reported %x", key)
+			}
+		}
+	}
+}

core/state/iterator_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // Tests that the node iterator indeed walks over the entire database contents.
@@ -50,6 +51,9 @@ func TestNodeIteratorCoverage(t *testing.T) {
 		if bytes.HasPrefix(key, []byte("secure-key-")) {
 			continue
 		}
+		if bytes.HasPrefix(key, trie.ParentReferenceIndexPrefix) {
+			continue
+		}
 		if _, ok := hashes[common.BytesToHash(key)]; !ok {
 			t.Errorf("state entry not reported %x", key)
 		}

core/state/statedb.go

@@ -206,15 +206,17 @@ func (self *StateDB) Delete(addr common.Address) bool {
 
 // Update the given state object and apply it to state trie
 func (self *StateDB) UpdateStateObject(stateObject *StateObject) {
+	refs := [][]byte{stateObject.Root()}
 	if len(stateObject.code) > 0 {
 		self.db.Put(stateObject.codeHash, stateObject.code)
+		refs = append(refs, stateObject.codeHash)
 	}
 	addr := stateObject.Address()
 	data, err := rlp.EncodeToBytes(stateObject)
 	if err != nil {
 		panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
 	}
-	self.trie.Update(addr[:], data)
+	self.trie.UpdateIndexed(addr[:], data, refs)
 }
 
 // Delete the given state object and delete it from the state trie

core/state/sync_test.go

@@ -84,6 +84,9 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou
 	if err := checkStateConsistency(db, root); err != nil {
 		t.Fatalf("inconsistent state trie at %x: %v", root, err)
 	}
+	if err := checkStateIndex(db, root); err != nil {
+		t.Fatalf("index error at %x: %v", root, err)
+	}
 	for i, acc := range accounts {
 		if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 {
 			t.Errorf("account %d: balance mismatch: have %v, want %v", i, balance, acc.balance)
@@ -121,6 +124,31 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) (failure error)
 	return nil
 }
 
+// checkStateIndex iterates over the entire state trie and checks that all required
+// database indexes have been generated by the synchronizer.
+func checkStateIndex(db ethdb.Database, root common.Hash) error {
+	// Remove any potentially cached data from the test state creation or previous checks
+	trie.ClearGlobalCache()
+
+	// Create and iterate a state trie rooted in a sub-node
+	if _, err := db.Get(root.Bytes()); err != nil {
+		return err
+	}
+	state, err := New(root, db)
+	if err != nil {
+		return err
+	}
+	for it := NewNodeIterator(state); it.Next(); {
+		if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+			parentRef := trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes())
+			if _, err := db.Get(parentRef); err != nil {
+				return fmt.Errorf("parent reference lookup %x: %v", parentRef, err)
+			}
+		}
+	}
+	return nil
+}
+
 // Tests that an empty state is not scheduled for syncing.
 func TestEmptyStateSync(t *testing.T) {
 	empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")

trie/index.go

@@ -0,0 +1,68 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import "bytes"
+
+var (
+	// ParentReferenceIndexPrefix is the database key prefix storing parent references index entries
+	ParentReferenceIndexPrefix = []byte("ref-")
+)
+
+// ParentReferenceIndexKey constructs a child->parent database index key.
+func ParentReferenceIndexKey(parent []byte, child []byte) []byte {
+	return append(append(ParentReferenceIndexPrefix, child...), parent...)
+}
+
+// storeParentReferences expands a trie node to find all its stored children and
+// adds a database reference pointing to the parent to permit reference tracking.
+func storeParentReferences(key []byte, node node, db DatabaseWriter) error {
+	switch node := node.(type) {
+	case fullNode:
+		for _, child := range node {
+			if child != nil {
+				if err := storeParentReferences(key, child, db); err != nil {
+					return err
+				}
+			}
+		}
+	case shortNode:
+		if child := node.Val; child != nil {
+			return storeParentReferences(key, child, db)
+		}
+	case hashNode:
+		return db.Put(ParentReferenceIndexKey(key, node), nil)
+
+	case valueNode:
+		for _, child := range node.refs {
+			if bytes.Compare(child, emptyRoot.Bytes()) == 0 {
+				continue // don't index an empty trie
+			}
+			if err := db.Put(ParentReferenceIndexKey(key, child), nil); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// storeParentReferenceEntry manually inserts a parent->child reference entry into
+// the database index without requiring direct, trie-internal descendancy. This is
+// useful to store references to external entities, like accounts or blocks.
+func storeParentReferenceEntry(parent []byte, child []byte, db DatabaseWriter) error {
+	return db.Put(ParentReferenceIndexKey(parent, child), nil)
+}

trie/index_test.go

@@ -0,0 +1,52 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+// Tests that all index entries are stored in the database after a trie commit.
+func TestTrieIndex(t *testing.T) {
+	// Create some arbitrary test trie to iterate
+	db, trie, _ := makeTestTrie()
+
+	// Gather all the indexes that should be present in the database
+	indexes := make(map[string]struct{})
+	for it := NewNodeIterator(trie); it.Next(); {
+		if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+			indexes[string(ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()))] = struct{}{}
+		}
+	}
+	// Cross check the indexes and the database itself
+	for index, _ := range indexes {
+		if _, err := db.Get([]byte(index)); err != nil {
+			t.Errorf("failed to retrieve reported index %x: %v", index, err)
+		}
+	}
+	for _, key := range db.(*ethdb.MemDatabase).Keys() {
+		if bytes.HasPrefix(key, ParentReferenceIndexPrefix) {
+			if _, ok := indexes[string(key)]; !ok {
+				t.Errorf("index entry not reported %x", key)
+			}
+		}
+	}
+}

trie/iterator.go

@@ -86,11 +86,11 @@ func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byt
 			switch bytes.Compare([]byte(k), key) {
 			case 0:
 				if isIterStart {
-					self.Value = vnode
+					self.Value = vnode.Value
 					return k
 				}
 			case 1:
-				self.Value = vnode
+				self.Value = vnode.Value
 				return k
 			}
 		} else {
@@ -125,13 +125,13 @@ func (self *Iterator) key(node interface{}) []byte {
 		// Leaf node
 		k := remTerm(node.Key)
 		if vnode, ok := node.Val.(valueNode); ok {
-			self.Value = vnode
+			self.Value = vnode.Value
 			return k
 		}
 		return append(k, self.key(node.Val)...)
 	case fullNode:
 		if node[16] != nil {
-			self.Value = node[16].(valueNode)
+			self.Value = node[16].(valueNode).Value
 			return []byte{16}
 		}
 		for i := 0; i < 16; i++ {
@@ -265,8 +265,8 @@ func (it *NodeIterator) retrieve() bool {
 	state := it.stack[len(it.stack)-1]
 
 	it.Hash, it.Node, it.Parent = state.hash, state.node, state.parent
-	if value, ok := it.Node.(valueNode); ok {
+	if node, ok := it.Node.(valueNode); ok {
-		it.Leaf, it.LeafBlob = true, []byte(value)
+		it.Leaf, it.LeafBlob = true, []byte(node.Value)
 	}
 	return true
 }

trie/iterator_test.go

@@ -72,8 +72,10 @@ func TestNodeIteratorCoverage(t *testing.T) {
 		}
 	}
 	for _, key := range db.(*ethdb.MemDatabase).Keys() {
+		if len(key) == common.HashLength {
 			if _, ok := hashes[common.BytesToHash(key)]; !ok {
 				t.Errorf("state entry not reported %x", key)
 			}
 		}
 	}
+}

trie/node.go

@@ -38,9 +38,22 @@ type (
 		Val node
 	}
 	hashNode  []byte
-	valueNode []byte
+	valueNode struct {
+		Value []byte
+		refs  [][]byte
+	}
 )
 
+func (n valueNode) EncodeRLP(w io.Writer) error {
+	return rlp.Encode(w, n.Value)
+}
+
+func (n valueNode) DecodeRLP(s *rlp.Stream) (err error) {
+	fmt.Println("Decoding value node")
+	n.Value, err = s.Bytes()
+	return
+}
+
 // Pretty printing.
 func (n fullNode) String() string  { return n.fstring("") }
 func (n shortNode) String() string { return n.fstring("") }
@@ -65,7 +78,7 @@ func (n hashNode) fstring(ind string) string {
 	return fmt.Sprintf("<%x> ", []byte(n))
 }
 func (n valueNode) fstring(ind string) string {
-	return fmt.Sprintf("%x ", []byte(n))
+	return fmt.Sprintf("%x ", []byte(n.Value))
 }
 
 func mustDecodeNode(dbkey, buf []byte) node {
@@ -109,7 +122,7 @@ func decodeShort(buf []byte) (node, error) {
 		if err != nil {
 			return nil, fmt.Errorf("invalid value node: %v", err)
 		}
-		return shortNode{key, valueNode(val)}, nil
+		return shortNode{key, valueNode{Value: val}}, nil
 	}
 	r, _, err := decodeRef(rest)
 	if err != nil {
@@ -132,7 +145,7 @@ func decodeFull(buf []byte) (fullNode, error) {
 		return n, err
 	}
 	if len(val) > 0 {
-		n[16] = valueNode(val)
+		n[16] = valueNode{Value: val}
 	}
 	return n, nil
 }

trie/proof.go

@@ -107,7 +107,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value
 			if i != len(proof)-1 {
 				return nil, errors.New("additional nodes at end of proof")
 			}
-			return cld, nil
+			return cld.Value, nil
 		}
 	}
 	return nil, errors.New("unexpected end of proof")

trie/secure_trie.go

@@ -109,6 +109,34 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error {
 	return nil
 }
 
+// UpdateIndexed is an extended version of Update, where state trie index entries
+// are also generated for all entities referencing the current node.
+//
+// The value bytes must not be modified by the caller while they are
+// stored in the trie.
+func (t *SecureTrie) UpdateIndexed(key, value []byte, refs [][]byte) {
+	if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) {
+		glog.Errorf("Unhandled trie error: %v", err)
+	}
+}
+
+// TryUpdateIndexed is an extended version of Update, where state trie index
+// entries are also generated for all entities referencing the current node.
+//
+// The value bytes must not be modified by the caller while they are
+// stored in the trie.
+//
+// If a node was not found in the database, a MissingNodeError is returned.
+func (t *SecureTrie) TryUpdateIndexed(key, value []byte, refs [][]byte) error {
+	hk := t.hashKey(key)
+	err := t.Trie.TryUpdateIndexed(hk, value, refs)
+	if err != nil {
+		return err
+	}
+	t.Trie.db.Put(t.secKey(hk), key)
+	return nil
+}
+
 // Delete removes any existing value for key from the trie.
 func (t *SecureTrie) Delete(key []byte) {
 	if err := t.TryDelete(key); err != nil && glog.V(logger.Error) {

trie/sync.go

@@ -34,6 +34,8 @@ type request struct {
 	depth   int        // Depth level within the trie the node is located to prioritise DFS
 	deps    int        // Number of dependencies before allowed to commit this node
 
+	externals []common.Hash // External children of this node to index in addition to internal ones
+
 	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
 }
 
@@ -94,6 +96,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c
 			panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
 		}
 		ancestor.deps++
+		ancestor.externals = append(ancestor.externals, root)
 		req.parents = append(req.parents, ancestor)
 	}
 	s.schedule(req)
@@ -123,6 +126,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash)
 			panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
 		}
 		ancestor.deps++
+		ancestor.externals = append(ancestor.externals, hash)
 		req.parents = append(req.parents, ancestor)
 	}
 	s.schedule(req)
@@ -229,7 +233,7 @@ func (s *TrieSync) children(req *request) ([]*request, error) {
 		// Notify any external watcher of a new key/value node
 		if req.callback != nil {
 			if node, ok := (*child.node).(valueNode); ok {
-				if err := req.callback(node, req.hash); err != nil {
+				if err := req.callback(node.Value, req.hash); err != nil {
 					return nil, err
 				}
 			}
@@ -266,7 +270,17 @@ func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) {
 			err = batch.Write()
 		}()
 	}
-	// Write the node content to disk
+	// Write the node content and indices to disk
+	if req.object != nil {
+		if err := storeParentReferences(req.hash[:], *req.object, batch); err != nil {
+			return err
+		}
+	}
+	for _, ext := range req.externals {
+		if err := storeParentReferenceEntry(req.hash[:], ext[:], batch); err != nil {
+			return err
+		}
+	}
 	if err := batch.Put(req.hash[:], req.data); err != nil {
 		return err
 	}

trie/sync_test.go

@@ -73,6 +73,9 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin
 	if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil {
 		t.Fatalf("inconsistent trie at %x: %v", root, err)
 	}
+	if err := checkTrieIndex(db, common.BytesToHash(root)); err != nil {
+		t.Fatalf("index error at %x: %v", root, err)
+	}
 	for key, val := range content {
 		if have := trie.Get([]byte(key)); bytes.Compare(have, val) != 0 {
 			t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val)
@@ -101,6 +104,28 @@ func checkTrieConsistency(db Database, root common.Hash) (failure error) {
 	return nil
 }
 
+// checkTrieIndex iterates over the entire state trie and checks that all required
+// database indexes have been generated by the synchronizer.
+func checkTrieIndex(db Database, root common.Hash) error {
+	// Remove any potentially cached data from the test trie creation or previous checks
+	globalCache.Clear()
+
+	// Create and iterate a trie
+	trie, err := New(root, db)
+	if err != nil {
+		return err
+	}
+	for it := NewNodeIterator(trie); it.Next(); {
+		if (it.Hash != common.Hash{}) && (it.Parent != common.Hash{}) {
+			parentRef := ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes())
+			if _, err := db.Get(parentRef); err != nil {
+				return fmt.Errorf("parent reference lookup %x: %v", parentRef, err)
+			}
+		}
+	}
+	return nil
+}
+
 // Tests that an empty trie is not scheduled for syncing.
 func TestEmptyTrieSync(t *testing.T) {
 	emptyA, _ := New(common.Hash{}, nil)

trie/trie.go

@@ -143,7 +143,7 @@ func (t *Trie) TryGet(key []byte) ([]byte, error) {
 			panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
 		}
 	}
-	return tn.(valueNode), nil
+	return tn.(valueNode).Value, nil
 }
 
 // Update associates key with value in the trie. Subsequent calls to
@@ -167,9 +167,35 @@ func (t *Trie) Update(key, value []byte) {
 //
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *Trie) TryUpdate(key, value []byte) error {
+	return t.tryUpdateIndexed(key, value, nil)
+}
+
+// UpdateIndexed is an extended version of Update, where state trie index entries
+// are also generated for all entities referencing the current node.
+//
+// The value bytes must not be modified by the caller while they are
+// stored in the trie.
+func (t *Trie) UpdateIndexed(key, value []byte, refs [][]byte) {
+	if err := t.TryUpdateIndexed(key, value, refs); err != nil && glog.V(logger.Error) {
+		glog.Errorf("Unhandled trie error: %v", err)
+	}
+}
+
+// TryUpdateIndexed is an extended version of Update, where state trie index
+// entries are also generated for all entities referencing the current node.
+//
+// The value bytes must not be modified by the caller while they are
+// stored in the trie.
+//
+// If a node was not found in the database, a MissingNodeError is returned.
+func (t *Trie) TryUpdateIndexed(key, value []byte, refs [][]byte) error {
+	return t.tryUpdateIndexed(key, value, refs)
+}
+
+func (t *Trie) tryUpdateIndexed(key, value []byte, refs [][]byte) error {
 	k := compactHexDecode(key)
 	if len(value) != 0 {
-		n, err := t.insert(t.root, nil, k, valueNode(value))
+		n, err := t.insert(t.root, nil, k, valueNode{Value: value, refs: refs})
 		if err != nil {
 			return err
 		}
@@ -489,7 +515,7 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) {
 		}
 		if n.Val == nil {
 			// Ensure that nil children are encoded as empty strings.
-			n.Val = valueNode(nil)
+			n.Val = valueNode{Value: nil}
 		}
 		return n, nil
 	case fullNode:
@@ -500,11 +526,11 @@ func (h *hasher) replaceChildren(n node, db DatabaseWriter) (node, error) {
 				}
 			} else {
 				// Ensure that nil children are encoded as empty strings.
-				n[i] = valueNode(nil)
+				n[i] = valueNode{Value: nil}
 			}
 		}
 		if n[16] == nil {
-			n[16] = valueNode(nil)
+			n[16] = valueNode{Value: nil}
 		}
 		return n, nil
 	default:
@@ -530,8 +556,13 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
 	h.sha.Write(h.tmp.Bytes())
 	key := hashNode(h.sha.Sum(nil))
 	if db != nil {
-		err := db.Put(key, h.tmp.Bytes())
+		if err := storeParentReferences(key, n, db); err != nil {
+			return key, err
+		}
+		if err := db.Put(key, h.tmp.Bytes()); err != nil {
 			return key, err
 		}
 		return key, nil
 	}
+	return key, nil
+}