cmd/geth, core/state, trie: don't iterate known subtries

cmd/geth/chaincmd.go

@@ -118,8 +118,11 @@ func importChain(ctx *cli.Context) error {
 		utils.Fatalf("Failed to read database stats: %v", err)
 	}
 	fmt.Println(stats)
-	fmt.Printf("Trie cache misses:  %d\n", trie.CacheMisses())
+	fmt.Printf("Trie cache misses:   %d\n", trie.CacheMisses())
-	fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
+	fmt.Printf("Trie cache unloads:  %d\n", trie.CacheUnloads())
+	fmt.Printf("Direct cache reads:  %d\n", trie.DirectCacheReads())
+	fmt.Printf("Direct cache writes: %d\n", trie.DirectCacheWrites())
+	fmt.Printf("Direct cache misses: %d\n\n", trie.DirectCacheMisses())
 
 	// Print the memory statistics used by the importing
 	mem := new(runtime.MemStats)
@@ -308,7 +311,7 @@ func buildCache(ctx *cli.Context) error {
 		}
 
 		i += 1
-		if i % 10000 == 0 {
+		if i%10000 == 0 {
 			fmt.Printf("Processed %d accounts, at %v\n", i, common.ToHex(iter.Key))
 		}
 	}

core/state/sync.go

@@ -53,12 +53,12 @@ func NewStateSync(number uint64, hash common.Hash, root common.Hash, database et
 			}
 		}
 		// Schedule downloading all the dependencies of the account
-		syncer.AddSubTrie(obj.Root, 64, parent, nil)
+		syncer.AddSubTrie(obj.Root, 64, parent, nil, nil)
 		syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent)
 
 		return nil
 	}
-	syncer = trie.NewTrieSync(root, database, callback)
+	syncer = trie.NewTrieSync(root, database, callback, CachePrefix)
 	return (*StateSync)(syncer)
 }
 

trie/directcache.go

@@ -30,6 +30,27 @@ var directCacheWrites = metrics.NewCounter("directcache/writes")
 var directCacheHitTimer = metrics.NewTimer("directcache/timer/hits")
 var directCacheMissTimer = metrics.NewTimer("directcache/timer/misses")
 
+// DirectCacheReads retrieves a global counter measuring the number of direct
+// cache reads from the disk since process startup. This isn't useful for anything
+// apart from trie debugging purposes.
+func DirectCacheReads() int64 {
+	return directCacheHitTimer.Count() + directCacheMissTimer.Count()
+}
+
+// DirectCacheWrites retrieves a global counter measuring the number of direct
+// cache writes from the disk since process startup. This isn't useful for anything
+// apart from trie debugging purposes.
+func DirectCacheWrites() int64 {
+	return directCacheWrites.Count()
+}
+
+// DirectCacheMisses retrieves a global counter measuring the number of direct
+// cache writes from the disk since process startup. This isn't useful for anything
+// apart from trie debugging purposes.
+func DirectCacheMisses() int64 {
+	return directCacheMissTimer.Count()
+}
+
 type cachedValue struct {
 	Value     []byte
 	BlockNum  uint64
@@ -157,7 +178,6 @@ func (dc *DirectCache) TryDelete(key []byte) error {
 }
 
 func (dc *DirectCache) CommitTo(dbw DatabaseWriter) (root common.Hash, err error) {
-	directCacheWrites.Inc(int64(len(dc.dirty)))
 	for k, _ := range dc.dirty {
 		v, err := dc.data.TryGet([]byte(k))
 		if err, ok := err.(*MissingNodeError); err != nil && !ok {
@@ -175,7 +195,23 @@ func (dc *DirectCache) putCache(dbw DatabaseWriter, key, value []byte) error {
 	return WriteDirectCache(dc.keyPrefix, key, value, dc.blockNum, dc.blockHash, dbw)
 }
 
+// WriteDirectCache places a value node directly into the database along with
+// block metadata to validate its relevancy.
+//
+// The method is meant to be used by code that circumvents the state database
+// and its integrated cache, namely during fast sync and database upgrades.
 func WriteDirectCache(prefix, key, value []byte, number uint64, hash common.Hash, dbw DatabaseWriter) error {
+	directCacheWrites.Inc(1)
 	enc, _ := rlp.EncodeToBytes(cachedValue{value, number, hash})
 	return dbw.Put(append(prefix, key...), enc)
 }
+
+// GetDirectCache retrieves a value node directly from the database along with
+// block metadata to validate its relevancy.
+//
+// The method is meant to be used by code that circumvents the state database
+// and its integrated cache, namely during fast sync and database upgrades.
+func GetDirectCache(prefix, key []byte, db Database) ([]byte, error) {
+	defer func(start time.Time) { directCacheHitTimer.UpdateSince(start) }(time.Now())
+	return db.Get(append(prefix, key...))
+}

trie/sync.go

@@ -41,6 +41,7 @@ type request struct {
 	deps    int        // Number of dependencies before allowed to commit this node
 
 	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
+	dcPrefix []byte               // Database prefix to use for leaf direct caching
 }
 
 // keys traverses the request ancestry tree, reconstructing the key paths leading
@@ -92,18 +93,18 @@ type TrieSync struct {
 }
 
 // NewTrieSync creates a new trie data download scheduler.
-func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
+func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback, dcPrefix []byte) *TrieSync {
 	ts := &TrieSync{
 		database: database,
 		requests: make(map[common.Hash]*request),
 		queue:    prque.New(),
 	}
-	ts.AddSubTrie(root, 0, common.Hash{}, callback)
+	ts.AddSubTrie(root, 0, common.Hash{}, callback, dcPrefix)
 	return ts
 }
 
 // AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
-func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
+func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback, dcPrefix []byte) {
 	// Short circuit if the trie is empty or already known
 	if root == emptyRoot {
 		return
@@ -118,6 +119,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c
 		hash:     root,
 		depth:    depth,
 		callback: callback,
+		dcPrefix: dcPrefix,
 	}
 	// If this sub-trie has a designated parent, link them together
 	if parent != (common.Hash{}) {
@@ -280,13 +282,26 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
 			blob, _ := s.database.Get(node)
 			if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil {
 				// Local node found, iterate and invoke the leaf callback for all subleaves
-				if req.callback != nil {
+				if req.callback != nil && req.dcPrefix != nil {
 					trie, _ := New(common.BytesToHash(node[:]), s.database, 0)
+					unknown := false
 
 					it := NewNodeIterator(trie)
 					for it.Next() {
 						if it.Leaf {
-							req.callback(req.keys(append(child.path, it.Nibbles...)), nil, req.hash)
+							// If this branch was already visited, abort iteration
+							keys := req.keys(append(child.path, it.Nibbles...))
+							if !unknown {
+								if val, err := GetDirectCache(req.dcPrefix, keys[0], s.database); val != nil && err == nil {
+									break
+								}
+								// Branch is unknown, mark as so to avoid repeated db lookups
+								unknown = true
+							}
+							// Otherwise write out all accounts ending in this leaf
+							if err := req.callback(keys, nil, req.hash); err != nil {
+								return nil, err
+							}
 						}
 					}
 				}
@@ -299,6 +314,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
 				nibbles:  [][]byte{child.path},
 				depth:    child.depth,
 				callback: req.callback,
+				dcPrefix: req.dcPrefix,
 			})
 		}
 	}

trie/sync_test.go

@@ -115,7 +115,7 @@ func TestEmptyTrieSync(t *testing.T) {
 
 	for i, trie := range []*Trie{emptyA, emptyB} {
 		db, _ := ethdb.NewMemDatabase()
-		if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 {
+		if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil, nil).Missing(1); len(req) != 0 {
 			t.Errorf("test %d: content requested for empty trie: %v", i, req)
 		}
 	}
@@ -132,7 +132,7 @@ func testIterativeTrieSync(t *testing.T, batch int) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(batch)...)
 	for len(queue) > 0 {
@@ -161,7 +161,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(10000)...)
 	for len(queue) > 0 {
@@ -195,7 +195,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(batch) {
@@ -232,7 +232,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(10000) {
@@ -275,7 +275,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(0)...)
 	requested := make(map[common.Hash]struct{})
@@ -311,7 +311,7 @@ func TestIncompleteTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	added := []common.Hash{}
 	queue := append([]common.Hash{}, sched.Missing(1)...)
@@ -371,7 +371,7 @@ func TestAllLeafCallbackTrieSync(t *testing.T) {
 	}
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, callback)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, callback, []byte("leaf:"))
 
 	queue := append([]common.Hash{}, sched.Missing(1)...)
 	for len(queue) > 0 {