core/state: cache storage tries across blocks and pending state

cmd/geth/chaincmd.go

@@ -259,8 +259,10 @@ func importChain(ctx *cli.Context) error {
 	}
 	fmt.Println(ioStats)
 
-	fmt.Printf("Trie cache misses:  %d\n", trie.CacheMisses())
-	fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
+	fmt.Printf("State storage cache misses:  %d\n", state.StorageTrieMisses())
+	fmt.Printf("State storage cache unloads: %d\n", state.StorageTrieUnloads())
+	fmt.Printf("Global trie cache misses:  %d\n", trie.CacheMisses())
+	fmt.Printf("Global trie cache unloads: %d\n\n", trie.CacheUnloads())
 
 	// Print the memory statistics used by the importing
 	mem := new(runtime.MemStats)

core/state/database.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/trie"
 	lru "github.com/hashicorp/golang-lru"
 )
@@ -30,14 +31,36 @@ import (
 var MaxTrieCacheGen = uint16(120)
 
 const (
-	// Number of past tries to keep. This value is chosen such that
-	// reasonable chain reorg depths will hit an existing trie.
+	// maxPastTries is the number of past tries to keep. This value is chosen such
+	// that reasonable chain reorg depths will hit an existing trie.
 	maxPastTries = 12
 
-	// Number of codehash->size associations to keep.
+	// maxPastStorageTries is the number of past storage tries to keep.
+	maxPastStorageTries = 4096
+
+	// codeSizeCacheSize is the number of codehash->size associations to keep.
 	codeSizeCacheSize = 100000
 )
 
+var (
+	storageTrieMissCounter   = metrics.NewRegisteredCounter("state/storage/miss", nil)
+	storageTrieUnloadCounter = metrics.NewRegisteredCounter("state/storage/unload", nil)
+)
+
+// StorageTrieMisses retrieves a global counter measuring the number of storage
+// trie cache misses the state had since process startup. This isn't useful for
+// anything apart from state debugging purposes.
+func StorageTrieMisses() int64 {
+	return storageTrieMissCounter.Count()
+}
+
+// StorageTrieUnloads retrieves a global counter measuring the number of storage
+// trie cache unloads the state had since process startup. This isn't useful for
+// anything apart from state debugging purposes.
+func StorageTrieUnloads() int64 {
+	return storageTrieUnloadCounter.Count()
+}
+
 // Database wraps access to tries and contract code.
 type Database interface {
 	// OpenTrie opens the main account trie.
@@ -77,17 +100,23 @@ type Trie interface {
 // high level trie abstraction.
 func NewDatabase(db ethdb.Database) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
+	str, _ := lru.NewWithEvict(maxPastStorageTries, func(key interface{}, value interface{}) {
+		storageTrieUnloadCounter.Inc(1)
+	})
+
 	return &cachingDB{
-		db:            trie.NewDatabase(db),
-		codeSizeCache: csc,
+		db:               trie.NewDatabase(db),
+		pastStorageTries: str,
+		codeSizeCache:    csc,
 	}
 }
 
 type cachingDB struct {
-	db            *trie.Database
-	mu            sync.Mutex
-	pastTries     []*trie.SecureTrie
-	codeSizeCache *lru.Cache
+	db               *trie.Database
+	mu               sync.Mutex
+	pastTries        []*trie.SecureTrie
+	pastStorageTries *lru.Cache
+	codeSizeCache    *lru.Cache
 }
 
 // OpenTrie opens the main account trie.
@@ -121,7 +150,36 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
-	return trie.NewSecure(root, db.db, 0)
+	db.mu.Lock()
+	defer db.mu.Unlock()
+
+	// Retrieve a storage trie from the cache if available
+	if t, ok := db.pastStorageTries.Get(root); ok {
+		return &cachedStorageTrie{reader: t.(*syncedTrie), db: db}, nil
+	}
+	// Trie not cached, construct a brand new one and cache it if non-empty
+	tr, err := trie.NewSecure(root, db.db, MaxTrieCacheGen)
+	if err != nil {
+		return nil, err
+	}
+	str := &syncedTrie{tr: tr}
+
+	if root != emptyRoot {
+		db.pastStorageTries.Add(root, str)
+		storageTrieMissCounter.Inc(1)
+	}
+	return &cachedStorageTrie{reader: str, db: db}, nil
+}
+
+func (db *cachingDB) pushStorageTrie(t *syncedTrie) {
+	// Refuse to cache the empty trie
+	if t.Hash() == emptyRoot {
+		return
+	}
+	db.mu.Lock()
+	defer db.mu.Unlock()
+
+	db.pastStorageTries.Add(t.Hash(), t)
 }
 
 // CopyTrie returns an independent copy of the given trie.
@@ -129,6 +187,11 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 	switch t := t.(type) {
 	case cachedTrie:
 		return cachedTrie{t.SecureTrie.Copy(), db}
+	case *cachedStorageTrie:
+		if t.writer != nil {
+			return &cachedStorageTrie{reader: t.writer.Copy(), db: db}
+		}
+		return &cachedStorageTrie{reader: t.reader.Copy(), db: db}
 	case *trie.SecureTrie:
 		return t.Copy()
 	default:
@@ -176,3 +239,151 @@ func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
 func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
 	return m.SecureTrie.Prove(key, fromLevel, proofDb)
 }
+
+// syncedTrie is a synchronized wrapper around a trie to support multiple threads
+// accessing and read-expanding the same trie (i.e. caching loaded trie nodes).
+type syncedTrie struct {
+	tr   *trie.SecureTrie
+	lock sync.Mutex // Any operation on a trie can expand it (RWMutex is not enough)
+}
+
+func (s *syncedTrie) Copy() *syncedTrie {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return &syncedTrie{tr: s.tr.Copy()}
+}
+
+func (s *syncedTrie) TryGet(key []byte) ([]byte, error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.TryGet(key)
+}
+
+func (s *syncedTrie) TryUpdate(key, value []byte) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.TryUpdate(key, value)
+}
+
+func (s *syncedTrie) TryDelete(key []byte) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.TryDelete(key)
+}
+
+func (s *syncedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.Commit(onleaf)
+}
+
+func (s *syncedTrie) Hash() common.Hash {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.Hash()
+}
+
+func (s *syncedTrie) NodeIterator(startKey []byte) trie.NodeIterator {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.NodeIterator(startKey)
+}
+
+func (s *syncedTrie) GetKey(key []byte) []byte {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.GetKey(key)
+}
+
+func (s *syncedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.tr.Prove(key, fromLevel, proofDb)
+}
+
+// cachedStorageTrie is a wrapper around a read-only trie shared by possibly
+// multiple goroutines and a write-enabled trie unique to each goroutine.
+//
+// Important! Reads always go through the expand-only disk-backed trie whereas
+// writes go through the write-enabled one. This wrapper assumes that any trie
+// item read and modified will not be read again, rather used from a higher cache.
+type cachedStorageTrie struct {
+	reader *syncedTrie // Original trie for expand-only operations
+	writer *syncedTrie // Copy trie for update and delete operations
+
+	db *cachingDB
+}
+
+func (c *cachedStorageTrie) TryGet(key []byte) ([]byte, error) {
+	res, err := c.reader.TryGet(key)
+	if c.writer != nil {
+		return c.writer.TryGet(key)
+	}
+	return res, err
+}
+
+func (c *cachedStorageTrie) TryUpdate(key, value []byte) error {
+	if c.writer == nil {
+		c.writer = c.reader.Copy()
+	}
+	return c.writer.TryUpdate(key, value)
+}
+
+func (c *cachedStorageTrie) TryDelete(key []byte) error {
+	if c.writer == nil {
+		c.writer = c.reader.Copy()
+	}
+	return c.writer.TryDelete(key)
+}
+
+func (c *cachedStorageTrie) Hash() common.Hash {
+	if c.writer != nil {
+		return c.writer.Hash()
+	}
+	return c.reader.Hash()
+}
+
+func (c *cachedStorageTrie) NodeIterator(startKey []byte) trie.NodeIterator {
+	if c.writer != nil {
+		return c.writer.NodeIterator(startKey)
+	}
+	return c.reader.NodeIterator(startKey)
+}
+
+func (c *cachedStorageTrie) GetKey(key []byte) []byte {
+	if c.writer != nil {
+		return c.writer.GetKey(key)
+	}
+	return c.reader.GetKey(key)
+}
+
+func (c *cachedStorageTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
+	if c.writer != nil {
+		return c.writer.Prove(key, fromLevel, proofDb)
+	}
+	return c.reader.Prove(key, fromLevel, proofDb)
+}
+
+func (c *cachedStorageTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
+	// Retrieve the hash of the read-only trie (should be free)
+	origin, err := c.reader.Commit(nil)
+
+	// If there have been modifications made, push the writer into the cache
+	if c.writer != nil {
+		updated, err := c.writer.Commit(onleaf)
+		if err == nil && updated != origin {
+			c.db.pushStorageTrie(c.writer)
+		}
+		return updated, err
+	}
+	return origin, err
+}

core/state/statedb.go

@@ -37,8 +37,8 @@ type revision struct {
 }
 
 var (
-	// emptyState is the known hash of an empty state trie entry.
-	emptyState = crypto.Keccak256Hash(nil)
+	// emptyRoot is the known root hash of an empty trie.
+	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
 
 	// emptyCode is the known hash of the empty EVM bytecode.
 	emptyCode = crypto.Keccak256Hash(nil)
@@ -653,7 +653,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
 		if err := rlp.DecodeBytes(leaf, &account); err != nil {
 			return nil
 		}
-		if account.Root != emptyState {
+		if account.Root != emptyRoot {
 			s.db.TrieDB().Reference(account.Root, parent)
 		}
 		code := common.BytesToHash(account.CodeHash)