core/state: expose the state reader stats (#31998)

This pull request introduces a mechanism to expose statistics from the
state reader, specifically related to cache utilization during state prefetching.

To improve state access performance, a pair of state readers is constructed 
with a shared local cache. One reader to execute transactions  ahead of time
to warm up the cache. The other reader is used by the actual chain processing 
logic, which can benefit from the prefetched states.

This PR adds visibility into how effective the cache is by exposing relevant 
usage statistics.

---------

Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
Co-authored-by: Csaba Kiraly <csaba.kiraly@gmail.com>

core/blockchain.go

@@ -77,6 +77,16 @@ var (
 	storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil)
 	storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil)
 
+	accountCacheHitMeter  = metrics.NewRegisteredMeter("chain/account/reads/cache/process/hit", nil)
+	accountCacheMissMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/process/miss", nil)
+	storageCacheHitMeter  = metrics.NewRegisteredMeter("chain/storage/reads/cache/process/hit", nil)
+	storageCacheMissMeter = metrics.NewRegisteredMeter("chain/storage/reads/cache/process/miss", nil)
+
+	accountCacheHitPrefetchMeter  = metrics.NewRegisteredMeter("chain/account/reads/cache/prefetch/hit", nil)
+	accountCacheMissPrefetchMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/prefetch/miss", nil)
+	storageCacheHitPrefetchMeter  = metrics.NewRegisteredMeter("chain/storage/reads/cache/prefetch/hit", nil)
+	storageCacheMissPrefetchMeter = metrics.NewRegisteredMeter("chain/storage/reads/cache/prefetch/miss", nil)
+
 	accountReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/account/single/reads", nil)
 	storageReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/storage/single/reads", nil)
 
@@ -1944,18 +1954,32 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
 		//
 		// Note: the main processor and prefetcher share the same reader with a local
 		// cache for mitigating the overhead of state access.
-		reader, err := bc.statedb.ReaderWithCache(parentRoot)
+		prefetch, process, err := bc.statedb.ReadersWithCacheStats(parentRoot)
 		if err != nil {
 			return nil, err
 		}
-		throwaway, err := state.NewWithReader(parentRoot, bc.statedb, reader)
+		throwaway, err := state.NewWithReader(parentRoot, bc.statedb, prefetch)
 		if err != nil {
 			return nil, err
 		}
-		statedb, err = state.NewWithReader(parentRoot, bc.statedb, reader)
+		statedb, err = state.NewWithReader(parentRoot, bc.statedb, process)
 		if err != nil {
 			return nil, err
 		}
+		// Upload the statistics of reader at the end
+		defer func() {
+			stats := prefetch.GetStats()
+			accountCacheHitPrefetchMeter.Mark(stats.AccountHit)
+			accountCacheMissPrefetchMeter.Mark(stats.AccountMiss)
+			storageCacheHitPrefetchMeter.Mark(stats.StorageHit)
+			storageCacheMissPrefetchMeter.Mark(stats.StorageMiss)
+			stats = process.GetStats()
+			accountCacheHitMeter.Mark(stats.AccountHit)
+			accountCacheMissMeter.Mark(stats.AccountMiss)
+			storageCacheHitMeter.Mark(stats.StorageHit)
+			storageCacheMissMeter.Mark(stats.StorageMiss)
+		}()
+
 		go func(start time.Time, throwaway *state.StateDB, block *types.Block) {
 			// Disable tracing for prefetcher executions.
 			vmCfg := bc.cfg.VmConfig

core/state/database.go

@@ -209,13 +209,16 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
 	return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil
 }
 
-// ReaderWithCache creates a state reader with internal local cache.
-func (db *CachingDB) ReaderWithCache(stateRoot common.Hash) (Reader, error) {
+// ReadersWithCacheStats creates a pair of state readers sharing the same internal cache and
+// same backing Reader, but exposing separate statistics.
+// and statistics.
+func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithStats, ReaderWithStats, error) {
 	reader, err := db.Reader(stateRoot)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return newReaderWithCache(reader), nil
+	shared := newReaderWithCache(reader)
+	return newReaderWithCacheStats(shared), newReaderWithCacheStats(shared), nil
 }
 
 // OpenTrie opens the main account trie at a specific root hash.

core/state/reader.go

@@ -19,6 +19,7 @@ package state
 import (
 	"errors"
 	"sync"
+	"sync/atomic"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/lru"
@@ -82,6 +83,20 @@ type Reader interface {
 	StateReader
 }
 
+// ReaderStats wraps the statistics of reader.
+type ReaderStats struct {
+	AccountHit  int64
+	AccountMiss int64
+	StorageHit  int64
+	StorageMiss int64
+}
+
+// ReaderWithStats wraps the additional method to retrieve the reader statistics from.
+type ReaderWithStats interface {
+	Reader
+	GetStats() ReaderStats
+}
+
 // cachingCodeReader implements ContractCodeReader, accessing contract code either in
 // local key-value store or the shared code cache.
 //
@@ -414,35 +429,43 @@ func newReaderWithCache(reader Reader) *readerWithCache {
 	return r
 }
 
-// Account implements StateReader, retrieving the account specified by the address.
-// The returned account might be nil if it's not existent.
+// account retrieves the account specified by the address along with a flag
+// indicating whether it's found in the cache or not. The returned account
+// might be nil if it's not existent.
 //
 // An error will be returned if the state is corrupted in the underlying reader.
-func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, error) {
+func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, bool, error) {
 	// Try to resolve the requested account in the local cache
 	r.accountLock.RLock()
 	acct, ok := r.accounts[addr]
 	r.accountLock.RUnlock()
 	if ok {
-		return acct, nil
+		return acct, true, nil
 	}
 	// Try to resolve the requested account from the underlying reader
 	acct, err := r.Reader.Account(addr)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	r.accountLock.Lock()
 	r.accounts[addr] = acct
 	r.accountLock.Unlock()
-	return acct, nil
+	return acct, false, nil
 }
 
-// Storage implements StateReader, retrieving the storage slot specified by the
-// address and slot key. The returned storage slot might be empty if it's not
-// existent.
+// Account implements StateReader, retrieving the account specified by the address.
+// The returned account might be nil if it's not existent.
 //
 // An error will be returned if the state is corrupted in the underlying reader.
-func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
+func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, error) {
+	account, _, err := r.account(addr)
+	return account, err
+}
+
+// storage retrieves the storage slot specified by the address and slot key, along
+// with a flag indicating whether it's found in the cache or not. The returned
+// storage slot might be empty if it's not existent.
+func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) {
 	var (
 		value  common.Hash
 		ok     bool
@@ -456,12 +479,12 @@ func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common
 	}
 	bucket.lock.RUnlock()
 	if ok {
-		return value, nil
+		return value, true, nil
 	}
 	// Try to resolve the requested storage slot from the underlying reader
 	value, err := r.Reader.Storage(addr, slot)
 	if err != nil {
-		return common.Hash{}, err
+		return common.Hash{}, false, err
 	}
 	bucket.lock.Lock()
 	slots, ok = bucket.storages[addr]
@@ -472,5 +495,75 @@ func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common
 	slots[slot] = value
 	bucket.lock.Unlock()
 
+	return value, false, nil
+}
+
+// Storage implements StateReader, retrieving the storage slot specified by the
+// address and slot key. The returned storage slot might be empty if it's not
+// existent.
+//
+// An error will be returned if the state is corrupted in the underlying reader.
+func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
+	value, _, err := r.storage(addr, slot)
+	return value, err
+}
+
+type readerWithCacheStats struct {
+	*readerWithCache
+	accountHit  atomic.Int64
+	accountMiss atomic.Int64
+	storageHit  atomic.Int64
+	storageMiss atomic.Int64
+}
+
+// newReaderWithCacheStats constructs the reader with additional statistics tracked.
+func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats {
+	return &readerWithCacheStats{
+		readerWithCache: reader,
+	}
+}
+
+// Account implements StateReader, retrieving the account specified by the address.
+// The returned account might be nil if it's not existent.
+//
+// An error will be returned if the state is corrupted in the underlying reader.
+func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount, error) {
+	account, incache, err := r.readerWithCache.account(addr)
+	if err != nil {
+		return nil, err
+	}
+	if incache {
+		r.accountHit.Add(1)
+	} else {
+		r.accountMiss.Add(1)
+	}
+	return account, nil
+}
+
+// Storage implements StateReader, retrieving the storage slot specified by the
+// address and slot key. The returned storage slot might be empty if it's not
+// existent.
+//
+// An error will be returned if the state is corrupted in the underlying reader.
+func (r *readerWithCacheStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
+	value, incache, err := r.readerWithCache.storage(addr, slot)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	if incache {
+		r.storageHit.Add(1)
+	} else {
+		r.storageMiss.Add(1)
+	}
 	return value, nil
 }
+
+// GetStats implements ReaderWithStats, returning the statistics of state reader.
+func (r *readerWithCacheStats) GetStats() ReaderStats {
+	return ReaderStats{
+		AccountHit:  r.accountHit.Load(),
+		AccountMiss: r.accountMiss.Load(),
+		StorageHit:  r.storageHit.Load(),
+		StorageMiss: r.storageMiss.Load(),
+	}
+}