core,trie: rework the witness stats

This commit is contained in:
Gary Rong 2025-08-20 15:41:48 +08:00
parent 23d24f3ce3
commit 3b354570a7
11 changed files with 168 additions and 139 deletions

View file

@ -112,13 +112,6 @@ var (
errChainStopped = errors.New("blockchain is stopped") errChainStopped = errors.New("blockchain is stopped")
errInvalidOldChain = errors.New("invalid old chain") errInvalidOldChain = errors.New("invalid old chain")
errInvalidNewChain = errors.New("invalid new chain") errInvalidNewChain = errors.New("invalid new chain")
accountAccessDepthAvg = metrics.NewRegisteredGauge("trie/access/account/avg", nil)
accountAccessDepthMin = metrics.NewRegisteredGauge("trie/access/account/min", nil)
accountAccessDepthMax = metrics.NewRegisteredGauge("trie/access/account/max", nil)
storageAccessDepthAvg = metrics.NewRegisteredGauge("trie/access/storage/avg", nil)
storageAccessDepthMin = metrics.NewRegisteredGauge("trie/access/storage/min", nil)
storageAccessDepthMax = metrics.NewRegisteredGauge("trie/access/storage/max", nil)
) )
var ( var (
@ -2018,7 +2011,10 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
// If we are past Byzantium, enable prefetching to pull in trie node paths // If we are past Byzantium, enable prefetching to pull in trie node paths
// while processing transactions. Before Byzantium the prefetcher is mostly // while processing transactions. Before Byzantium the prefetcher is mostly
// useless due to the intermediate root hashing after each transaction. // useless due to the intermediate root hashing after each transaction.
var witness *stateless.Witness var (
witness *stateless.Witness
witnessStats *stateless.WitnessStats
)
if bc.chainConfig.IsByzantium(block.Number()) { if bc.chainConfig.IsByzantium(block.Number()) {
// Generate witnesses either if we're self-testing, or if it's the // Generate witnesses either if we're self-testing, or if it's the
// only block being inserted. A bit crude, but witnesses are huge, // only block being inserted. A bit crude, but witnesses are huge,
@ -2028,8 +2024,11 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
if err != nil { if err != nil {
return nil, err return nil, err
} }
if bc.cfg.VmConfig.EnableWitnessStats {
witnessStats = stateless.NewWitnessStats()
} }
statedb.StartPrefetcher("chain", witness) }
statedb.StartPrefetcher("chain", witness, witnessStats)
defer statedb.StopPrefetcher() defer statedb.StopPrefetcher()
} }
@ -2126,25 +2125,9 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Report the collected witness statistics
// If witness was generated, update metrics regarding the access paths. if witnessStats != nil {
if witness != nil { witnessStats.ReportMetrics()
a := witness.AccountDepthMetrics
if a.Count > 0 {
accountAccessDepthAvg.Update(int64(a.Sum) / int64(a.Count))
if a.Min != -1 {
accountAccessDepthMin.Update(int64(a.Min))
}
accountAccessDepthMax.Update(int64(a.Max))
}
s := witness.StateDepthMetrics
if s.Count > 0 {
storageAccessDepthAvg.Update(int64(s.Sum) / int64(s.Count))
if s.Min != -1 {
storageAccessDepthMin.Update(int64(s.Min))
}
storageAccessDepthMax.Update(int64(s.Max))
}
} }
// Update the metrics touched during block commit // Update the metrics touched during block commit

View file

@ -132,10 +132,6 @@ type Trie interface {
// The returned map could be nil if the witness is empty. // The returned map could be nil if the witness is empty.
Witness() map[string][]byte Witness() map[string][]byte
// Owner returns the owner of the trie, allowing us to distinguish between
// storage and account tries.
Owner() common.Hash
// NodeIterator returns an iterator that returns nodes of the trie. Iteration // NodeIterator returns an iterator that returns nodes of the trie. Iteration
// starts at the key after the given start key. And error will be returned // starts at the key after the given start key. And error will be returned
// if fails to create node iterator. // if fails to create node iterator.

View file

@ -137,6 +137,7 @@ type StateDB struct {
// State witness if cross validation is needed // State witness if cross validation is needed
witness *stateless.Witness witness *stateless.Witness
witnessStats *stateless.WitnessStats
// Measurements gathered during execution for debugging purposes // Measurements gathered during execution for debugging purposes
AccountReads time.Duration AccountReads time.Duration
@ -191,12 +192,13 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro
// StartPrefetcher initializes a new trie prefetcher to pull in nodes from the // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the
// state trie concurrently while the state is mutated so that when we reach the // state trie concurrently while the state is mutated so that when we reach the
// commit phase, most of the needed data is already hot. // commit phase, most of the needed data is already hot.
func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) { func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness, witnessStats *stateless.WitnessStats) {
// Terminate any previously running prefetcher // Terminate any previously running prefetcher
s.StopPrefetcher() s.StopPrefetcher()
// Enable witness collection if requested // Enable witness collection if requested
s.witness = witness s.witness = witness
s.witnessStats = witnessStats
// With the switch to the Proof-of-Stake consensus algorithm, block production // With the switch to the Proof-of-Stake consensus algorithm, block production
// rewards are now handled at the consensus layer. Consequently, a block may // rewards are now handled at the consensus layer. Consequently, a block may
@ -841,7 +843,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
// If witness building is enabled and the state object has a trie, // If witness building is enabled and the state object has a trie,
// gather the witnesses for its specific storage trie // gather the witnesses for its specific storage trie
if s.witness != nil && obj.trie != nil { if s.witness != nil && obj.trie != nil {
s.witness.AddState(obj.trie.Witness(), obj.trie.Owner()) s.witness.AddState(obj.trie.Witness())
} }
} }
return nil return nil
@ -858,9 +860,17 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
continue continue
} }
if trie := obj.getPrefetchedTrie(); trie != nil { if trie := obj.getPrefetchedTrie(); trie != nil {
s.witness.AddState(trie.Witness(), trie.Owner()) witness := trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
} else if obj.trie != nil { } else if obj.trie != nil {
s.witness.AddState(obj.trie.Witness(), obj.trie.Owner()) witness := obj.trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
} }
} }
// Pull in only-read and non-destructed trie witnesses // Pull in only-read and non-destructed trie witnesses
@ -874,9 +884,17 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
continue continue
} }
if trie := obj.getPrefetchedTrie(); trie != nil { if trie := obj.getPrefetchedTrie(); trie != nil {
s.witness.AddState(trie.Witness(), trie.Owner()) witness := trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
} else if obj.trie != nil { } else if obj.trie != nil {
s.witness.AddState(obj.trie.Witness(), trie.Owner()) witness := trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
} }
} }
} }
@ -942,7 +960,11 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
// If witness building is enabled, gather the account trie witness // If witness building is enabled, gather the account trie witness
if s.witness != nil { if s.witness != nil {
s.witness.AddState(s.trie.Witness(), s.trie.Owner()) witness := s.trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, common.Hash{})
}
} }
return hash return hash
} }
@ -1295,17 +1317,6 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag
if err != nil { if err != nil {
return nil, err return nil, err
} }
// This iterates through nodeset and tracks path depths for tracking
// state and account trie modifications.
if s.witness != nil && ret != nil && !ret.empty() {
for owner, set := range ret.nodes.Sets {
for path := range set.Origins {
s.witness.AddStateModify(len(path), owner)
}
}
}
// Commit dirty contract code if any exists // Commit dirty contract code if any exists
if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 {
batch := db.NewBatch() batch := db.NewBatch()

108
core/stateless/stats.go Normal file
View file

@ -0,0 +1,108 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stateless
import (
"maps"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/metrics"
)
var (
accountTrieDepthAvg = metrics.NewRegisteredGauge("witness/trie/account/depth/avg", nil)
accountTrieDepthMin = metrics.NewRegisteredGauge("witness/trie/account/depth/min", nil)
accountTrieDepthMax = metrics.NewRegisteredGauge("witness/trie/account/depth/max", nil)
storageTrieDepthAvg = metrics.NewRegisteredGauge("witness/trie/storage/depth/avg", nil)
storageTrieDepthMin = metrics.NewRegisteredGauge("witness/trie/storage/depth/min", nil)
storageTrieDepthMax = metrics.NewRegisteredGauge("witness/trie/storage/depth/max", nil)
)
// depthStats tracks min/avg/max statistics for trie access depths.
type depthStats struct {
totalDepth int64
samples int64
minDepth int64
maxDepth int64
}
// newDepthStats creates a new depthStats with default values.
func newDepthStats() *depthStats {
return &depthStats{minDepth: -1}
}
// Add records a new depth sample.
func (d *depthStats) Add(n int64) {
if n < 0 {
return
}
d.totalDepth += n
d.samples++
if d.minDepth == -1 || n < d.minDepth {
d.minDepth = n
}
if n > d.maxDepth {
d.maxDepth = n
}
}
// report uploads the collected statistics into the provided gauges.
func (d *depthStats) report(maxGauge, minGauge, avgGauge *metrics.Gauge) {
if d.samples == 0 {
return
}
maxGauge.Update(d.maxDepth)
minGauge.Update(d.minDepth)
avgGauge.Update(d.totalDepth / d.samples)
}
// WitnessStats aggregates statistics for account and storage trie accesses.
type WitnessStats struct {
accountTrie *depthStats
storageTrie *depthStats
}
// NewWitnessStats creates a new WitnessStats collector.
func NewWitnessStats() *WitnessStats {
return &WitnessStats{
accountTrie: newDepthStats(),
storageTrie: newDepthStats(),
}
}
// Add records trie access depths from the given node paths.
// If `owner` is the zero hash, accesses are attributed to the account trie;
// otherwise, they are attributed to the storage trie of that account.
func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) {
if owner == (common.Hash{}) {
for path := range maps.Keys(nodes) {
s.accountTrie.Add(int64(len(path)))
}
} else {
for path := range maps.Keys(nodes) {
s.storageTrie.Add(int64(len(path)))
}
}
}
// ReportMetrics reports the collected statistics to the global metrics registry.
func (s *WitnessStats) ReportMetrics() {
s.accountTrie.report(accountTrieDepthMax, accountTrieDepthMin, accountTrieDepthAvg)
s.storageTrie.report(storageTrieDepthMax, storageTrieDepthMin, storageTrieDepthAvg)
}

View file

@ -33,13 +33,6 @@ type HeaderReader interface {
GetHeader(hash common.Hash, number uint64) *types.Header GetHeader(hash common.Hash, number uint64) *types.Header
} }
type DepthTracker struct {
Sum int
Count int
Min int
Max int
}
// Witness encompasses the state required to apply a set of transactions and // Witness encompasses the state required to apply a set of transactions and
// derive a post state/receipt root. // derive a post state/receipt root.
type Witness struct { type Witness struct {
@ -49,8 +42,6 @@ type Witness struct {
Codes map[string]struct{} // Set of bytecodes ran or accessed Codes map[string]struct{} // Set of bytecodes ran or accessed
State map[string]struct{} // Set of MPT state trie nodes (account and storage together) State map[string]struct{} // Set of MPT state trie nodes (account and storage together)
StateDepthMetrics DepthTracker // Metrics about the state trie paths, used for debugging
AccountDepthMetrics DepthTracker // Metrics about the account trie paths, used for debugging
chain HeaderReader // Chain reader to convert block hash ops to header proofs chain HeaderReader // Chain reader to convert block hash ops to header proofs
lock sync.Mutex // Lock to allow concurrent state insertions lock sync.Mutex // Lock to allow concurrent state insertions
} }
@ -67,35 +58,16 @@ func NewWitness(context *types.Header, chain HeaderReader) (*Witness, error) {
} }
headers = append(headers, parent) headers = append(headers, parent)
} }
var stateDepthTracker = DepthTracker{Sum: 0, Count: 0, Min: -1, Max: 0}
var accountDepthTracker = DepthTracker{Sum: 0, Count: 0, Min: -1, Max: 0}
// Create the witness with a reconstructed gutted out block // Create the witness with a reconstructed gutted out block
return &Witness{ return &Witness{
context: context, context: context,
Headers: headers, Headers: headers,
Codes: make(map[string]struct{}), Codes: make(map[string]struct{}),
State: make(map[string]struct{}), State: make(map[string]struct{}),
StateDepthMetrics: stateDepthTracker,
AccountDepthMetrics: accountDepthTracker,
chain: chain, chain: chain,
}, nil }, nil
} }
func (d *DepthTracker) Add(n int) {
if n < 0 {
n = 0
}
d.Sum += n
d.Count++
if d.Min == -1 || n < d.Min {
d.Min = n
}
if n > d.Max {
d.Max = n
}
}
// AddBlockHash adds a "blockhash" to the witness with the designated offset from // AddBlockHash adds a "blockhash" to the witness with the designated offset from
// chain head. Under the hood, this method actually pulls in enough headers from // chain head. Under the hood, this method actually pulls in enough headers from
// the chain to cover the block being added. // the chain to cover the block being added.
@ -116,33 +88,15 @@ func (w *Witness) AddCode(code []byte) {
} }
// AddState inserts a batch of MPT trie nodes into the witness. // AddState inserts a batch of MPT trie nodes into the witness.
func (w *Witness) AddState(nodemap map[string][]byte, owner common.Hash) { func (w *Witness) AddState(nodes map[string][]byte) {
if len(nodemap) == 0 { if len(nodes) == 0 {
return return
} }
w.lock.Lock() w.lock.Lock()
defer w.lock.Unlock() defer w.lock.Unlock()
for path, value := range nodemap {
w.State[string(value)] = struct{}{}
if owner != (common.Hash{}) {
w.AccountDepthMetrics.Add(len(path))
} else {
w.StateDepthMetrics.Add(len(path))
}
}
}
// AddStateModify tracks the modification of a node in the witness. for _, value := range nodes {
func (w *Witness) AddStateModify(path int, owner common.Hash) { w.State[string(value)] = struct{}{}
if path < 0 {
return
}
w.lock.Lock()
defer w.lock.Unlock()
if owner != (common.Hash{}) {
w.AccountDepthMetrics.Add(path)
} else {
w.StateDepthMetrics.Add(path)
} }
} }
@ -153,8 +107,6 @@ func (w *Witness) Copy() *Witness {
Headers: slices.Clone(w.Headers), Headers: slices.Clone(w.Headers),
Codes: maps.Clone(w.Codes), Codes: maps.Clone(w.Codes),
State: maps.Clone(w.State), State: maps.Clone(w.State),
StateDepthMetrics: w.StateDepthMetrics,
AccountDepthMetrics: w.AccountDepthMetrics,
chain: w.chain, chain: w.chain,
} }
if w.context != nil { if w.context != nil {

View file

@ -33,6 +33,7 @@ type Config struct {
ExtraEips []int // Additional EIPS that are to be enabled ExtraEips []int // Additional EIPS that are to be enabled
StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose)
EnableWitnessStats bool // Whether trie access statistics collection is enabled
} }
// ScopeContext contains the things that are per-call, such as stack and memory, // ScopeContext contains the things that are per-call, such as stack and memory,

View file

@ -271,7 +271,7 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase
if err != nil { if err != nil {
return nil, err return nil, err
} }
state.StartPrefetcher("miner", bundle) state.StartPrefetcher("miner", bundle, nil)
} }
// Note the passed coinbase may be different with header.Coinbase. // Note the passed coinbase may be different with header.Coinbase.
return &environment{ return &environment{

View file

@ -96,10 +96,6 @@ func NewStateTrie(id *ID, db database.NodeDatabase) (*StateTrie, error) {
return tr, nil return tr, nil
} }
func (t *StateTrie) Owner() common.Hash {
return t.trie.Owner()
}
// MustGet returns the value for key stored in the trie. // MustGet returns the value for key stored in the trie.
// The value bytes must not be modified by the caller. // The value bytes must not be modified by the caller.
// //

View file

@ -45,10 +45,6 @@ func NewTransitionTree(base *SecureTrie, overlay *VerkleTrie, st bool) *Transiti
} }
} }
func (t *TransitionTrie) Owner() common.Hash {
return t.overlay.Owner()
}
// Base returns the base trie. // Base returns the base trie.
func (t *TransitionTrie) Base() *SecureTrie { func (t *TransitionTrie) Base() *SecureTrie {
return t.base return t.base
@ -226,6 +222,6 @@ func (t *TransitionTrie) UpdateContractCode(addr common.Address, codeHash common
} }
// Witness returns a set containing all trie nodes that have been accessed. // Witness returns a set containing all trie nodes that have been accessed.
func (t *TransitionTrie) Witness() map[string]struct{} { func (t *TransitionTrie) Witness() map[string][]byte {
panic("not implemented") panic("not implemented")
} }

View file

@ -133,12 +133,6 @@ func (t *Trie) NodeIterator(start []byte) (NodeIterator, error) {
return newNodeIterator(t, start), nil return newNodeIterator(t, start), nil
} }
// Owner returns the owner of the trie, allowing us to distinguis between
// storage and account tries.
func (t *Trie) Owner() common.Hash {
return t.owner
}
// MustGet is a wrapper of Get and will omit any encountered error but just // MustGet is a wrapper of Get and will omit any encountered error but just
// print out an error message. // print out an error message.
func (t *Trie) MustGet(key []byte) []byte { func (t *Trie) MustGet(key []byte) []byte {
@ -765,11 +759,7 @@ func (t *Trie) hashRoot() []byte {
// Witness returns a set containing all trie nodes that have been accessed. // Witness returns a set containing all trie nodes that have been accessed.
func (t *Trie) Witness() map[string][]byte { func (t *Trie) Witness() map[string][]byte {
dataMap := t.prevalueTracer.getMap() return t.prevalueTracer.values()
if len(dataMap) == 0 {
return nil
}
return dataMap
} }
// Reset drops the referenced root node and cleans all internal state. // Reset drops the referenced root node and cleans all internal state.

View file

@ -78,10 +78,6 @@ func (t *VerkleTrie) GetKey(key []byte) []byte {
return key return key
} }
func (t *VerkleTrie) Owner() common.Hash {
panic("VerkleTrie does not have an owner")
}
// GetAccount implements state.Trie, retrieving the account with the specified // GetAccount implements state.Trie, retrieving the account with the specified
// account address. If the specified account is not in the verkle tree, nil will // account address. If the specified account is not in the verkle tree, nil will
// be returned. If the tree is corrupted, an error will be returned. // be returned. If the tree is corrupted, an error will be returned.