From 3b354570a7f7e5aa0a266f785b2304e2519bc680 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Wed, 20 Aug 2025 15:41:48 +0800 Subject: [PATCH] core,trie: rework the witness stats --- core/blockchain.go | 39 ++++---------- core/state/database.go | 4 -- core/state/statedb.go | 49 ++++++++++------- core/stateless/stats.go | 108 ++++++++++++++++++++++++++++++++++++++ core/stateless/witness.go | 78 ++++++--------------------- core/vm/interpreter.go | 1 + miner/worker.go | 2 +- trie/secure_trie.go | 4 -- trie/transition.go | 6 +-- trie/trie.go | 12 +---- trie/verkle.go | 4 -- 11 files changed, 168 insertions(+), 139 deletions(-) create mode 100644 core/stateless/stats.go diff --git a/core/blockchain.go b/core/blockchain.go index b56562b109..5205483af9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -112,13 +112,6 @@ var ( errChainStopped = errors.New("blockchain is stopped") errInvalidOldChain = errors.New("invalid old chain") errInvalidNewChain = errors.New("invalid new chain") - - accountAccessDepthAvg = metrics.NewRegisteredGauge("trie/access/account/avg", nil) - accountAccessDepthMin = metrics.NewRegisteredGauge("trie/access/account/min", nil) - accountAccessDepthMax = metrics.NewRegisteredGauge("trie/access/account/max", nil) - storageAccessDepthAvg = metrics.NewRegisteredGauge("trie/access/storage/avg", nil) - storageAccessDepthMin = metrics.NewRegisteredGauge("trie/access/storage/min", nil) - storageAccessDepthMax = metrics.NewRegisteredGauge("trie/access/storage/max", nil) ) var ( @@ -2018,7 +2011,10 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s // If we are past Byzantium, enable prefetching to pull in trie node paths // while processing transactions. Before Byzantium the prefetcher is mostly // useless due to the intermediate root hashing after each transaction. - var witness *stateless.Witness + var ( + witness *stateless.Witness + witnessStats *stateless.WitnessStats + ) if bc.chainConfig.IsByzantium(block.Number()) { // Generate witnesses either if we're self-testing, or if it's the // only block being inserted. A bit crude, but witnesses are huge, @@ -2028,8 +2024,11 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s if err != nil { return nil, err } + if bc.cfg.VmConfig.EnableWitnessStats { + witnessStats = stateless.NewWitnessStats() + } } - statedb.StartPrefetcher("chain", witness) + statedb.StartPrefetcher("chain", witness, witnessStats) defer statedb.StopPrefetcher() } @@ -2126,25 +2125,9 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s if err != nil { return nil, err } - - // If witness was generated, update metrics regarding the access paths. - if witness != nil { - a := witness.AccountDepthMetrics - if a.Count > 0 { - accountAccessDepthAvg.Update(int64(a.Sum) / int64(a.Count)) - if a.Min != -1 { - accountAccessDepthMin.Update(int64(a.Min)) - } - accountAccessDepthMax.Update(int64(a.Max)) - } - s := witness.StateDepthMetrics - if s.Count > 0 { - storageAccessDepthAvg.Update(int64(s.Sum) / int64(s.Count)) - if s.Min != -1 { - storageAccessDepthMin.Update(int64(s.Min)) - } - storageAccessDepthMax.Update(int64(s.Max)) - } + // Report the collected witness statistics + if witnessStats != nil { + witnessStats.ReportMetrics() } // Update the metrics touched during block commit diff --git a/core/state/database.go b/core/state/database.go index 4567a92ea1..3a0ac422ee 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -132,10 +132,6 @@ type Trie interface { // The returned map could be nil if the witness is empty. Witness() map[string][]byte - // Owner returns the owner of the trie, allowing us to distinguish between - // storage and account tries. - Owner() common.Hash - // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. And error will be returned // if fails to create node iterator. diff --git a/core/state/statedb.go b/core/state/statedb.go index 36f05887b2..688414e298 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -136,7 +136,8 @@ type StateDB struct { journal *journal // State witness if cross validation is needed - witness *stateless.Witness + witness *stateless.Witness + witnessStats *stateless.WitnessStats // Measurements gathered during execution for debugging purposes AccountReads time.Duration @@ -191,12 +192,13 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. -func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) { +func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness, witnessStats *stateless.WitnessStats) { // Terminate any previously running prefetcher s.StopPrefetcher() // Enable witness collection if requested s.witness = witness + s.witnessStats = witnessStats // With the switch to the Proof-of-Stake consensus algorithm, block production // rewards are now handled at the consensus layer. Consequently, a block may @@ -841,7 +843,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // If witness building is enabled and the state object has a trie, // gather the witnesses for its specific storage trie if s.witness != nil && obj.trie != nil { - s.witness.AddState(obj.trie.Witness(), obj.trie.Owner()) + s.witness.AddState(obj.trie.Witness()) } } return nil @@ -858,9 +860,17 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { continue } if trie := obj.getPrefetchedTrie(); trie != nil { - s.witness.AddState(trie.Witness(), trie.Owner()) + witness := trie.Witness() + s.witness.AddState(witness) + if s.witnessStats != nil { + s.witnessStats.Add(witness, obj.addrHash) + } } else if obj.trie != nil { - s.witness.AddState(obj.trie.Witness(), obj.trie.Owner()) + witness := obj.trie.Witness() + s.witness.AddState(witness) + if s.witnessStats != nil { + s.witnessStats.Add(witness, obj.addrHash) + } } } // Pull in only-read and non-destructed trie witnesses @@ -874,9 +884,17 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { continue } if trie := obj.getPrefetchedTrie(); trie != nil { - s.witness.AddState(trie.Witness(), trie.Owner()) + witness := trie.Witness() + s.witness.AddState(witness) + if s.witnessStats != nil { + s.witnessStats.Add(witness, obj.addrHash) + } } else if obj.trie != nil { - s.witness.AddState(obj.trie.Witness(), trie.Owner()) + witness := trie.Witness() + s.witness.AddState(witness) + if s.witnessStats != nil { + s.witnessStats.Add(witness, obj.addrHash) + } } } } @@ -942,7 +960,11 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // If witness building is enabled, gather the account trie witness if s.witness != nil { - s.witness.AddState(s.trie.Witness(), s.trie.Owner()) + witness := s.trie.Witness() + s.witness.AddState(witness) + if s.witnessStats != nil { + s.witnessStats.Add(witness, common.Hash{}) + } } return hash } @@ -1295,17 +1317,6 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag if err != nil { return nil, err } - - // This iterates through nodeset and tracks path depths for tracking - // state and account trie modifications. - if s.witness != nil && ret != nil && !ret.empty() { - for owner, set := range ret.nodes.Sets { - for path := range set.Origins { - s.witness.AddStateModify(len(path), owner) - } - } - } - // Commit dirty contract code if any exists if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { batch := db.NewBatch() diff --git a/core/stateless/stats.go b/core/stateless/stats.go new file mode 100644 index 0000000000..088e06c9bb --- /dev/null +++ b/core/stateless/stats.go @@ -0,0 +1,108 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stateless + +import ( + "maps" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + accountTrieDepthAvg = metrics.NewRegisteredGauge("witness/trie/account/depth/avg", nil) + accountTrieDepthMin = metrics.NewRegisteredGauge("witness/trie/account/depth/min", nil) + accountTrieDepthMax = metrics.NewRegisteredGauge("witness/trie/account/depth/max", nil) + + storageTrieDepthAvg = metrics.NewRegisteredGauge("witness/trie/storage/depth/avg", nil) + storageTrieDepthMin = metrics.NewRegisteredGauge("witness/trie/storage/depth/min", nil) + storageTrieDepthMax = metrics.NewRegisteredGauge("witness/trie/storage/depth/max", nil) +) + +// depthStats tracks min/avg/max statistics for trie access depths. +type depthStats struct { + totalDepth int64 + samples int64 + minDepth int64 + maxDepth int64 +} + +// newDepthStats creates a new depthStats with default values. +func newDepthStats() *depthStats { + return &depthStats{minDepth: -1} +} + +// Add records a new depth sample. +func (d *depthStats) Add(n int64) { + if n < 0 { + return + } + d.totalDepth += n + d.samples++ + + if d.minDepth == -1 || n < d.minDepth { + d.minDepth = n + } + if n > d.maxDepth { + d.maxDepth = n + } +} + +// report uploads the collected statistics into the provided gauges. +func (d *depthStats) report(maxGauge, minGauge, avgGauge *metrics.Gauge) { + if d.samples == 0 { + return + } + maxGauge.Update(d.maxDepth) + minGauge.Update(d.minDepth) + avgGauge.Update(d.totalDepth / d.samples) +} + +// WitnessStats aggregates statistics for account and storage trie accesses. +type WitnessStats struct { + accountTrie *depthStats + storageTrie *depthStats +} + +// NewWitnessStats creates a new WitnessStats collector. +func NewWitnessStats() *WitnessStats { + return &WitnessStats{ + accountTrie: newDepthStats(), + storageTrie: newDepthStats(), + } +} + +// Add records trie access depths from the given node paths. +// If `owner` is the zero hash, accesses are attributed to the account trie; +// otherwise, they are attributed to the storage trie of that account. +func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { + if owner == (common.Hash{}) { + for path := range maps.Keys(nodes) { + s.accountTrie.Add(int64(len(path))) + } + } else { + for path := range maps.Keys(nodes) { + s.storageTrie.Add(int64(len(path))) + } + } +} + +// ReportMetrics reports the collected statistics to the global metrics registry. +func (s *WitnessStats) ReportMetrics() { + s.accountTrie.report(accountTrieDepthMax, accountTrieDepthMin, accountTrieDepthAvg) + s.storageTrie.report(storageTrieDepthMax, storageTrieDepthMin, storageTrieDepthAvg) +} diff --git a/core/stateless/witness.go b/core/stateless/witness.go index 3316767c8d..371a128f48 100644 --- a/core/stateless/witness.go +++ b/core/stateless/witness.go @@ -33,13 +33,6 @@ type HeaderReader interface { GetHeader(hash common.Hash, number uint64) *types.Header } -type DepthTracker struct { - Sum int - Count int - Min int - Max int -} - // Witness encompasses the state required to apply a set of transactions and // derive a post state/receipt root. type Witness struct { @@ -49,10 +42,8 @@ type Witness struct { Codes map[string]struct{} // Set of bytecodes ran or accessed State map[string]struct{} // Set of MPT state trie nodes (account and storage together) - StateDepthMetrics DepthTracker // Metrics about the state trie paths, used for debugging - AccountDepthMetrics DepthTracker // Metrics about the account trie paths, used for debugging - chain HeaderReader // Chain reader to convert block hash ops to header proofs - lock sync.Mutex // Lock to allow concurrent state insertions + chain HeaderReader // Chain reader to convert block hash ops to header proofs + lock sync.Mutex // Lock to allow concurrent state insertions } // NewWitness creates an empty witness ready for population. @@ -67,35 +58,16 @@ func NewWitness(context *types.Header, chain HeaderReader) (*Witness, error) { } headers = append(headers, parent) } - - var stateDepthTracker = DepthTracker{Sum: 0, Count: 0, Min: -1, Max: 0} - var accountDepthTracker = DepthTracker{Sum: 0, Count: 0, Min: -1, Max: 0} // Create the witness with a reconstructed gutted out block return &Witness{ - context: context, - Headers: headers, - Codes: make(map[string]struct{}), - State: make(map[string]struct{}), - StateDepthMetrics: stateDepthTracker, - AccountDepthMetrics: accountDepthTracker, - chain: chain, + context: context, + Headers: headers, + Codes: make(map[string]struct{}), + State: make(map[string]struct{}), + chain: chain, }, nil } -func (d *DepthTracker) Add(n int) { - if n < 0 { - n = 0 - } - d.Sum += n - d.Count++ - if d.Min == -1 || n < d.Min { - d.Min = n - } - if n > d.Max { - d.Max = n - } -} - // AddBlockHash adds a "blockhash" to the witness with the designated offset from // chain head. Under the hood, this method actually pulls in enough headers from // the chain to cover the block being added. @@ -116,33 +88,15 @@ func (w *Witness) AddCode(code []byte) { } // AddState inserts a batch of MPT trie nodes into the witness. -func (w *Witness) AddState(nodemap map[string][]byte, owner common.Hash) { - if len(nodemap) == 0 { +func (w *Witness) AddState(nodes map[string][]byte) { + if len(nodes) == 0 { return } w.lock.Lock() defer w.lock.Unlock() - for path, value := range nodemap { - w.State[string(value)] = struct{}{} - if owner != (common.Hash{}) { - w.AccountDepthMetrics.Add(len(path)) - } else { - w.StateDepthMetrics.Add(len(path)) - } - } -} -// AddStateModify tracks the modification of a node in the witness. -func (w *Witness) AddStateModify(path int, owner common.Hash) { - if path < 0 { - return - } - w.lock.Lock() - defer w.lock.Unlock() - if owner != (common.Hash{}) { - w.AccountDepthMetrics.Add(path) - } else { - w.StateDepthMetrics.Add(path) + for _, value := range nodes { + w.State[string(value)] = struct{}{} } } @@ -150,12 +104,10 @@ func (w *Witness) AddStateModify(path int, owner common.Hash) { // is never mutated by Witness func (w *Witness) Copy() *Witness { cpy := &Witness{ - Headers: slices.Clone(w.Headers), - Codes: maps.Clone(w.Codes), - State: maps.Clone(w.State), - StateDepthMetrics: w.StateDepthMetrics, - AccountDepthMetrics: w.AccountDepthMetrics, - chain: w.chain, + Headers: slices.Clone(w.Headers), + Codes: maps.Clone(w.Codes), + State: maps.Clone(w.State), + chain: w.chain, } if w.context != nil { cpy.context = types.CopyHeader(w.context) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index a0637a6800..52dbe83d86 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -33,6 +33,7 @@ type Config struct { ExtraEips []int // Additional EIPS that are to be enabled StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) + EnableWitnessStats bool // Whether trie access statistics collection is enabled } // ScopeContext contains the things that are per-call, such as stack and memory, diff --git a/miner/worker.go b/miner/worker.go index 5405fb24b9..0e2560f844 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -271,7 +271,7 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase if err != nil { return nil, err } - state.StartPrefetcher("miner", bundle) + state.StartPrefetcher("miner", bundle, nil) } // Note the passed coinbase may be different with header.Coinbase. return &environment{ diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 4fb21d20ad..7c7bd184bf 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -96,10 +96,6 @@ func NewStateTrie(id *ID, db database.NodeDatabase) (*StateTrie, error) { return tr, nil } -func (t *StateTrie) Owner() common.Hash { - return t.trie.Owner() -} - // MustGet returns the value for key stored in the trie. // The value bytes must not be modified by the caller. // diff --git a/trie/transition.go b/trie/transition.go index e8962e106b..0e82cb2627 100644 --- a/trie/transition.go +++ b/trie/transition.go @@ -45,10 +45,6 @@ func NewTransitionTree(base *SecureTrie, overlay *VerkleTrie, st bool) *Transiti } } -func (t *TransitionTrie) Owner() common.Hash { - return t.overlay.Owner() -} - // Base returns the base trie. func (t *TransitionTrie) Base() *SecureTrie { return t.base @@ -226,6 +222,6 @@ func (t *TransitionTrie) UpdateContractCode(addr common.Address, codeHash common } // Witness returns a set containing all trie nodes that have been accessed. -func (t *TransitionTrie) Witness() map[string]struct{} { +func (t *TransitionTrie) Witness() map[string][]byte { panic("not implemented") } diff --git a/trie/trie.go b/trie/trie.go index c7e2e85642..fe8b5837d4 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -133,12 +133,6 @@ func (t *Trie) NodeIterator(start []byte) (NodeIterator, error) { return newNodeIterator(t, start), nil } -// Owner returns the owner of the trie, allowing us to distinguis between -// storage and account tries. -func (t *Trie) Owner() common.Hash { - return t.owner -} - // MustGet is a wrapper of Get and will omit any encountered error but just // print out an error message. func (t *Trie) MustGet(key []byte) []byte { @@ -765,11 +759,7 @@ func (t *Trie) hashRoot() []byte { // Witness returns a set containing all trie nodes that have been accessed. func (t *Trie) Witness() map[string][]byte { - dataMap := t.prevalueTracer.getMap() - if len(dataMap) == 0 { - return nil - } - return dataMap + return t.prevalueTracer.values() } // Reset drops the referenced root node and cleans all internal state. diff --git a/trie/verkle.go b/trie/verkle.go index 3ad638f089..e00ea21602 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -78,10 +78,6 @@ func (t *VerkleTrie) GetKey(key []byte) []byte { return key } -func (t *VerkleTrie) Owner() common.Hash { - panic("VerkleTrie does not have an owner") -} - // GetAccount implements state.Trie, retrieving the account with the specified // account address. If the specified account is not in the verkle tree, nil will // be returned. If the tree is corrupted, an error will be returned.