From f986915332b357de35c62d1225f5a420b842a723 Mon Sep 17 00:00:00 2001 From: shantichanal <158101918+shantichanal@users.noreply.github.com> Date: Tue, 19 Aug 2025 19:47:45 +0200 Subject: [PATCH] Changes to keep track of writes and account trie statistics. Also reduced memory for witness by only keeping statistics. --- core/blockchain.go | 33 ++++++++++------- core/state/database.go | 4 ++ core/state/statedb.go | 23 +++++++++--- core/stateless/witness.go | 78 ++++++++++++++++++++++++++++++--------- trie/secure_trie.go | 9 ++--- trie/verkle.go | 4 ++ 6 files changed, 110 insertions(+), 41 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f40bcf6819..b56562b109 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -113,8 +113,12 @@ var ( errInvalidOldChain = errors.New("invalid old chain") errInvalidNewChain = errors.New("invalid new chain") - avgAccessDepthInBlock = metrics.NewRegisteredGauge("trie/access/depth/avg", nil) - minAccessDepthInBlock = metrics.NewRegisteredGauge("trie/access/depth/min", nil) + accountAccessDepthAvg = metrics.NewRegisteredGauge("trie/access/account/avg", nil) + accountAccessDepthMin = metrics.NewRegisteredGauge("trie/access/account/min", nil) + accountAccessDepthMax = metrics.NewRegisteredGauge("trie/access/account/max", nil) + storageAccessDepthAvg = metrics.NewRegisteredGauge("trie/access/storage/avg", nil) + storageAccessDepthMin = metrics.NewRegisteredGauge("trie/access/storage/min", nil) + storageAccessDepthMax = metrics.NewRegisteredGauge("trie/access/storage/max", nil) ) var ( @@ -2125,18 +2129,21 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s // If witness was generated, update metrics regarding the access paths. if witness != nil { - paths := witness.Paths - totaldepth, pathnum, mindepth := 0, 0, -1 - if len(paths) > 0 { - for path, _ := range paths { - if len(path) < mindepth || mindepth < 0 { - mindepth = len(path) - } - totaldepth += len(path) - pathnum++ + a := witness.AccountDepthMetrics + if a.Count > 0 { + accountAccessDepthAvg.Update(int64(a.Sum) / int64(a.Count)) + if a.Min != -1 { + accountAccessDepthMin.Update(int64(a.Min)) } - avgAccessDepthInBlock.Update(int64(totaldepth) / int64(pathnum)) - minAccessDepthInBlock.Update(int64(mindepth)) + accountAccessDepthMax.Update(int64(a.Max)) + } + s := witness.StateDepthMetrics + if s.Count > 0 { + storageAccessDepthAvg.Update(int64(s.Sum) / int64(s.Count)) + if s.Min != -1 { + storageAccessDepthMin.Update(int64(s.Min)) + } + storageAccessDepthMax.Update(int64(s.Max)) } } diff --git a/core/state/database.go b/core/state/database.go index 3a0ac422ee..4567a92ea1 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -132,6 +132,10 @@ type Trie interface { // The returned map could be nil if the witness is empty. Witness() map[string][]byte + // Owner returns the owner of the trie, allowing us to distinguish between + // storage and account tries. + Owner() common.Hash + // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. And error will be returned // if fails to create node iterator. diff --git a/core/state/statedb.go b/core/state/statedb.go index efb09a08a0..36f05887b2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -841,7 +841,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // If witness building is enabled and the state object has a trie, // gather the witnesses for its specific storage trie if s.witness != nil && obj.trie != nil { - s.witness.AddState(obj.trie.Witness()) + s.witness.AddState(obj.trie.Witness(), obj.trie.Owner()) } } return nil @@ -858,9 +858,9 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { continue } if trie := obj.getPrefetchedTrie(); trie != nil { - s.witness.AddState(trie.Witness()) + s.witness.AddState(trie.Witness(), trie.Owner()) } else if obj.trie != nil { - s.witness.AddState(obj.trie.Witness()) + s.witness.AddState(obj.trie.Witness(), obj.trie.Owner()) } } // Pull in only-read and non-destructed trie witnesses @@ -874,9 +874,9 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { continue } if trie := obj.getPrefetchedTrie(); trie != nil { - s.witness.AddState(trie.Witness()) + s.witness.AddState(trie.Witness(), trie.Owner()) } else if obj.trie != nil { - s.witness.AddState(obj.trie.Witness()) + s.witness.AddState(obj.trie.Witness(), trie.Owner()) } } } @@ -942,7 +942,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // If witness building is enabled, gather the account trie witness if s.witness != nil { - s.witness.AddState(s.trie.Witness()) + s.witness.AddState(s.trie.Witness(), s.trie.Owner()) } return hash } @@ -1295,6 +1295,17 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag if err != nil { return nil, err } + + // This iterates through nodeset and tracks path depths for tracking + // state and account trie modifications. + if s.witness != nil && ret != nil && !ret.empty() { + for owner, set := range ret.nodes.Sets { + for path := range set.Origins { + s.witness.AddStateModify(len(path), owner) + } + } + } + // Commit dirty contract code if any exists if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { batch := db.NewBatch() diff --git a/core/stateless/witness.go b/core/stateless/witness.go index 824513d29b..3316767c8d 100644 --- a/core/stateless/witness.go +++ b/core/stateless/witness.go @@ -33,6 +33,13 @@ type HeaderReader interface { GetHeader(hash common.Hash, number uint64) *types.Header } +type DepthTracker struct { + Sum int + Count int + Min int + Max int +} + // Witness encompasses the state required to apply a set of transactions and // derive a post state/receipt root. type Witness struct { @@ -41,10 +48,11 @@ type Witness struct { Headers []*types.Header // Past headers in reverse order (0=parent, 1=parent's-parent, etc). First *must* be set. Codes map[string]struct{} // Set of bytecodes ran or accessed State map[string]struct{} // Set of MPT state trie nodes (account and storage together) - Paths map[string]struct{} // Set of MPT trie paths (i.e. all accessed nodes, not just the ones in state) - chain HeaderReader // Chain reader to convert block hash ops to header proofs - lock sync.Mutex // Lock to allow concurrent state insertions + StateDepthMetrics DepthTracker // Metrics about the state trie paths, used for debugging + AccountDepthMetrics DepthTracker // Metrics about the account trie paths, used for debugging + chain HeaderReader // Chain reader to convert block hash ops to header proofs + lock sync.Mutex // Lock to allow concurrent state insertions } // NewWitness creates an empty witness ready for population. @@ -59,18 +67,35 @@ func NewWitness(context *types.Header, chain HeaderReader) (*Witness, error) { } headers = append(headers, parent) } + + var stateDepthTracker = DepthTracker{Sum: 0, Count: 0, Min: -1, Max: 0} + var accountDepthTracker = DepthTracker{Sum: 0, Count: 0, Min: -1, Max: 0} // Create the witness with a reconstructed gutted out block return &Witness{ - context: context, - Headers: headers, - Codes: make(map[string]struct{}), - State: make(map[string]struct{}), - Paths: make(map[string]struct{}), - - chain: chain, + context: context, + Headers: headers, + Codes: make(map[string]struct{}), + State: make(map[string]struct{}), + StateDepthMetrics: stateDepthTracker, + AccountDepthMetrics: accountDepthTracker, + chain: chain, }, nil } +func (d *DepthTracker) Add(n int) { + if n < 0 { + n = 0 + } + d.Sum += n + d.Count++ + if d.Min == -1 || n < d.Min { + d.Min = n + } + if n > d.Max { + d.Max = n + } +} + // AddBlockHash adds a "blockhash" to the witness with the designated offset from // chain head. Under the hood, this method actually pulls in enough headers from // the chain to cover the block being added. @@ -91,7 +116,7 @@ func (w *Witness) AddCode(code []byte) { } // AddState inserts a batch of MPT trie nodes into the witness. -func (w *Witness) AddState(nodemap map[string][]byte) { +func (w *Witness) AddState(nodemap map[string][]byte, owner common.Hash) { if len(nodemap) == 0 { return } @@ -99,7 +124,25 @@ func (w *Witness) AddState(nodemap map[string][]byte) { defer w.lock.Unlock() for path, value := range nodemap { w.State[string(value)] = struct{}{} - w.Paths[string(path)] = struct{}{} // Also track the path for the node + if owner != (common.Hash{}) { + w.AccountDepthMetrics.Add(len(path)) + } else { + w.StateDepthMetrics.Add(len(path)) + } + } +} + +// AddStateModify tracks the modification of a node in the witness. +func (w *Witness) AddStateModify(path int, owner common.Hash) { + if path < 0 { + return + } + w.lock.Lock() + defer w.lock.Unlock() + if owner != (common.Hash{}) { + w.AccountDepthMetrics.Add(path) + } else { + w.StateDepthMetrics.Add(path) } } @@ -107,11 +150,12 @@ func (w *Witness) AddState(nodemap map[string][]byte) { // is never mutated by Witness func (w *Witness) Copy() *Witness { cpy := &Witness{ - Headers: slices.Clone(w.Headers), - Codes: maps.Clone(w.Codes), - State: maps.Clone(w.State), - Paths: maps.Clone(w.Paths), - chain: w.chain, + Headers: slices.Clone(w.Headers), + Codes: maps.Clone(w.Codes), + State: maps.Clone(w.State), + StateDepthMetrics: w.StateDepthMetrics, + AccountDepthMetrics: w.AccountDepthMetrics, + chain: w.chain, } if w.context != nil { cpy.context = types.CopyHeader(w.context) diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 85e77fe80e..4fb21d20ad 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -96,6 +96,10 @@ func NewStateTrie(id *ID, db database.NodeDatabase) (*StateTrie, error) { return tr, nil } +func (t *StateTrie) Owner() common.Hash { + return t.trie.Owner() +} + // MustGet returns the value for key stored in the trie. // The value bytes must not be modified by the caller. // @@ -277,11 +281,6 @@ func (t *StateTrie) Witness() map[string][]byte { return t.trie.Witness() } -// Witness returns a set containing all trie nodes that have been accessed. -func (t *StateTrie) WitnessPaths() map[string]struct{} { - return t.trie.WitnessPaths() -} - // Commit collects all dirty nodes in the trie and replaces them with the // corresponding node hash. All collected nodes (including dirty leaves if // collectLeaf is true) will be encapsulated into a nodeset for return. diff --git a/trie/verkle.go b/trie/verkle.go index e00ea21602..3ad638f089 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -78,6 +78,10 @@ func (t *VerkleTrie) GetKey(key []byte) []byte { return key } +func (t *VerkleTrie) Owner() common.Hash { + panic("VerkleTrie does not have an owner") +} + // GetAccount implements state.Trie, retrieving the account with the specified // account address. If the specified account is not in the verkle tree, nil will // be returned. If the tree is corrupted, an error will be returned.