diff --git a/core/state/statedb.go b/core/state/statedb.go index cdfd638221..2c41c2081b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -139,6 +139,9 @@ type StateDB struct { witness *stateless.Witness witnessStats *stateless.WitnessStats + // State size tracking + stateSizeGen *stateSizeGenerator + // Measurements gathered during execution for debugging purposes AccountReads time.Duration AccountHashes time.Duration @@ -186,6 +189,15 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro if db.TrieDB().IsVerkle() { sdb.accessEvents = NewAccessEvents(db.PointCache()) } + + // Initialize state size tracking + if cachingDB, ok := db.(*CachingDB); ok { + sdb.stateSizeGen = newStateSizeGenerator(cachingDB.disk, db, root) + if !db.TrieDB().IsVerkle() { + sdb.stateSizeGen.run() + } + } + return sdb, nil } @@ -1350,6 +1362,11 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } s.TrieDBCommits += time.Since(start) + + // Update state size metrics + if s.stateSizeGen != nil { + s.stateSizeGen.updateMetrics(ret) + } } } s.reader, _ = s.db.Reader(s.originalRoot) diff --git a/core/state/statesize.go b/core/state/statesize.go new file mode 100644 index 0000000000..d3424d0c9f --- /dev/null +++ b/core/state/statesize.go @@ -0,0 +1,305 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" +) + +// State size metrics +var ( + // Baseline state size metrics + stateSizeAccountsCountMeter = metrics.NewRegisteredMeter("statedb/statesize/accounts/count", nil) + stateSizeAccountsBytesMeter = metrics.NewRegisteredMeter("statedb/statesize/accounts/bytes", nil) + stateSizeStorageCountMeter = metrics.NewRegisteredMeter("statedb/statesize/storage/count", nil) + stateSizeStorageBytesMeter = metrics.NewRegisteredMeter("statedb/statesize/storage/bytes", nil) + stateSizeTrieNodesCountMeter = metrics.NewRegisteredMeter("statedb/statesize/trienodes/count", nil) + stateSizeTrieNodesBytesMeter = metrics.NewRegisteredMeter("statedb/statesize/trienodes/bytes", nil) + stateSizeContractsCountMeter = metrics.NewRegisteredMeter("statedb/statesize/contracts/count", nil) + stateSizeContractsBytesMeter = metrics.NewRegisteredMeter("statedb/statesize/contracts/bytes", nil) +) + +// StateSizeMetrics represents the current state size statistics +type StateSizeMetrics struct { + Root common.Hash // Root hash of the state trie + AccountCount uint64 + AccountBytes uint64 + StorageCount uint64 + StorageBytes uint64 + TrieNodeCount uint64 + TrieNodeBytes uint64 + ContractCount uint64 + ContractBytes uint64 +} + +// stateSizeGenerator handles the initialization and tracking of state size metrics +type stateSizeGenerator struct { + db ethdb.KeyValueStore + sdb Database + + // Generator state + running bool + abort chan chan struct{} + done chan struct{} + + // Metrics state + metrics *StateSizeMetrics + metricsLock sync.RWMutex +} + +// newStateSizeGenerator creates a new state size generator +func newStateSizeGenerator(db ethdb.KeyValueStore, sdb Database, root common.Hash) *stateSizeGenerator { + return &stateSizeGenerator{ + db: db, + sdb: sdb, + abort: make(chan chan struct{}), + done: make(chan struct{}), + metrics: &StateSizeMetrics{Root: root}, + } +} + +// run starts the state size initialization in the background +func (g *stateSizeGenerator) run() { + if g.running { + g.stop() + log.Warn("Paused the leftover state size generation cycle") + } + g.running = true + go g.generate() +} + +// stop terminates the background generation +func (g *stateSizeGenerator) stop() { + if !g.running { + return + } + ch := make(chan struct{}) + g.abort <- ch + <-ch + g.running = false +} + +// generate performs the state size initialization +func (g *stateSizeGenerator) generate() { + defer close(g.done) + start := time.Now() + + if g.hasExistingMetrics() { + log.Info("State size metrics already initialized") + return + } + + // Wait for snapshot generator to complete + if db := g.sdb.TrieDB(); db != nil { + for !db.SnapshotCompleted() { + time.Sleep(5 * time.Second) + } + } + + log.Info("Starting state size initialization") + g.initializeMetrics() + log.Info("Completed state size initialization", "elapsed", time.Since(start)) +} + +// hasExistingMetrics checks if state size metrics already exist in the database +func (g *stateSizeGenerator) hasExistingMetrics() bool { + // Check for existing metrics by looking for a marker key + marker := rawdb.ReadStateSizeMetrics(g.db) + // TODO: check if the marker's root is the same as the current root + return marker != nil +} + +// initializeMetrics performs the actual metrics initialization +func (g *stateSizeGenerator) initializeMetrics() { + var ( + accountCount, accountBytes uint64 + storageCount, storageBytes uint64 + trieNodeCount, trieNodeBytes uint64 + contractCount, contractBytes uint64 + ) + + // Process accounts + log.Info("Initializing account metrics") + accountIter := g.db.NewIterator(rawdb.SnapshotAccountPrefix, nil) + defer accountIter.Release() + + for accountIter.Next() { + key := accountIter.Key() + value := accountIter.Value() + + // Count account + accountCount++ + accountBytes += uint64(len(key) + len(value)) + + // Check if account has code (contract) + var account types.StateAccount + if err := rlp.DecodeBytes(value, &account); err == nil { + if !bytes.Equal(account.CodeHash, types.EmptyCodeHash[:]) { + contractCount++ + // Code size will be counted separately + } + } + + // Process storage for this account + storageIter := g.db.NewIterator(append(rawdb.SnapshotStoragePrefix, key[1:]...), nil) + for storageIter.Next() { + storageKey := storageIter.Key() + storageValue := storageIter.Value() + storageCount++ + storageBytes += uint64(len(storageKey) + len(storageValue)) + } + storageIter.Release() + + // Check for abort + select { + case abort := <-g.abort: + close(abort) + return + default: + } + } + + // Process trie nodes + log.Info("Initializing trie node metrics") + trieNodeIter := g.db.NewIterator(rawdb.TrieNodeAccountPrefix, nil) + defer trieNodeIter.Release() + + for trieNodeIter.Next() { + key := trieNodeIter.Key() + value := trieNodeIter.Value() + trieNodeCount++ + trieNodeBytes += uint64(len(key) + len(value)) + } + + // Process storage trie nodes + storageTrieIter := g.db.NewIterator(rawdb.TrieNodeStoragePrefix, nil) + defer storageTrieIter.Release() + + for storageTrieIter.Next() { + key := storageTrieIter.Key() + value := storageTrieIter.Value() + trieNodeCount++ + trieNodeBytes += uint64(len(key) + len(value)) + } + + // Process contract code + log.Info("Initializing contract code metrics") + codeIter := g.db.NewIterator(rawdb.CodePrefix, nil) + defer codeIter.Release() + + for codeIter.Next() { + key := codeIter.Key() + value := codeIter.Value() + contractBytes += uint64(len(key) + len(value)) + } + + // Update metrics + g.metricsLock.Lock() + g.metrics.AccountCount = accountCount + g.metrics.AccountBytes = accountBytes + g.metrics.StorageCount = storageCount + g.metrics.StorageBytes = storageBytes + g.metrics.TrieNodeCount = trieNodeCount + g.metrics.TrieNodeBytes = trieNodeBytes + g.metrics.ContractCount = contractCount + g.metrics.ContractBytes = contractBytes + g.metricsLock.Unlock() + + // Update metrics in database + g.persistMetrics() + + // Update global metrics + stateSizeAccountsCountMeter.Mark(int64(accountCount)) + stateSizeAccountsBytesMeter.Mark(int64(accountBytes)) + stateSizeStorageCountMeter.Mark(int64(storageCount)) + stateSizeStorageBytesMeter.Mark(int64(storageBytes)) + stateSizeTrieNodesCountMeter.Mark(int64(trieNodeCount)) + stateSizeTrieNodesBytesMeter.Mark(int64(trieNodeBytes)) + stateSizeContractsCountMeter.Mark(int64(contractCount)) + stateSizeContractsBytesMeter.Mark(int64(contractBytes)) +} + +// persistMetrics saves the current metrics to the database +func (g *stateSizeGenerator) persistMetrics() { + g.metricsLock.RLock() + metrics := *g.metrics + g.metricsLock.RUnlock() + + data, err := rlp.EncodeToBytes(metrics) + if err != nil { + log.Error("Failed to encode state size metrics", "err", err) + return + } + + batch := g.db.NewBatch() + rawdb.WriteStateSizeMetrics(batch, data) + if err := batch.Write(); err != nil { + log.Error("Failed to persist state size metrics", "err", err) + } +} + +// updateMetrics updates metrics based on state changes +func (g *stateSizeGenerator) updateMetrics(update *stateUpdate) { + var diff StateSizeMetrics + + // Calculate account changes + for _, data := range update.accounts { + if len(data) > 0 { + diff.AccountCount++ + diff.AccountBytes += uint64(common.HashLength + len(data)) + } + } + + // Calculate storage changes + for _, slots := range update.storages { + for _, data := range slots { + if len(data) > 0 { + diff.StorageCount++ + diff.StorageBytes += uint64(2*common.HashLength + len(data)) + } + } + } + + // Calculate trie node changes + for _, nodeSet := range update.nodes.Sets { + for _, node := range nodeSet.Nodes { + diff.TrieNodeCount++ + diff.TrieNodeBytes += uint64(len(node.Blob)) + } + } + + // Update local metrics + g.metricsLock.Lock() + g.metrics.Root = update.root + g.metrics.AccountCount += diff.AccountCount + g.metrics.AccountBytes += diff.AccountBytes + g.metrics.StorageCount += diff.StorageCount + g.metrics.StorageBytes += diff.StorageBytes + g.metrics.TrieNodeCount += diff.TrieNodeCount + g.metrics.TrieNodeBytes += diff.TrieNodeBytes + g.metricsLock.Unlock() +}