forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-22 20:56:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core/state: persist state size metrics

Browse source
This commit is contained in:
weiihann 2025-12-22 11:58:02 +08:00
parent 2e5cd21edf
commit 288588b852
5 changed files with 245 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
core/rawdb/accessors_state.go
View file

@ -360,3 +360,22 @@ func WriteTrienodeHistory(db ethdb.AncientWriter, id uint64, header []byte, keyS
})
return err
}
// HasStateSizeStats checks if the state size stats for the specified state root is present in the database.
func HasStateSizeStats(db ethdb.KeyValueReader, root common.Hash) bool {
ok, _ := db.Has(stateSizeStatsKey(root))
return ok
}
// ReadStateSizeStats retrieves the state size stats for the specified state root.
func ReadStateSizeStats(db ethdb.KeyValueReader, root common.Hash) []byte {
data, _ := db.Get(stateSizeStatsKey(root))
return data
}
// WriteStateSizeStats stores the state size stats for the specified state root.
func WriteStateSizeStats(db ethdb.KeyValueWriter, root common.Hash, stats []byte) {
if err := db.Put(stateSizeStatsKey(root), stats); err != nil {
log.Crit("Failed to store state size stats", "err", err)
}
}

6
core/rawdb/database.go
View file

@ -427,6 +427,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
filterMapRows stat
filterMapLastBlock stat
filterMapBlockLV stat
stateSizeStats stat
// Path-mode archive data
stateIndex stat
@ -527,6 +528,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength:
stateIndex.add(size)
// State size stats
case bytes.HasPrefix(key, stateSizeStatsPrefix):
stateSizeStats.add(size)
// Verkle trie data is detected, determine the sub-category
case bytes.HasPrefix(key, VerklePrefix):
remain := key[len(VerklePrefix):]
@ -631,6 +636,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()},
{"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()},
{"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()},
{"Key-Value store", "State size stats", stateSizeStats.sizeString(), stateSizeStats.countString()},
}
// Inspect all registered append-only file store then.

10
core/rawdb/schema.go
View file

@ -167,6 +167,9 @@ var (
// Verkle transition information
VerkleTransitionStatePrefix = []byte("verkle-transition-state-")
// State size metrics
stateSizeStatsPrefix = []byte("st") // stateSizeStatsPrefix + root -> stat (rlp)
)
// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
@ -459,3 +462,10 @@ func trienodeHistoryIndexBlockKey(addressHash common.Hash, path []byte, blockID
func transitionStateKey(hash common.Hash) []byte {
return append(VerkleTransitionStatePrefix, hash.Bytes()...)
}
func stateSizeStatsKey(root common.Hash) []byte {
result := make([]byte, len(stateSizeStatsPrefix)+common.HashLength)
copy(result, stateSizeStatsPrefix)
copy(result[len(stateSizeStatsPrefix):], root.Bytes())
return result
}

165
core/state/state_sizer.go
View file

@ -32,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/triedb"
"golang.org/x/sync/errgroup"
)
@ -263,7 +264,7 @@ type stateSizeQuery struct {
// SizeTracker handles the state size initialization and tracks of state size metrics.
type SizeTracker struct {
db ethdb.KeyValueStore
db ethdb.Database
triedb *triedb.Database
abort chan struct{}
aborted chan struct{}
@ -272,7 +273,7 @@ type SizeTracker struct {
}
// NewSizeTracker creates a new state size tracker and starts it automatically
func NewSizeTracker(db ethdb.KeyValueStore, triedb *triedb.Database) (*SizeTracker, error) {
func NewSizeTracker(db ethdb.Database, triedb *triedb.Database) (*SizeTracker, error) {
if triedb.Scheme() != rawdb.PathScheme {
return nil, errors.New("state size tracker is not compatible with hash mode")
}
@ -284,7 +285,26 @@ func NewSizeTracker(db ethdb.KeyValueStore, triedb *triedb.Database) (*SizeTrack
updateCh: make(chan *stateUpdate),
queryCh: make(chan *stateSizeQuery),
}
go t.run()
// Load the existing stats if they exist
stats := make(map[common.Hash]SizeStats, statEvictThreshold)
header := rawdb.ReadHeadHeader(t.db)
headNumber := header.Number.Uint64()
for i := 0; header != nil && i < statEvictThreshold; i++ {
if enc := rawdb.ReadStateSizeStats(t.db, header.Root); len(enc) > 0 {
stats[header.Root] = decodeSizeStats(enc)
}
header = rawdb.ReadHeader(t.db, header.ParentHash, header.Number.Uint64()-1)
}
var h sizeStatsHeap
if len(stats) > 0 {
h = sizeStatsHeap(slices.Collect(maps.Values(stats)))
heap.Init(&h)
log.Info("Loaded state size stats from db", "headNumber", headNumber, "count", len(stats))
}
go t.run(stats, h)
return t, nil
}
@ -314,17 +334,22 @@ func (h *sizeStatsHeap) Pop() any {
}
// run performs the state size initialization and handles updates
func (t *SizeTracker) run() {
func (t *SizeTracker) run(stats map[common.Hash]SizeStats, h sizeStatsHeap) {
defer close(t.aborted)
var last common.Hash
stats, err := t.init() // launch background thread for state size init
if err != nil {
return
}
h := sizeStatsHeap(slices.Collect(maps.Values(stats)))
heap.Init(&h)
var err error
if len(stats) == 0 { // if no stats from db, launch background thread for state size init
stats, err = t.init()
if err != nil {
log.Error("Failed to init state size tracker", "err", err)
return
}
h = sizeStatsHeap(slices.Collect(maps.Values(stats)))
heap.Init(&h)
}
var last common.Hash
for {
select {
case u := <-t.updateCh:
@ -341,6 +366,8 @@ func (t *SizeTracker) run() {
stats[u.root] = stat
last = u.root
rawdb.WriteStateSizeStats(t.db, u.root, encodeSizeStats(stat))
// Publish statistics to metric system
stat.publish()
@ -359,12 +386,23 @@ func (t *SizeTracker) run() {
} else {
root = last
}
if s, ok := stats[root]; ok {
// Check in-memory stats first
s, ok := stats[root]
if ok {
r.result <- &s
} else {
r.result <- nil
continue
}
// Load from db if exist
if enc := rawdb.ReadStateSizeStats(t.db, root); len(enc) > 0 {
s := decodeSizeStats(enc)
r.result <- &s
continue
}
// Does not exist, return nil result
r.result <- nil
case <-t.abort:
return
}
@ -381,23 +419,25 @@ type buildResult struct {
func (t *SizeTracker) init() (map[common.Hash]SizeStats, error) {
// Wait for snapshot completion and then init
ticker := time.NewTicker(10 * time.Second)
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
wait:
for {
select {
case <-ticker.C:
if t.triedb.SnapshotCompleted() {
break wait
if !t.triedb.SnapshotCompleted() {
wait:
for {
select {
case <-ticker.C:
if t.triedb.SnapshotCompleted() {
break wait
}
case <-t.updateCh:
continue
case r := <-t.queryCh:
r.err = errors.New("state size is not initialized yet")
r.result <- nil
case <-t.abort:
return nil, errors.New("size tracker closed")
}
case <-t.updateCh:
continue
case r := <-t.queryCh:
r.err = errors.New("state size is not initialized yet")
r.result <- nil
case <-t.abort:
return nil, errors.New("size tracker closed")
}
}
@ -413,7 +453,6 @@ wait:
updates[u.root] = u
children[u.originRoot] = append(children[u.originRoot], u.root)
log.Debug("Received state update", "root", u.root, "blockNumber", u.blockNumber)
case r := <-t.queryCh:
r.err = errors.New("state size is not initialized yet")
r.result <- nil
@ -453,7 +492,11 @@ wait:
if err != nil {
return err
}
stats[child] = base.add(diff)
stat := base.add(diff)
stats[child] = stat
rawdb.WriteStateSizeStats(t.db, child, encodeSizeStats(stat))
if err := apply(child, stats[child]); err != nil {
return err
}
@ -466,6 +509,7 @@ wait:
// Set initial latest stats
stats[result.root] = result.stat
rawdb.WriteStateSizeStats(t.db, result.root, encodeSizeStats(result.stat))
log.Info("Measured persistent state size", "root", result.root, "number", result.blockNumber, "stat", result.stat, "elapsed", common.PrettyDuration(result.elapsed))
return stats, nil
@ -672,3 +716,64 @@ func (t *SizeTracker) Query(root *common.Hash) (*SizeStats, error) {
return <-r.result, r.err
}
}
// sizeStatsRLP is the RLP-encodable version of SizeStats.
// RLP only supports unsigned integers, so we use uint64 for persistence.
// Cumulative stats are always non-negative, so this is safe for storage.
type sizeStatsRLP struct {
StateRoot common.Hash
BlockNumber uint64
Accounts uint64
AccountBytes uint64
Storages uint64
StorageBytes uint64
AccountTrienodes uint64
AccountTrienodeBytes uint64
StorageTrienodes uint64
StorageTrienodeBytes uint64
ContractCodes uint64
ContractCodeBytes uint64
}
func encodeSizeStats(stat SizeStats) []byte {
rlpStat := sizeStatsRLP{
StateRoot: stat.StateRoot,
BlockNumber: stat.BlockNumber,
Accounts: uint64(stat.Accounts),
AccountBytes: uint64(stat.AccountBytes),
Storages: uint64(stat.Storages),
StorageBytes: uint64(stat.StorageBytes),
AccountTrienodes: uint64(stat.AccountTrienodes),
AccountTrienodeBytes: uint64(stat.AccountTrienodeBytes),
StorageTrienodes: uint64(stat.StorageTrienodes),
StorageTrienodeBytes: uint64(stat.StorageTrienodeBytes),
ContractCodes: uint64(stat.ContractCodes),
ContractCodeBytes: uint64(stat.ContractCodeBytes),
}
data, err := rlp.EncodeToBytes(&rlpStat)
if err != nil {
log.Crit("Failed to encode state size stats", "err", err)
}
return data
}
func decodeSizeStats(data []byte) SizeStats {
var rlpStat sizeStatsRLP
if err := rlp.DecodeBytes(data, &rlpStat); err != nil {
log.Crit("Failed to decode state size stats", "err", err)
}
return SizeStats{
StateRoot: rlpStat.StateRoot,
BlockNumber: rlpStat.BlockNumber,
Accounts: int64(rlpStat.Accounts),
AccountBytes: int64(rlpStat.AccountBytes),
Storages: int64(rlpStat.Storages),
StorageBytes: int64(rlpStat.StorageBytes),
AccountTrienodes: int64(rlpStat.AccountTrienodes),
AccountTrienodeBytes: int64(rlpStat.AccountTrienodeBytes),
StorageTrienodes: int64(rlpStat.StorageTrienodes),
StorageTrienodeBytes: int64(rlpStat.StorageTrienodeBytes),
ContractCodes: int64(rlpStat.ContractCodes),
ContractCodeBytes: int64(rlpStat.ContractCodeBytes),
}
}

75
core/state/state_sizer_test.go
View file

@ -237,3 +237,78 @@ func TestSizeTracker(t *testing.T) {
t.Errorf("Storage trie node bytes mismatch: expected %d, got %d", expectedStats.StorageTrienodeBytes, actualStats.StorageTrienodeBytes)
}
}
func TestSizeStatsPersistence(t *testing.T) {
db := rawdb.NewMemoryDatabase()
defer db.Close()
root := common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")
originalStats := SizeStats{
StateRoot: root,
BlockNumber: 12345,
Accounts: 1000,
AccountBytes: 50000,
Storages: 5000,
StorageBytes: 200000,
AccountTrienodes: 300,
AccountTrienodeBytes: 15000,
StorageTrienodes: 1500,
StorageTrienodeBytes: 75000,
ContractCodes: 100,
ContractCodeBytes: 500000,
}
// Verify no stats exist initially
if rawdb.HasStateSizeStats(db, root) {
t.Fatal("Stats should not exist initially")
}
rawdb.WriteStateSizeStats(db, root, encodeSizeStats(originalStats))
// Verify stats exist
if !rawdb.HasStateSizeStats(db, root) {
t.Fatal("Stats should exist after write")
}
data := rawdb.ReadStateSizeStats(db, root)
loadedStats := decodeSizeStats(data)
// Verify all fields match
if loadedStats.StateRoot != originalStats.StateRoot {
t.Errorf("StateRoot mismatch: have %x, want %x", loadedStats.StateRoot, originalStats.StateRoot)
}
if loadedStats.BlockNumber != originalStats.BlockNumber {
t.Errorf("BlockNumber mismatch: have %d, want %d", loadedStats.BlockNumber, originalStats.BlockNumber)
}
if loadedStats.Accounts != originalStats.Accounts {
t.Errorf("Accounts mismatch: have %d, want %d", loadedStats.Accounts, originalStats.Accounts)
}
if loadedStats.AccountBytes != originalStats.AccountBytes {
t.Errorf("AccountBytes mismatch: have %d, want %d", loadedStats.AccountBytes, originalStats.AccountBytes)
}
if loadedStats.Storages != originalStats.Storages {
t.Errorf("Storages mismatch: have %d, want %d", loadedStats.Storages, originalStats.Storages)
}
if loadedStats.StorageBytes != originalStats.StorageBytes {
t.Errorf("StorageBytes mismatch: have %d, want %d", loadedStats.StorageBytes, originalStats.StorageBytes)
}
if loadedStats.AccountTrienodes != originalStats.AccountTrienodes {
t.Errorf("AccountTrienodes mismatch: have %d, want %d", loadedStats.AccountTrienodes, originalStats.AccountTrienodes)
}
if loadedStats.AccountTrienodeBytes != originalStats.AccountTrienodeBytes {
t.Errorf("AccountTrienodeBytes mismatch: have %d, want %d", loadedStats.AccountTrienodeBytes, originalStats.AccountTrienodeBytes)
}
if loadedStats.StorageTrienodes != originalStats.StorageTrienodes {
t.Errorf("StorageTrienodes mismatch: have %d, want %d", loadedStats.StorageTrienodes, originalStats.StorageTrienodes)
}
if loadedStats.StorageTrienodeBytes != originalStats.StorageTrienodeBytes {
t.Errorf("StorageTrienodeBytes mismatch: have %d, want %d", loadedStats.StorageTrienodeBytes, originalStats.StorageTrienodeBytes)
}
if loadedStats.ContractCodes != originalStats.ContractCodes {
t.Errorf("ContractCodes mismatch: have %d, want %d", loadedStats.ContractCodes, originalStats.ContractCodes)
}
if loadedStats.ContractCodeBytes != originalStats.ContractCodeBytes {
t.Errorf("ContractCodeBytes mismatch: have %d, want %d", loadedStats.ContractCodeBytes, originalStats.ContractCodeBytes)
}
}