forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-18 19:00:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

trie,core: modify triestat to be used by stateless WitnessStats

Browse source
This commit is contained in:
lightclient 2026-02-17 05:08:42 +00:00
parent 889feb2e08
commit 97cdc16c3b
No known key found for this signature in database
GPG key ID: 91289C7FE4ADADBD
5 changed files with 274 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
core/stateless/stats.go
View file

@ -27,6 +27,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/trie"
)
var accountTrieLeavesAtDepth [16]*metrics.Counter
@ -41,59 +42,68 @@ func init() {
// WitnessStats aggregates statistics for account and storage trie accesses.
type WitnessStats struct {
accountTrieLeaves [16]int64
storageTrieLeaves [16]int64
accountTrie *trie.LevelStats
storageTrie *trie.LevelStats
}
// NewWitnessStats creates a new WitnessStats collector.
func NewWitnessStats() *WitnessStats {
return &WitnessStats{}
return &WitnessStats{
accountTrie: trie.NewLevelStats(),
storageTrie: trie.NewLevelStats(),
}
}
func (s *WitnessStats) init() {
if s.accountTrie == nil {
s.accountTrie = trie.NewLevelStats()
}
if s.storageTrie == nil {
s.storageTrie = trie.NewLevelStats()
}
}
// Add records trie access depths from the given node paths.
// If `owner` is the zero hash, accesses are attributed to the account trie;
// otherwise, they are attributed to the storage trie of that account.
func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) {
// Extract paths from the nodes map
s.init()
// Extract paths from the nodes map.
paths := slices.Collect(maps.Keys(nodes))
sort.Strings(paths)
ownerStat := s.accountTrie
if owner != (common.Hash{}) {
ownerStat = s.storageTrie
}
for i, path := range paths {
// If current path is a prefix of the next path, it's not a leaf.
// The last path is always a leaf.
if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) {
depth := len(path)
if owner == (common.Hash{}) {
if depth >= len(s.accountTrieLeaves) {
depth = len(s.accountTrieLeaves) - 1
}
s.accountTrieLeaves[depth] += 1
} else {
if depth >= len(s.storageTrieLeaves) {
depth = len(s.storageTrieLeaves) - 1
}
s.storageTrieLeaves[depth] += 1
}
ownerStat.AddLeaf(len(path))
}
}
}
// ReportMetrics reports the collected statistics to the global metrics registry.
func (s *WitnessStats) ReportMetrics(blockNumber uint64) {
// Encode the metrics as JSON for easier consumption
accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves)
storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves)
s.init()
// Log account trie depth statistics
log.Info("Account trie depth stats",
"block", blockNumber,
"leavesAtDepth", string(accountLeavesJson))
log.Info("Storage trie depth stats",
"block", blockNumber,
"leavesAtDepth", string(storageLeavesJson))
accountTrieLeaves := s.accountTrie.LeafDepths()
storageTrieLeaves := s.storageTrie.LeafDepths()
for i := 0; i < 16; i++ {
accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i])
storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i])
// Encode the metrics as JSON for easier consumption.
accountLeavesJSON, _ := json.Marshal(accountTrieLeaves)
storageLeavesJSON, _ := json.Marshal(storageTrieLeaves)
// Log account trie depth statistics.
log.Info("Account trie depth stats", "block", blockNumber, "leavesAtDepth", string(accountLeavesJSON))
log.Info("Storage trie depth stats", "block", blockNumber, "leavesAtDepth", string(storageLeavesJSON))
for i := 0; i < len(accountTrieLeavesAtDepth); i++ {
accountTrieLeavesAtDepth[i].Inc(accountTrieLeaves[i])
storageTrieLeavesAtDepth[i].Inc(storageTrieLeaves[i])
}
}

108
core/stateless/stats_test.go
View file

@ -17,18 +17,27 @@
package stateless
import (
"strings"
"testing"
"github.com/ethereum/go-ethereum/common"
)
func expectedLeaves(counts map[int]int64) [16]int64 {
var leaves [16]int64
for depth, count := range counts {
leaves[depth] = count
}
return leaves
}
func TestWitnessStatsAdd(t *testing.T) {
tests := []struct {
name string
nodes map[string][]byte
owner common.Hash
expectedAccountLeaves map[int64]int64
expectedStorageLeaves map[int64]int64
expectedAccountLeaves map[int]int64
expectedStorageLeaves map[int]int64
}{
{
name: "empty nodes",
@ -41,7 +50,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"": []byte("data"),
},
owner: common.Hash{},
expectedAccountLeaves: map[int64]int64{0: 1},
expectedAccountLeaves: map[int]int64{0: 1},
},
{
name: "single account trie leaf",
@ -49,7 +58,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"abc": []byte("data"),
},
owner: common.Hash{},
expectedAccountLeaves: map[int64]int64{3: 1},
expectedAccountLeaves: map[int]int64{3: 1},
},
{
name: "account trie with internal nodes",
@ -59,7 +68,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"abc": []byte("data3"),
},
owner: common.Hash{},
expectedAccountLeaves: map[int64]int64{3: 1}, // Only "abc" is a leaf
expectedAccountLeaves: map[int]int64{3: 1}, // Only "abc" is a leaf
},
{
name: "multiple account trie branches",
@ -72,7 +81,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"bcd": []byte("data6"),
},
owner: common.Hash{},
expectedAccountLeaves: map[int64]int64{3: 2}, // "abc" (3) + "bcd" (3)
expectedAccountLeaves: map[int]int64{3: 2}, // "abc" (3) + "bcd" (3)
},
{
name: "siblings are all leaves",
@ -82,7 +91,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"ac": []byte("data3"),
},
owner: common.Hash{},
expectedAccountLeaves: map[int64]int64{2: 3},
expectedAccountLeaves: map[int]int64{2: 3},
},
{
name: "storage trie leaves",
@ -93,7 +102,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"124": []byte("data4"),
},
owner: common.HexToHash("0x1234"),
expectedStorageLeaves: map[int64]int64{3: 2}, // "123" (3) + "124" (3)
expectedStorageLeaves: map[int]int64{3: 2}, // "123" (3) + "124" (3)
},
{
name: "complex trie structure",
@ -109,7 +118,7 @@ func TestWitnessStatsAdd(t *testing.T) {
"3": []byte("data9"),
},
owner: common.Hash{},
expectedAccountLeaves: map[int64]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1)
expectedAccountLeaves: map[int]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1)
},
}
@ -118,32 +127,59 @@ func TestWitnessStatsAdd(t *testing.T) {
stats := NewWitnessStats()
stats.Add(tt.nodes, tt.owner)
var expectedAccountTrieLeaves [16]int64
for depth, count := range tt.expectedAccountLeaves {
expectedAccountTrieLeaves[depth] = count
if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(tt.expectedAccountLeaves); got != want {
t.Errorf("account trie leaves = %v, want %v", got, want)
}
var expectedStorageTrieLeaves [16]int64
for depth, count := range tt.expectedStorageLeaves {
expectedStorageTrieLeaves[depth] = count
}
// Check account trie depth
if stats.accountTrieLeaves != expectedAccountTrieLeaves {
t.Errorf("Account trie total depth = %v, want %v", stats.accountTrieLeaves, expectedAccountTrieLeaves)
}
// Check storage trie depth
if stats.storageTrieLeaves != expectedStorageTrieLeaves {
t.Errorf("Storage trie total depth = %v, want %v", stats.storageTrieLeaves, expectedStorageTrieLeaves)
if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(tt.expectedStorageLeaves); got != want {
t.Errorf("storage trie leaves = %v, want %v", got, want)
}
})
}
}
func TestWitnessStatsStorageTrieAggregation(t *testing.T) {
stats := NewWitnessStats()
ownerA := common.HexToHash("0xa")
ownerB := common.HexToHash("0xb")
stats.Add(map[string][]byte{
"a": []byte("data1"),
"ab": []byte("data2"),
"abc": []byte("data3"),
}, ownerA)
stats.Add(map[string][]byte{
"xy": []byte("data4"),
}, ownerA)
stats.Add(map[string][]byte{
"1": []byte("data5"),
"12": []byte("data6"),
"123": []byte("data7"),
"124": []byte("data8"),
}, ownerB)
if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(map[int]int64{2: 1, 3: 3}); got != want {
t.Errorf("storage leaves = %v, want %v", got, want)
}
if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(nil); got != want {
t.Errorf("account leaves = %v, want %v", got, want)
}
}
func TestWitnessStatsPanicsOnDeepLeaf(t *testing.T) {
stats := NewWitnessStats()
defer func() {
if r := recover(); r == nil {
t.Fatal("expected panic for depth >= 16")
}
}()
stats.Add(map[string][]byte{strings.Repeat("a", 16): []byte("data")}, common.Hash{})
}
func TestWitnessStatsMinMax(t *testing.T) {
stats := NewWitnessStats()
// Add some account trie nodes with varying depths
// Add some account trie nodes with varying depths.
stats.Add(map[string][]byte{
"a": []byte("data1"),
"ab": []byte("data2"),
@ -152,21 +188,21 @@ func TestWitnessStatsMinMax(t *testing.T) {
"abcde": []byte("data5"),
}, common.Hash{})
// Only "abcde" is a leaf (depth 5)
for i, v := range stats.accountTrieLeaves {
// Only "abcde" is a leaf (depth 5).
for i, v := range stats.accountTrie.LeafDepths() {
if v != 0 && i != 5 {
t.Errorf("leaf found at invalid depth %d", i)
}
}
// Add more leaves with different depths
// Add more leaves with different depths.
stats.Add(map[string][]byte{
"x": []byte("data6"),
"yz": []byte("data7"),
}, common.Hash{})
// Now we have leaves at depths 1, 2, and 5
for i, v := range stats.accountTrieLeaves {
// Now we have leaves at depths 1, 2, and 5.
for i, v := range stats.accountTrie.LeafDepths() {
if v != 0 && (i != 5 && i != 2 && i != 1) {
t.Errorf("leaf found at invalid depth %d", i)
}
@ -176,7 +212,7 @@ func TestWitnessStatsMinMax(t *testing.T) {
func TestWitnessStatsAverage(t *testing.T) {
stats := NewWitnessStats()
// Add nodes that will create leaves at depths 2, 3, and 4
// Add nodes that will create leaves at depths 2, 3, and 4.
stats.Add(map[string][]byte{
"aa": []byte("data1"),
"bb": []byte("data2"),
@ -184,22 +220,22 @@ func TestWitnessStatsAverage(t *testing.T) {
"dddd": []byte("data4"),
}, common.Hash{})
// All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples
// All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples.
expectedAvg := int64(11) / int64(4)
var actualAvg, totalSamples int64
for i, c := range stats.accountTrieLeaves {
for i, c := range stats.accountTrie.LeafDepths() {
actualAvg += c * int64(i)
totalSamples += c
}
actualAvg = actualAvg / totalSamples
if actualAvg != expectedAvg {
t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg)
t.Errorf("account trie average depth = %d, want %d", actualAvg, expectedAvg)
}
}
func BenchmarkWitnessStatsAdd(b *testing.B) {
// Create a realistic trie node structure
// Create a realistic trie node structure.
nodes := make(map[string][]byte)
for i := 0; i < 100; i++ {
base := string(rune('a' + i%26))

93
trie/inspect.go
View file

@ -24,7 +24,6 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@ -41,7 +40,7 @@ type inspector struct {
root common.Hash
config *InspectConfig
stats map[common.Hash]*triestat
stats map[common.Hash]*LevelStats
m sync.Mutex // protects stats
sem *semaphore.Weighted
@ -72,10 +71,10 @@ func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConf
triedb: triedb,
root: root,
config: config,
stats: make(map[common.Hash]*triestat),
stats: make(map[common.Hash]*LevelStats),
sem: semaphore.NewWeighted(int64(128)),
}
in.stats[root] = &triestat{}
in.stats[root] = NewLevelStats()
in.inspect(trie, trie.root, 0, []byte{}, in.stats[root])
in.wg.Wait()
@ -91,7 +90,7 @@ func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConf
// inspect is called recursively down the trie. At each level it records the
// node type encountered.
func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *triestat) {
func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *LevelStats) {
if n == nil {
return
}
@ -152,7 +151,7 @@ func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, sta
log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path))
break
}
stat := &triestat{}
stat := NewLevelStats()
in.m.Lock()
in.stats[owner] = stat
@ -181,7 +180,7 @@ func (in *inspector) displayResult() {
if !in.config.NoStorage {
// Sort stats by max node depth.
keys, stats := sortedTriestat(in.stats).sort()
keys, stats := sortedLevelStats(in.stats).sort()
fmt.Println("Results for top storage tries")
for i := range keys[0:min(in.config.TopN, len(keys))] {
@ -204,7 +203,7 @@ func (in *inspector) writeJSON() error {
}
if !in.config.NoStorage {
// Sort stats by max node depth.
keys, stats := sortedTriestat(in.stats).sort()
keys, stats := sortedLevelStats(in.stats).sort()
for i := range keys[0:min(in.config.TopN, len(keys))] {
storageTrie := newJsonStat(stats[i], fmt.Sprintf("%x", keys[i]))
if err := enc.Encode(storageTrie); err != nil {
@ -215,36 +214,14 @@ func (in *inspector) writeJSON() error {
return nil
}
// triestat tracks the type and count of trie nodes at each level in the trie.
//
// Note: theoretically it is possible to have up to 64 trie level. Since it is
// unlikely to encounter such a large trie, the stats are capped at 16 levels to
// avoid substantial unneeded allocation.
type triestat struct {
level [16]stat
}
// sortedLevelStats implements sort().
type sortedLevelStats map[common.Hash]*LevelStats
// maxDepth iterates each level and finds the deepest level with at least one
// trie node.
func (s *triestat) maxDepth() int {
depth := 0
for i := range s.level {
if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 {
depth = i
}
}
return depth
}
// sortedTriestat implements sort().
type sortedTriestat map[common.Hash]*triestat
// sort returns the keys and triestats in decending order of the maximum trie
// node depth.
func (s sortedTriestat) sort() ([]common.Hash, []*triestat) {
// sort returns the keys and stats in descending order of maximum trie node depth.
func (s sortedLevelStats) sort() ([]common.Hash, []*LevelStats) {
var (
keys = make([]common.Hash, 0, len(s))
stats = make([]*triestat, 0, len(s))
stats = make([]*LevelStats, 0, len(s))
)
for k := range s {
keys = append(keys, k)
@ -256,50 +233,8 @@ func (s sortedTriestat) sort() ([]common.Hash, []*triestat) {
return keys, stats
}
// add increases the node count by one for the specified node type and depth.
func (s *triestat) add(n node, d uint32) {
switch (n).(type) {
case *shortNode:
s.level[d].short.Add(1)
case *fullNode:
s.level[d].full.Add(1)
case valueNode:
s.level[d].value.Add(1)
default:
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
}
}
// stat is a specific level's count of each node type.
type stat struct {
short atomic.Uint64
full atomic.Uint64
value atomic.Uint64
}
// empty is a helper that returns whether there are any trie nodes at the level.
func (s *stat) empty() bool {
if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 {
return true
}
return false
}
// load is a helper that loads each node type's value.
func (s *stat) load() (uint64, uint64, uint64) {
return s.short.Load(), s.full.Load(), s.value.Load()
}
// add is a helper that adds two level's stats together.
func (s *stat) add(other *stat) *stat {
s.short.Add(other.short.Load())
s.full.Add(other.full.Load())
s.value.Add(other.value.Load())
return s
}
// display will print a table displaying the trie's node statistics.
func (s *triestat) display(title string) {
func (s *LevelStats) display(title string) {
// Shorten title if too long.
if len(title) > 32 {
title = title[0:8] + "..." + title[len(title)-8:]
@ -338,7 +273,7 @@ type jsonStat struct {
Summary jsonLevel
}
func newJsonStat(s *triestat, name string) *jsonStat {
func newJsonStat(s *LevelStats, name string) *jsonStat {
ret := jsonStat{Name: name, Summary: jsonLevel{}}
for i := 0; i < len(s.level); i++ {
// only count non-empty levels

112
trie/levelstats.go Normal file
View file

@ -0,0 +1,112 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"fmt"
"sync/atomic"
)
const trieStatLevels = 16
// LevelStats tracks the type and count of trie nodes at each level in a trie.
//
// Note: theoretically it is possible to have up to 64 trie levels, but
// LevelStats supports exactly 16 levels and panics on deeper paths.
type LevelStats struct {
level [trieStatLevels]stat
}
// NewLevelStats creates an empty trie statistics collector.
func NewLevelStats() *LevelStats {
return &LevelStats{}
}
// maxDepth iterates each level and finds the deepest level with at least one
// trie node.
func (s *LevelStats) maxDepth() int {
depth := 0
for i := range s.level {
if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 {
depth = i
}
}
return depth
}
// MaxDepth returns the deepest level with at least one trie node.
func (s *LevelStats) MaxDepth() int {
return s.maxDepth()
}
// add increases the node count by one for the specified node type and depth.
func (s *LevelStats) add(n node, depth uint32) {
d := int(depth)
switch (n).(type) {
case *shortNode:
s.level[d].short.Add(1)
case *fullNode:
s.level[d].full.Add(1)
case valueNode:
s.level[d].value.Add(1)
default:
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
}
}
// AddLeaf records a leaf depth. Witness collection reuses the value-node bucket
// for leaf accounting. It panics if the depth is outside [0, 15].
func (s *LevelStats) AddLeaf(depth int) {
s.level[depth].value.Add(1)
}
// LeafDepths returns leaf counts grouped by depth.
func (s *LevelStats) LeafDepths() [trieStatLevels]int64 {
var leaves [trieStatLevels]int64
for i := range s.level {
leaves[i] = int64(s.level[i].value.Load())
}
return leaves
}
// stat is a specific level's count of each node type.
type stat struct {
short atomic.Uint64
full atomic.Uint64
value atomic.Uint64
}
// empty is a helper that returns whether there are any trie nodes at the level.
func (s *stat) empty() bool {
if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 {
return true
}
return false
}
// load is a helper that loads each node type's value.
func (s *stat) load() (uint64, uint64, uint64) {
return s.short.Load(), s.full.Load(), s.value.Load()
}
// add is a helper that adds two level's stats together.
func (s *stat) add(other *stat) *stat {
s.short.Add(other.short.Load())
s.full.Add(other.full.Load())
s.value.Add(other.value.Load())
return s
}

37
trie/levelstats_test.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import "testing"
func TestLevelStatsAddLeafDepthBounds(t *testing.T) {
stats := NewLevelStats()
stats.AddLeaf(15)
if got := stats.LeafDepths()[15]; got != 1 {
t.Fatalf("leaf count at depth 15 = %d, want 1", got)
}
}
func TestLevelStatsAddLeafPanicsOnDepth16(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Fatal("expected panic for depth >= 16")
}
}()
NewLevelStats().AddLeaf(16)
}