From 97cdc16c3b036dbd49745b4a7324f18b71c11a69 Mon Sep 17 00:00:00 2001 From: lightclient Date: Tue, 17 Feb 2026 05:08:42 +0000 Subject: [PATCH] trie,core: modify triestat to be used by stateless WitnessStats --- core/stateless/stats.go | 68 ++++++++++++--------- core/stateless/stats_test.go | 108 ++++++++++++++++++++++----------- trie/inspect.go | 93 +++++------------------------ trie/levelstats.go | 112 +++++++++++++++++++++++++++++++++++ trie/levelstats_test.go | 37 ++++++++++++ 5 files changed, 274 insertions(+), 144 deletions(-) create mode 100644 trie/levelstats.go create mode 100644 trie/levelstats_test.go diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 73ce031bff..7f4473a67c 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/trie" ) var accountTrieLeavesAtDepth [16]*metrics.Counter @@ -41,59 +42,68 @@ func init() { // WitnessStats aggregates statistics for account and storage trie accesses. type WitnessStats struct { - accountTrieLeaves [16]int64 - storageTrieLeaves [16]int64 + accountTrie *trie.LevelStats + storageTrie *trie.LevelStats } // NewWitnessStats creates a new WitnessStats collector. func NewWitnessStats() *WitnessStats { - return &WitnessStats{} + return &WitnessStats{ + accountTrie: trie.NewLevelStats(), + storageTrie: trie.NewLevelStats(), + } +} + +func (s *WitnessStats) init() { + if s.accountTrie == nil { + s.accountTrie = trie.NewLevelStats() + } + if s.storageTrie == nil { + s.storageTrie = trie.NewLevelStats() + } } // Add records trie access depths from the given node paths. // If `owner` is the zero hash, accesses are attributed to the account trie; // otherwise, they are attributed to the storage trie of that account. func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { - // Extract paths from the nodes map + s.init() + + // Extract paths from the nodes map. paths := slices.Collect(maps.Keys(nodes)) sort.Strings(paths) + ownerStat := s.accountTrie + if owner != (common.Hash{}) { + ownerStat = s.storageTrie + } + for i, path := range paths { // If current path is a prefix of the next path, it's not a leaf. // The last path is always a leaf. if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { - depth := len(path) - if owner == (common.Hash{}) { - if depth >= len(s.accountTrieLeaves) { - depth = len(s.accountTrieLeaves) - 1 - } - s.accountTrieLeaves[depth] += 1 - } else { - if depth >= len(s.storageTrieLeaves) { - depth = len(s.storageTrieLeaves) - 1 - } - s.storageTrieLeaves[depth] += 1 - } + ownerStat.AddLeaf(len(path)) } } } // ReportMetrics reports the collected statistics to the global metrics registry. func (s *WitnessStats) ReportMetrics(blockNumber uint64) { - // Encode the metrics as JSON for easier consumption - accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves) - storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves) + s.init() - // Log account trie depth statistics - log.Info("Account trie depth stats", - "block", blockNumber, - "leavesAtDepth", string(accountLeavesJson)) - log.Info("Storage trie depth stats", - "block", blockNumber, - "leavesAtDepth", string(storageLeavesJson)) + accountTrieLeaves := s.accountTrie.LeafDepths() + storageTrieLeaves := s.storageTrie.LeafDepths() - for i := 0; i < 16; i++ { - accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i]) - storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i]) + // Encode the metrics as JSON for easier consumption. + accountLeavesJSON, _ := json.Marshal(accountTrieLeaves) + storageLeavesJSON, _ := json.Marshal(storageTrieLeaves) + + // Log account trie depth statistics. + log.Info("Account trie depth stats", "block", blockNumber, "leavesAtDepth", string(accountLeavesJSON)) + log.Info("Storage trie depth stats", "block", blockNumber, "leavesAtDepth", string(storageLeavesJSON)) + + for i := 0; i < len(accountTrieLeavesAtDepth); i++ { + accountTrieLeavesAtDepth[i].Inc(accountTrieLeaves[i]) + storageTrieLeavesAtDepth[i].Inc(storageTrieLeaves[i]) } } diff --git a/core/stateless/stats_test.go b/core/stateless/stats_test.go index e77084df5d..6dc1fa0844 100644 --- a/core/stateless/stats_test.go +++ b/core/stateless/stats_test.go @@ -17,18 +17,27 @@ package stateless import ( + "strings" "testing" "github.com/ethereum/go-ethereum/common" ) +func expectedLeaves(counts map[int]int64) [16]int64 { + var leaves [16]int64 + for depth, count := range counts { + leaves[depth] = count + } + return leaves +} + func TestWitnessStatsAdd(t *testing.T) { tests := []struct { name string nodes map[string][]byte owner common.Hash - expectedAccountLeaves map[int64]int64 - expectedStorageLeaves map[int64]int64 + expectedAccountLeaves map[int]int64 + expectedStorageLeaves map[int]int64 }{ { name: "empty nodes", @@ -41,7 +50,7 @@ func TestWitnessStatsAdd(t *testing.T) { "": []byte("data"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{0: 1}, + expectedAccountLeaves: map[int]int64{0: 1}, }, { name: "single account trie leaf", @@ -49,7 +58,7 @@ func TestWitnessStatsAdd(t *testing.T) { "abc": []byte("data"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 1}, + expectedAccountLeaves: map[int]int64{3: 1}, }, { name: "account trie with internal nodes", @@ -59,7 +68,7 @@ func TestWitnessStatsAdd(t *testing.T) { "abc": []byte("data3"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 1}, // Only "abc" is a leaf + expectedAccountLeaves: map[int]int64{3: 1}, // Only "abc" is a leaf }, { name: "multiple account trie branches", @@ -72,7 +81,7 @@ func TestWitnessStatsAdd(t *testing.T) { "bcd": []byte("data6"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 2}, // "abc" (3) + "bcd" (3) + expectedAccountLeaves: map[int]int64{3: 2}, // "abc" (3) + "bcd" (3) }, { name: "siblings are all leaves", @@ -82,7 +91,7 @@ func TestWitnessStatsAdd(t *testing.T) { "ac": []byte("data3"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{2: 3}, + expectedAccountLeaves: map[int]int64{2: 3}, }, { name: "storage trie leaves", @@ -93,7 +102,7 @@ func TestWitnessStatsAdd(t *testing.T) { "124": []byte("data4"), }, owner: common.HexToHash("0x1234"), - expectedStorageLeaves: map[int64]int64{3: 2}, // "123" (3) + "124" (3) + expectedStorageLeaves: map[int]int64{3: 2}, // "123" (3) + "124" (3) }, { name: "complex trie structure", @@ -109,7 +118,7 @@ func TestWitnessStatsAdd(t *testing.T) { "3": []byte("data9"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1) + expectedAccountLeaves: map[int]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1) }, } @@ -118,32 +127,59 @@ func TestWitnessStatsAdd(t *testing.T) { stats := NewWitnessStats() stats.Add(tt.nodes, tt.owner) - var expectedAccountTrieLeaves [16]int64 - for depth, count := range tt.expectedAccountLeaves { - expectedAccountTrieLeaves[depth] = count + if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(tt.expectedAccountLeaves); got != want { + t.Errorf("account trie leaves = %v, want %v", got, want) } - var expectedStorageTrieLeaves [16]int64 - for depth, count := range tt.expectedStorageLeaves { - expectedStorageTrieLeaves[depth] = count - } - - // Check account trie depth - if stats.accountTrieLeaves != expectedAccountTrieLeaves { - t.Errorf("Account trie total depth = %v, want %v", stats.accountTrieLeaves, expectedAccountTrieLeaves) - } - - // Check storage trie depth - if stats.storageTrieLeaves != expectedStorageTrieLeaves { - t.Errorf("Storage trie total depth = %v, want %v", stats.storageTrieLeaves, expectedStorageTrieLeaves) + if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(tt.expectedStorageLeaves); got != want { + t.Errorf("storage trie leaves = %v, want %v", got, want) } }) } } +func TestWitnessStatsStorageTrieAggregation(t *testing.T) { + stats := NewWitnessStats() + ownerA := common.HexToHash("0xa") + ownerB := common.HexToHash("0xb") + + stats.Add(map[string][]byte{ + "a": []byte("data1"), + "ab": []byte("data2"), + "abc": []byte("data3"), + }, ownerA) + stats.Add(map[string][]byte{ + "xy": []byte("data4"), + }, ownerA) + stats.Add(map[string][]byte{ + "1": []byte("data5"), + "12": []byte("data6"), + "123": []byte("data7"), + "124": []byte("data8"), + }, ownerB) + + if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(map[int]int64{2: 1, 3: 3}); got != want { + t.Errorf("storage leaves = %v, want %v", got, want) + } + if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(nil); got != want { + t.Errorf("account leaves = %v, want %v", got, want) + } +} + +func TestWitnessStatsPanicsOnDeepLeaf(t *testing.T) { + stats := NewWitnessStats() + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + stats.Add(map[string][]byte{strings.Repeat("a", 16): []byte("data")}, common.Hash{}) +} + func TestWitnessStatsMinMax(t *testing.T) { stats := NewWitnessStats() - // Add some account trie nodes with varying depths + // Add some account trie nodes with varying depths. stats.Add(map[string][]byte{ "a": []byte("data1"), "ab": []byte("data2"), @@ -152,21 +188,21 @@ func TestWitnessStatsMinMax(t *testing.T) { "abcde": []byte("data5"), }, common.Hash{}) - // Only "abcde" is a leaf (depth 5) - for i, v := range stats.accountTrieLeaves { + // Only "abcde" is a leaf (depth 5). + for i, v := range stats.accountTrie.LeafDepths() { if v != 0 && i != 5 { t.Errorf("leaf found at invalid depth %d", i) } } - // Add more leaves with different depths + // Add more leaves with different depths. stats.Add(map[string][]byte{ "x": []byte("data6"), "yz": []byte("data7"), }, common.Hash{}) - // Now we have leaves at depths 1, 2, and 5 - for i, v := range stats.accountTrieLeaves { + // Now we have leaves at depths 1, 2, and 5. + for i, v := range stats.accountTrie.LeafDepths() { if v != 0 && (i != 5 && i != 2 && i != 1) { t.Errorf("leaf found at invalid depth %d", i) } @@ -176,7 +212,7 @@ func TestWitnessStatsMinMax(t *testing.T) { func TestWitnessStatsAverage(t *testing.T) { stats := NewWitnessStats() - // Add nodes that will create leaves at depths 2, 3, and 4 + // Add nodes that will create leaves at depths 2, 3, and 4. stats.Add(map[string][]byte{ "aa": []byte("data1"), "bb": []byte("data2"), @@ -184,22 +220,22 @@ func TestWitnessStatsAverage(t *testing.T) { "dddd": []byte("data4"), }, common.Hash{}) - // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples + // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples. expectedAvg := int64(11) / int64(4) var actualAvg, totalSamples int64 - for i, c := range stats.accountTrieLeaves { + for i, c := range stats.accountTrie.LeafDepths() { actualAvg += c * int64(i) totalSamples += c } actualAvg = actualAvg / totalSamples if actualAvg != expectedAvg { - t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg) + t.Errorf("account trie average depth = %d, want %d", actualAvg, expectedAvg) } } func BenchmarkWitnessStatsAdd(b *testing.B) { - // Create a realistic trie node structure + // Create a realistic trie node structure. nodes := make(map[string][]byte) for i := 0; i < 100; i++ { base := string(rune('a' + i%26)) diff --git a/trie/inspect.go b/trie/inspect.go index 930f79c59b..1ddb0e6d1f 100644 --- a/trie/inspect.go +++ b/trie/inspect.go @@ -24,7 +24,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -41,7 +40,7 @@ type inspector struct { root common.Hash config *InspectConfig - stats map[common.Hash]*triestat + stats map[common.Hash]*LevelStats m sync.Mutex // protects stats sem *semaphore.Weighted @@ -72,10 +71,10 @@ func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConf triedb: triedb, root: root, config: config, - stats: make(map[common.Hash]*triestat), + stats: make(map[common.Hash]*LevelStats), sem: semaphore.NewWeighted(int64(128)), } - in.stats[root] = &triestat{} + in.stats[root] = NewLevelStats() in.inspect(trie, trie.root, 0, []byte{}, in.stats[root]) in.wg.Wait() @@ -91,7 +90,7 @@ func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConf // inspect is called recursively down the trie. At each level it records the // node type encountered. -func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *triestat) { +func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *LevelStats) { if n == nil { return } @@ -152,7 +151,7 @@ func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, sta log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path)) break } - stat := &triestat{} + stat := NewLevelStats() in.m.Lock() in.stats[owner] = stat @@ -181,7 +180,7 @@ func (in *inspector) displayResult() { if !in.config.NoStorage { // Sort stats by max node depth. - keys, stats := sortedTriestat(in.stats).sort() + keys, stats := sortedLevelStats(in.stats).sort() fmt.Println("Results for top storage tries") for i := range keys[0:min(in.config.TopN, len(keys))] { @@ -204,7 +203,7 @@ func (in *inspector) writeJSON() error { } if !in.config.NoStorage { // Sort stats by max node depth. - keys, stats := sortedTriestat(in.stats).sort() + keys, stats := sortedLevelStats(in.stats).sort() for i := range keys[0:min(in.config.TopN, len(keys))] { storageTrie := newJsonStat(stats[i], fmt.Sprintf("%x", keys[i])) if err := enc.Encode(storageTrie); err != nil { @@ -215,36 +214,14 @@ func (in *inspector) writeJSON() error { return nil } -// triestat tracks the type and count of trie nodes at each level in the trie. -// -// Note: theoretically it is possible to have up to 64 trie level. Since it is -// unlikely to encounter such a large trie, the stats are capped at 16 levels to -// avoid substantial unneeded allocation. -type triestat struct { - level [16]stat -} +// sortedLevelStats implements sort(). +type sortedLevelStats map[common.Hash]*LevelStats -// maxDepth iterates each level and finds the deepest level with at least one -// trie node. -func (s *triestat) maxDepth() int { - depth := 0 - for i := range s.level { - if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 { - depth = i - } - } - return depth -} - -// sortedTriestat implements sort(). -type sortedTriestat map[common.Hash]*triestat - -// sort returns the keys and triestats in decending order of the maximum trie -// node depth. -func (s sortedTriestat) sort() ([]common.Hash, []*triestat) { +// sort returns the keys and stats in descending order of maximum trie node depth. +func (s sortedLevelStats) sort() ([]common.Hash, []*LevelStats) { var ( keys = make([]common.Hash, 0, len(s)) - stats = make([]*triestat, 0, len(s)) + stats = make([]*LevelStats, 0, len(s)) ) for k := range s { keys = append(keys, k) @@ -256,50 +233,8 @@ func (s sortedTriestat) sort() ([]common.Hash, []*triestat) { return keys, stats } -// add increases the node count by one for the specified node type and depth. -func (s *triestat) add(n node, d uint32) { - switch (n).(type) { - case *shortNode: - s.level[d].short.Add(1) - case *fullNode: - s.level[d].full.Add(1) - case valueNode: - s.level[d].value.Add(1) - default: - panic(fmt.Sprintf("%T: invalid node: %v", n, n)) - } -} - -// stat is a specific level's count of each node type. -type stat struct { - short atomic.Uint64 - full atomic.Uint64 - value atomic.Uint64 -} - -// empty is a helper that returns whether there are any trie nodes at the level. -func (s *stat) empty() bool { - if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 { - return true - } - return false -} - -// load is a helper that loads each node type's value. -func (s *stat) load() (uint64, uint64, uint64) { - return s.short.Load(), s.full.Load(), s.value.Load() -} - -// add is a helper that adds two level's stats together. -func (s *stat) add(other *stat) *stat { - s.short.Add(other.short.Load()) - s.full.Add(other.full.Load()) - s.value.Add(other.value.Load()) - return s -} - // display will print a table displaying the trie's node statistics. -func (s *triestat) display(title string) { +func (s *LevelStats) display(title string) { // Shorten title if too long. if len(title) > 32 { title = title[0:8] + "..." + title[len(title)-8:] @@ -338,7 +273,7 @@ type jsonStat struct { Summary jsonLevel } -func newJsonStat(s *triestat, name string) *jsonStat { +func newJsonStat(s *LevelStats, name string) *jsonStat { ret := jsonStat{Name: name, Summary: jsonLevel{}} for i := 0; i < len(s.level); i++ { // only count non-empty levels diff --git a/trie/levelstats.go b/trie/levelstats.go new file mode 100644 index 0000000000..639ff4ef24 --- /dev/null +++ b/trie/levelstats.go @@ -0,0 +1,112 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "sync/atomic" +) + +const trieStatLevels = 16 + +// LevelStats tracks the type and count of trie nodes at each level in a trie. +// +// Note: theoretically it is possible to have up to 64 trie levels, but +// LevelStats supports exactly 16 levels and panics on deeper paths. +type LevelStats struct { + level [trieStatLevels]stat +} + +// NewLevelStats creates an empty trie statistics collector. +func NewLevelStats() *LevelStats { + return &LevelStats{} +} + +// maxDepth iterates each level and finds the deepest level with at least one +// trie node. +func (s *LevelStats) maxDepth() int { + depth := 0 + for i := range s.level { + if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 { + depth = i + } + } + return depth +} + +// MaxDepth returns the deepest level with at least one trie node. +func (s *LevelStats) MaxDepth() int { + return s.maxDepth() +} + +// add increases the node count by one for the specified node type and depth. +func (s *LevelStats) add(n node, depth uint32) { + d := int(depth) + switch (n).(type) { + case *shortNode: + s.level[d].short.Add(1) + case *fullNode: + s.level[d].full.Add(1) + case valueNode: + s.level[d].value.Add(1) + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} + +// AddLeaf records a leaf depth. Witness collection reuses the value-node bucket +// for leaf accounting. It panics if the depth is outside [0, 15]. +func (s *LevelStats) AddLeaf(depth int) { + s.level[depth].value.Add(1) +} + +// LeafDepths returns leaf counts grouped by depth. +func (s *LevelStats) LeafDepths() [trieStatLevels]int64 { + var leaves [trieStatLevels]int64 + for i := range s.level { + leaves[i] = int64(s.level[i].value.Load()) + } + return leaves +} + +// stat is a specific level's count of each node type. +type stat struct { + short atomic.Uint64 + full atomic.Uint64 + value atomic.Uint64 +} + +// empty is a helper that returns whether there are any trie nodes at the level. +func (s *stat) empty() bool { + if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 { + return true + } + return false +} + +// load is a helper that loads each node type's value. +func (s *stat) load() (uint64, uint64, uint64) { + return s.short.Load(), s.full.Load(), s.value.Load() +} + +// add is a helper that adds two level's stats together. +func (s *stat) add(other *stat) *stat { + s.short.Add(other.short.Load()) + s.full.Add(other.full.Load()) + s.value.Add(other.value.Load()) + return s +} diff --git a/trie/levelstats_test.go b/trie/levelstats_test.go new file mode 100644 index 0000000000..90581eb1ab --- /dev/null +++ b/trie/levelstats_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import "testing" + +func TestLevelStatsAddLeafDepthBounds(t *testing.T) { + stats := NewLevelStats() + stats.AddLeaf(15) + + if got := stats.LeafDepths()[15]; got != 1 { + t.Fatalf("leaf count at depth 15 = %d, want 1", got) + } +} + +func TestLevelStatsAddLeafPanicsOnDepth16(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + NewLevelStats().AddLeaf(16) +}