From 2d3704c4d8332b0663ae3128e1ba472a99927be0 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 17 Sep 2025 15:06:39 +0200 Subject: [PATCH] core/stateless: add vmwitnessstats cli flag to report leaf stats + log to console (#32619) The format that is currently reported by the chain isn't very useful, as it gives an average for ALL the nodes, and not only the leaves, which skews the results. Also, until now there was no way to activate the reporting of errors. We also decided that metrics weren't the right tool to report this data, so we decided to dump it to the console if the flag is enabled. A better system should be built, but for now, printing to the logs does the job. --- cmd/geth/main.go | 2 ++ cmd/utils/flags.go | 22 ++++++++++++++ core/blockchain.go | 2 +- core/stateless/stats.go | 16 +++++++++- core/stateless/stats_test.go | 58 ++++++++++++++++++++++++++++++++++++ eth/backend.go | 2 ++ eth/ethconfig/config.go | 6 ++++ eth/ethconfig/gen_config.go | 12 ++++++++ 8 files changed, 118 insertions(+), 2 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 750bf55927..b661228681 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -133,6 +133,8 @@ var ( utils.VMEnableDebugFlag, utils.VMTraceFlag, utils.VMTraceJsonConfigFlag, + utils.VMWitnessStatsFlag, + utils.VMStatelessSelfValidationFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, utils.GpoBlocksFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a134ea4308..83d1c8bda5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -571,6 +571,16 @@ var ( Value: "{}", Category: flags.VMCategory, } + VMWitnessStatsFlag = &cli.BoolFlag{ + Name: "vmwitnessstats", + Usage: "Enable collection of witness trie access statistics (automatically enables witness generation)", + Category: flags.VMCategory, + } + VMStatelessSelfValidationFlag = &cli.BoolFlag{ + Name: "stateless-self-validation", + Usage: "Generate execution witnesses and self-check against them (testing purpose)", + Category: flags.VMCategory, + } // API options. RPCGlobalGasCapFlag = &cli.Uint64Flag{ Name: "rpc.gascap", @@ -1707,6 +1717,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(VMEnableDebugFlag.Name) { cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name) } + if ctx.IsSet(VMWitnessStatsFlag.Name) { + cfg.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name) + } + if ctx.IsSet(VMStatelessSelfValidationFlag.Name) { + cfg.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name) + } + // Auto-enable StatelessSelfValidation when witness stats are enabled + if ctx.Bool(VMWitnessStatsFlag.Name) { + cfg.StatelessSelfValidation = true + } if ctx.IsSet(RPCGlobalGasCapFlag.Name) { cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name) @@ -2243,6 +2263,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } vmcfg := vm.Config{ EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name), + EnableWitnessStats: ctx.Bool(VMWitnessStatsFlag.Name), + StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name), } if ctx.IsSet(VMTraceFlag.Name) { if name := ctx.String(VMTraceFlag.Name); name != "" { diff --git a/core/blockchain.go b/core/blockchain.go index 939b46634f..3466923648 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2157,7 +2157,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s } // Report the collected witness statistics if witnessStats != nil { - witnessStats.ReportMetrics() + witnessStats.ReportMetrics(block.NumberU64()) } // Update the metrics touched during block commit diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 1a6389284c..94f5587f99 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -17,6 +17,7 @@ package stateless import ( + "encoding/json" "maps" "slices" "sort" @@ -24,6 +25,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" ) @@ -70,7 +72,19 @@ func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { } // ReportMetrics reports the collected statistics to the global metrics registry. -func (s *WitnessStats) ReportMetrics() { +func (s *WitnessStats) ReportMetrics(blockNumber uint64) { + // Encode the metrics as JSON for easier consumption + accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves) + storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves) + + // Log account trie depth statistics + log.Info("Account trie depth stats", + "block", blockNumber, + "leavesAtDepth", string(accountLeavesJson)) + log.Info("Storage trie depth stats", + "block", blockNumber, + "leavesAtDepth", string(storageLeavesJson)) + for i := 0; i < 16; i++ { accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i]) storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i]) diff --git a/core/stateless/stats_test.go b/core/stateless/stats_test.go index 6219e622f5..e77084df5d 100644 --- a/core/stateless/stats_test.go +++ b/core/stateless/stats_test.go @@ -140,6 +140,64 @@ func TestWitnessStatsAdd(t *testing.T) { } } +func TestWitnessStatsMinMax(t *testing.T) { + stats := NewWitnessStats() + + // Add some account trie nodes with varying depths + stats.Add(map[string][]byte{ + "a": []byte("data1"), + "ab": []byte("data2"), + "abc": []byte("data3"), + "abcd": []byte("data4"), + "abcde": []byte("data5"), + }, common.Hash{}) + + // Only "abcde" is a leaf (depth 5) + for i, v := range stats.accountTrieLeaves { + if v != 0 && i != 5 { + t.Errorf("leaf found at invalid depth %d", i) + } + } + + // Add more leaves with different depths + stats.Add(map[string][]byte{ + "x": []byte("data6"), + "yz": []byte("data7"), + }, common.Hash{}) + + // Now we have leaves at depths 1, 2, and 5 + for i, v := range stats.accountTrieLeaves { + if v != 0 && (i != 5 && i != 2 && i != 1) { + t.Errorf("leaf found at invalid depth %d", i) + } + } +} + +func TestWitnessStatsAverage(t *testing.T) { + stats := NewWitnessStats() + + // Add nodes that will create leaves at depths 2, 3, and 4 + stats.Add(map[string][]byte{ + "aa": []byte("data1"), + "bb": []byte("data2"), + "ccc": []byte("data3"), + "dddd": []byte("data4"), + }, common.Hash{}) + + // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples + expectedAvg := int64(11) / int64(4) + var actualAvg, totalSamples int64 + for i, c := range stats.accountTrieLeaves { + actualAvg += c * int64(i) + totalSamples += c + } + actualAvg = actualAvg / totalSamples + + if actualAvg != expectedAvg { + t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg) + } +} + func BenchmarkWitnessStatsAdd(b *testing.B) { // Create a realistic trie node structure nodes := make(map[string][]byte) diff --git a/eth/backend.go b/eth/backend.go index 4356733189..3bfe0765f4 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -235,6 +235,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, + EnableWitnessStats: config.EnableWitnessStats, + StatelessSelfValidation: config.StatelessSelfValidation, }, // Enables file journaling for the trie database. The journal files will be stored // within the data directory. The corresponding paths will be either: diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index dc77141081..ba0a7762c7 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -144,6 +144,12 @@ type Config struct { // Enables tracking of SHA3 preimages in the VM EnablePreimageRecording bool + // Enables collection of witness trie access statistics + EnableWitnessStats bool + + // Generate execution witnesses and self-check against them (testing purpose) + StatelessSelfValidation bool + // Enables tracking of state size EnableStateSizeTracking bool diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 2fdd219dee..b54ba14d68 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -49,6 +49,8 @@ func (c Config) MarshalTOML() (interface{}, error) { BlobPool blobpool.Config GPO gasprice.Config EnablePreimageRecording bool + EnableWitnessStats bool + StatelessSelfValidation bool EnableStateSizeTracking bool VMTrace string VMTraceJsonConfig string @@ -91,6 +93,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.BlobPool = c.BlobPool enc.GPO = c.GPO enc.EnablePreimageRecording = c.EnablePreimageRecording + enc.EnableWitnessStats = c.EnableWitnessStats + enc.StatelessSelfValidation = c.StatelessSelfValidation enc.EnableStateSizeTracking = c.EnableStateSizeTracking enc.VMTrace = c.VMTrace enc.VMTraceJsonConfig = c.VMTraceJsonConfig @@ -137,6 +141,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { BlobPool *blobpool.Config GPO *gasprice.Config EnablePreimageRecording *bool + EnableWitnessStats *bool + StatelessSelfValidation *bool EnableStateSizeTracking *bool VMTrace *string VMTraceJsonConfig *string @@ -246,6 +252,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.EnablePreimageRecording != nil { c.EnablePreimageRecording = *dec.EnablePreimageRecording } + if dec.EnableWitnessStats != nil { + c.EnableWitnessStats = *dec.EnableWitnessStats + } + if dec.StatelessSelfValidation != nil { + c.StatelessSelfValidation = *dec.StatelessSelfValidation + } if dec.EnableStateSizeTracking != nil { c.EnableStateSizeTracking = *dec.EnableStateSizeTracking }