core/stateless: add vmwitnessstats cli flag to report leaf stats + log to console (#32619)

The format that is currently reported by the chain isn't very useful, as
it gives an average for ALL the nodes, and not only the leaves, which
skews the results.

Also, until now there was no way to activate the reporting of errors.

We also decided that metrics weren't the right tool to report this data,
so we decided to dump it to the console if the flag is enabled. A better
system should be built, but for now, printing to the logs does the job.

cmd/geth/main.go

@@ -133,6 +133,8 @@ var (
 		utils.VMEnableDebugFlag,
 		utils.VMTraceFlag,
 		utils.VMTraceJsonConfigFlag,
+		utils.VMWitnessStatsFlag,
+		utils.VMStatelessSelfValidationFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,
 		utils.GpoBlocksFlag,

cmd/utils/flags.go

@@ -571,6 +571,16 @@ var (
 		Value:    "{}",
 		Category: flags.VMCategory,
 	}
+	VMWitnessStatsFlag = &cli.BoolFlag{
+		Name:     "vmwitnessstats",
+		Usage:    "Enable collection of witness trie access statistics (automatically enables witness generation)",
+		Category: flags.VMCategory,
+	}
+	VMStatelessSelfValidationFlag = &cli.BoolFlag{
+		Name:     "stateless-self-validation",
+		Usage:    "Generate execution witnesses and self-check against them (testing purpose)",
+		Category: flags.VMCategory,
+	}
 	// API options.
 	RPCGlobalGasCapFlag = &cli.Uint64Flag{
 		Name:     "rpc.gascap",
@@ -1707,6 +1717,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(VMEnableDebugFlag.Name) {
 		cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name)
 	}
+	if ctx.IsSet(VMWitnessStatsFlag.Name) {
+		cfg.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name)
+	}
+	if ctx.IsSet(VMStatelessSelfValidationFlag.Name) {
+		cfg.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name)
+	}
+	// Auto-enable StatelessSelfValidation when witness stats are enabled
+	if ctx.Bool(VMWitnessStatsFlag.Name) {
+		cfg.StatelessSelfValidation = true
+	}
 
 	if ctx.IsSet(RPCGlobalGasCapFlag.Name) {
 		cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name)
@@ -2243,6 +2263,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	}
 	vmcfg := vm.Config{
 		EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name),
+		EnableWitnessStats:      ctx.Bool(VMWitnessStatsFlag.Name),
+		StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name),
 	}
 	if ctx.IsSet(VMTraceFlag.Name) {
 		if name := ctx.String(VMTraceFlag.Name); name != "" {

core/blockchain.go

@@ -2157,7 +2157,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s
 	}
 	// Report the collected witness statistics
 	if witnessStats != nil {
-		witnessStats.ReportMetrics()
+		witnessStats.ReportMetrics(block.NumberU64())
 	}
 
 	// Update the metrics touched during block commit

core/stateless/stats.go

@@ -17,6 +17,7 @@
 package stateless
 
 import (
+	"encoding/json"
 	"maps"
 	"slices"
 	"sort"
@@ -24,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 )
 
@@ -70,7 +72,19 @@ func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) {
 }
 
 // ReportMetrics reports the collected statistics to the global metrics registry.
-func (s *WitnessStats) ReportMetrics() {
+func (s *WitnessStats) ReportMetrics(blockNumber uint64) {
+	// Encode the metrics as JSON for easier consumption
+	accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves)
+	storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves)
+
+	// Log account trie depth statistics
+	log.Info("Account trie depth stats",
+		"block", blockNumber,
+		"leavesAtDepth", string(accountLeavesJson))
+	log.Info("Storage trie depth stats",
+		"block", blockNumber,
+		"leavesAtDepth", string(storageLeavesJson))
+
 	for i := 0; i < 16; i++ {
 		accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i])
 		storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i])

core/stateless/stats_test.go

@@ -140,6 +140,64 @@ func TestWitnessStatsAdd(t *testing.T) {
 	}
 }
 
+func TestWitnessStatsMinMax(t *testing.T) {
+	stats := NewWitnessStats()
+
+	// Add some account trie nodes with varying depths
+	stats.Add(map[string][]byte{
+		"a":     []byte("data1"),
+		"ab":    []byte("data2"),
+		"abc":   []byte("data3"),
+		"abcd":  []byte("data4"),
+		"abcde": []byte("data5"),
+	}, common.Hash{})
+
+	// Only "abcde" is a leaf (depth 5)
+	for i, v := range stats.accountTrieLeaves {
+		if v != 0 && i != 5 {
+			t.Errorf("leaf found at invalid depth %d", i)
+		}
+	}
+
+	// Add more leaves with different depths
+	stats.Add(map[string][]byte{
+		"x":  []byte("data6"),
+		"yz": []byte("data7"),
+	}, common.Hash{})
+
+	// Now we have leaves at depths 1, 2, and 5
+	for i, v := range stats.accountTrieLeaves {
+		if v != 0 && (i != 5 && i != 2 && i != 1) {
+			t.Errorf("leaf found at invalid depth %d", i)
+		}
+	}
+}
+
+func TestWitnessStatsAverage(t *testing.T) {
+	stats := NewWitnessStats()
+
+	// Add nodes that will create leaves at depths 2, 3, and 4
+	stats.Add(map[string][]byte{
+		"aa":   []byte("data1"),
+		"bb":   []byte("data2"),
+		"ccc":  []byte("data3"),
+		"dddd": []byte("data4"),
+	}, common.Hash{})
+
+	// All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples
+	expectedAvg := int64(11) / int64(4)
+	var actualAvg, totalSamples int64
+	for i, c := range stats.accountTrieLeaves {
+		actualAvg += c * int64(i)
+		totalSamples += c
+	}
+	actualAvg = actualAvg / totalSamples
+
+	if actualAvg != expectedAvg {
+		t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg)
+	}
+}
+
 func BenchmarkWitnessStatsAdd(b *testing.B) {
 	// Create a realistic trie node structure
 	nodes := make(map[string][]byte)

eth/backend.go

@@ -235,6 +235,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			TxLookupLimit:    int64(min(config.TransactionHistory, math.MaxInt64)),
 			VmConfig: vm.Config{
 				EnablePreimageRecording: config.EnablePreimageRecording,
+				EnableWitnessStats:      config.EnableWitnessStats,
+				StatelessSelfValidation: config.StatelessSelfValidation,
 			},
 			// Enables file journaling for the trie database. The journal files will be stored
 			// within the data directory. The corresponding paths will be either:

eth/ethconfig/config.go

@@ -144,6 +144,12 @@ type Config struct {
 	// Enables tracking of SHA3 preimages in the VM
 	EnablePreimageRecording bool
 
+	// Enables collection of witness trie access statistics
+	EnableWitnessStats bool
+
+	// Generate execution witnesses and self-check against them (testing purpose)
+	StatelessSelfValidation bool
+
 	// Enables tracking of state size
 	EnableStateSizeTracking bool
 

eth/ethconfig/gen_config.go

@@ -49,6 +49,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		BlobPool                blobpool.Config
 		GPO                     gasprice.Config
 		EnablePreimageRecording bool
+		EnableWitnessStats      bool
+		StatelessSelfValidation bool
 		EnableStateSizeTracking bool
 		VMTrace                 string
 		VMTraceJsonConfig       string
@@ -91,6 +93,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.BlobPool = c.BlobPool
 	enc.GPO = c.GPO
 	enc.EnablePreimageRecording = c.EnablePreimageRecording
+	enc.EnableWitnessStats = c.EnableWitnessStats
+	enc.StatelessSelfValidation = c.StatelessSelfValidation
 	enc.EnableStateSizeTracking = c.EnableStateSizeTracking
 	enc.VMTrace = c.VMTrace
 	enc.VMTraceJsonConfig = c.VMTraceJsonConfig
@@ -137,6 +141,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		BlobPool                *blobpool.Config
 		GPO                     *gasprice.Config
 		EnablePreimageRecording *bool
+		EnableWitnessStats      *bool
+		StatelessSelfValidation *bool
 		EnableStateSizeTracking *bool
 		VMTrace                 *string
 		VMTraceJsonConfig       *string
@@ -246,6 +252,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.EnablePreimageRecording != nil {
 		c.EnablePreimageRecording = *dec.EnablePreimageRecording
 	}
+	if dec.EnableWitnessStats != nil {
+		c.EnableWitnessStats = *dec.EnableWitnessStats
+	}
+	if dec.StatelessSelfValidation != nil {
+		c.StatelessSelfValidation = *dec.StatelessSelfValidation
+	}
 	if dec.EnableStateSizeTracking != nil {
 		c.EnableStateSizeTracking = *dec.EnableStateSizeTracking
 	}