From fe3a74e61057459d4398ff03550eb65dd916bcb8 Mon Sep 17 00:00:00 2001 From: DeFi Junkie Date: Wed, 4 Mar 2026 08:42:25 +0300 Subject: [PATCH 01/52] core/vm: use amsterdam jump table in lookup (#33947) Return the Amsterdam instruction set from `LookupInstructionSet` when `IsAmsterdam` is true, so Amsterdam rules no longer fall through to the Osaka jump table. --------- Co-authored-by: rjl493456442 --- core/vm/jump_table_export.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index 89a2ebf6f4..fdf814d64c 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -28,6 +28,8 @@ func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { case rules.IsVerkle: return newCancunInstructionSet(), errors.New("verkle-fork not defined yet") + case rules.IsAmsterdam: + return newAmsterdamInstructionSet(), nil case rules.IsOsaka: return newOsakaInstructionSet(), nil case rules.IsPrague: From 6d99759f01dc1f8697f424a6baee93621f269069 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 4 Mar 2026 14:40:45 +0800 Subject: [PATCH 02/52] cmd, core, eth, tests: prevent state flushing in RPC (#33931) Fixes https://github.com/ethereum/go-ethereum/issues/33572 --- cmd/utils/flags.go | 5 +- core/blockchain.go | 112 ++++++++++++++++++++++++------------ core/vm/interpreter.go | 6 +- eth/api_debug.go | 33 ++++------- eth/backend.go | 5 +- internal/web3ext/web3ext.go | 11 ++-- tests/block_test_util.go | 4 +- 7 files changed, 103 insertions(+), 73 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e114eb2cd4..75b5b4785a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2475,8 +2475,6 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } vmcfg := vm.Config{ EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name), - EnableWitnessStats: ctx.Bool(VMWitnessStatsFlag.Name), - StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name), } if ctx.IsSet(VMTraceFlag.Name) { if name := ctx.String(VMTraceFlag.Name); name != "" { @@ -2490,6 +2488,9 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } options.VmConfig = vmcfg + options.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name) + options.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name) + chain, err := core.NewBlockChain(chainDb, gspec, engine, options) if err != nil { Fatalf("Can't create BlockChain: %v", err) diff --git a/core/blockchain.go b/core/blockchain.go index d41f301243..ed186ccf5e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -219,6 +219,10 @@ type BlockChainConfig struct { // detailed statistics will be logged. Negative value means disabled (default), // zero logs all blocks, positive value filters blocks by execution time. SlowBlockThreshold time.Duration + + // Execution configs + StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) + EnableWitnessStats bool // Whether trie access statistics collection is enabled } // DefaultConfig returns the default config. @@ -1990,7 +1994,15 @@ func (bc *BlockChain) insertChain(ctx context.Context, chain types.Blocks, setHe } // The traced section of block import. start := time.Now() - res, err := bc.ProcessBlock(ctx, parent.Root, block, setHead, makeWitness && len(chain) == 1) + config := ExecuteConfig{ + WriteState: true, + WriteHead: setHead, + EnableTracer: true, + MakeWitness: makeWitness && len(chain) == 1, + StatelessSelfValidation: bc.cfg.StatelessSelfValidation, + EnableWitnessStats: bc.cfg.EnableWitnessStats, + } + res, err := bc.ProcessBlock(ctx, parent.Root, block, config) if err != nil { return nil, it.index, err } @@ -2073,9 +2085,36 @@ func (bpr *blockProcessingResult) Stats() *ExecuteStats { return bpr.stats } +// ExecuteConfig defines optional behaviors during execution. +type ExecuteConfig struct { + // WriteState controls whether the computed state changes are persisted to + // the underlying storage. If false, execution is performed in-memory only. + WriteState bool + + // WriteHead indicates whether the execution result should update the canonical + // chain head. It's only relevant with WriteState == True. + WriteHead bool + + // EnableTracer enables execution tracing. This is typically used for debugging + // or analysis and may significantly impact performance. + EnableTracer bool + + // MakeWitness indicates whether to generate execution witness data during + // execution. Enabling this may introduce additional memory and CPU overhead. + MakeWitness bool + + // StatelessSelfValidation indicates whether the execution witnesses generation + // and self-validation (testing purpose) is enabled. + StatelessSelfValidation bool + + // EnableWitnessStats indicates whether to enable collection of witness trie + // access statistics + EnableWitnessStats bool +} + // ProcessBlock executes and validates the given block. If there was no error // it writes the block and associated state to database. -func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (result *blockProcessingResult, blockEndErr error) { +func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, block *types.Block, config ExecuteConfig) (result *blockProcessingResult, blockEndErr error) { var ( err error startTime = time.Now() @@ -2138,12 +2177,12 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, // Generate witnesses either if we're self-testing, or if it's the // only block being inserted. A bit crude, but witnesses are huge, // so we refuse to make an entire chain of them. - if bc.cfg.VmConfig.StatelessSelfValidation || makeWitness { + if config.StatelessSelfValidation || config.MakeWitness { witness, err = stateless.NewWitness(block.Header(), bc) if err != nil { return nil, err } - if bc.cfg.VmConfig.EnableWitnessStats { + if config.EnableWitnessStats { witnessStats = stateless.NewWitnessStats() } } @@ -2151,17 +2190,20 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, defer statedb.StopPrefetcher() } - if bc.logger != nil && bc.logger.OnBlockStart != nil { - bc.logger.OnBlockStart(tracing.BlockEvent{ - Block: block, - Finalized: bc.CurrentFinalBlock(), - Safe: bc.CurrentSafeBlock(), - }) - } - if bc.logger != nil && bc.logger.OnBlockEnd != nil { - defer func() { - bc.logger.OnBlockEnd(blockEndErr) - }() + // Instrument the blockchain tracing + if config.EnableTracer { + if bc.logger != nil && bc.logger.OnBlockStart != nil { + bc.logger.OnBlockStart(tracing.BlockEvent{ + Block: block, + Finalized: bc.CurrentFinalBlock(), + Safe: bc.CurrentSafeBlock(), + }) + } + if bc.logger != nil && bc.logger.OnBlockEnd != nil { + defer func() { + bc.logger.OnBlockEnd(blockEndErr) + }() + } } // Process block using the parent state as reference point @@ -2191,7 +2233,7 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, // witness builder/runner, which would otherwise be impossible due to the // various invalid chain states/behaviors being contained in those tests. xvstart := time.Now() - if witness := statedb.Witness(); witness != nil && bc.cfg.VmConfig.StatelessSelfValidation { + if witness := statedb.Witness(); witness != nil && config.StatelessSelfValidation { log.Warn("Running stateless self-validation", "block", block.Number(), "hash", block.Hash()) // Remove critical computed fields from the block to force true recalculation @@ -2244,31 +2286,29 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, stats.CrossValidation = xvtime // The time spent on stateless cross validation // Write the block to the chain and get the status. - var ( - wstart = time.Now() - status WriteStatus - ) - if !setHead { - // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, res.Receipts, statedb) - } else { - status, err = bc.writeBlockAndSetHead(block, res.Receipts, res.Logs, statedb, false) - } - if err != nil { - return nil, err + var status WriteStatus + if config.WriteState { + wstart := time.Now() + if !config.WriteHead { + // Don't set the head, only insert the block + err = bc.writeBlockWithState(block, res.Receipts, statedb) + } else { + status, err = bc.writeBlockAndSetHead(block, res.Receipts, res.Logs, statedb, false) + } + if err != nil { + return nil, err + } + // Update the metrics touched during block commit + stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them + stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them + stats.SnapshotCommit = statedb.SnapshotCommits // Snapshot commits are complete, we can mark them + stats.TrieDBCommit = statedb.TrieDBCommits // Trie database commits are complete, we can mark them + stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits } // Report the collected witness statistics if witnessStats != nil { witnessStats.ReportMetrics(block.NumberU64()) } - - // Update the metrics touched during block commit - stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them - stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them - stats.SnapshotCommit = statedb.SnapshotCommits // Snapshot commits are complete, we can mark them - stats.TrieDBCommit = statedb.TrieDBCommits // Trie database commits are complete, we can mark them - stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits - elapsed := time.Since(startTime) + 1 // prevent zero division stats.TotalTime = elapsed stats.MgasPerSecond = float64(res.GasUsed) * 1000 / float64(elapsed) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 52dbe83d86..620c069fc8 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -27,13 +27,11 @@ import ( // Config are the configuration options for the Interpreter type Config struct { - Tracer *tracing.Hooks + Tracer *tracing.Hooks + NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages ExtraEips []int // Additional EIPS that are to be enabled - - StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) - EnableWitnessStats bool // Whether trie access statistics collection is enabled } // ScopeContext contains the things that are per-call, such as stack and memory, diff --git a/eth/api_debug.go b/eth/api_debug.go index d4ef4cc87d..b8267902b2 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/stateless" @@ -493,34 +494,22 @@ func (api *DebugAPI) StateSize(blockHashOrNumber *rpc.BlockNumberOrHash) (interf }, nil } -func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumber) (*stateless.ExtWitness, error) { +func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumberOrHash) (*stateless.ExtWitness, error) { bc := api.eth.blockchain - block, err := api.eth.APIBackend.BlockByNumber(context.Background(), bn) + block, err := api.eth.APIBackend.BlockByNumberOrHash(context.Background(), bn) if err != nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %v not found", bn) + return &stateless.ExtWitness{}, fmt.Errorf("block %v not found", bn) } parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %v found, but parent missing", bn) + return &stateless.ExtWitness{}, fmt.Errorf("block %v found, but parent missing", bn) } - result, err := bc.ProcessBlock(context.Background(), parent.Root, block, false, true) - if err != nil { - return nil, err - } - return result.Witness().ToExtWitness(), nil -} - -func (api *DebugAPI) ExecutionWitnessByHash(hash common.Hash) (*stateless.ExtWitness, error) { - bc := api.eth.blockchain - block := bc.GetBlockByHash(hash) - if block == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block hash %x not found", hash) - } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %x found, but parent missing", hash) - } - result, err := bc.ProcessBlock(context.Background(), parent.Root, block, false, true) + config := core.ExecuteConfig{ + WriteState: false, + EnableTracer: false, + MakeWitness: true, + } + result, err := bc.ProcessBlock(context.Background(), parent.Root, block, config) if err != nil { return nil, err } diff --git a/eth/backend.go b/eth/backend.go index eaa68b501c..72228614f0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -237,8 +237,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, - EnableWitnessStats: config.EnableWitnessStats, - StatelessSelfValidation: config.StatelessSelfValidation, }, // Enables file journaling for the trie database. The journal files will be stored // within the data directory. The corresponding paths will be either: @@ -247,6 +245,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieJournalDirectory: stack.ResolvePath("triedb"), StateSizeTracking: config.EnableStateSizeTracking, SlowBlockThreshold: config.SlowBlockThreshold, + + StatelessSelfValidation: config.StatelessSelfValidation, + EnableWitnessStats: config.EnableWitnessStats, } ) if config.VMTrace != "" { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 9ba8776360..1d1b5fbcd1 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -427,11 +427,6 @@ web3._extend({ params: 2, inputFormatter:[null, null], }), - new web3._extend.Method({ - name: 'freezeClient', - call: 'debug_freezeClient', - params: 1, - }), new web3._extend.Method({ name: 'getAccessibleState', call: 'debug_getAccessibleState', @@ -474,6 +469,12 @@ web3._extend({ params: 1, inputFormatter: [null], }), + new web3._extend.Method({ + name: 'executionWitness', + call: 'debug_executionWitness', + params: 1, + inputFormatter: [null], + }), ], properties: [] }); diff --git a/tests/block_test_util.go b/tests/block_test_util.go index dc680fea14..00411073e2 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -161,9 +161,9 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t Preimages: true, TxLookupLimit: -1, // disable tx indexing VmConfig: vm.Config{ - Tracer: tracer, - StatelessSelfValidation: witness, + Tracer: tracer, }, + StatelessSelfValidation: witness, } if snapshotter { options.SnapshotLimit = 1 From 814edc5308e08a0eee5c0b1c57c8bf57e008ab33 Mon Sep 17 00:00:00 2001 From: Jonny Rhea <5555162+jrhea@users.noreply.github.com> Date: Wed, 4 Mar 2026 03:34:27 -0600 Subject: [PATCH 03/52] core/vm: Switch to branchless normalization and extend EXCHANGE (#33869) For bal-devnet-3 we need to update the EIP-8024 implementation to the latest spec changes: https://github.com/ethereum/EIPs/pull/11306 > Note: I deleted tests not specified in the EIP bc maintaining them through EIP changes is too error prone. --- core/vm/instructions.go | 34 ++++++++----- core/vm/instructions_test.go | 97 ++++++++++-------------------------- 2 files changed, 48 insertions(+), 83 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index a4c4b0703b..4e4a33acda 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -946,24 +946,34 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro return nil, errStopToken } +// decodeSingle decodes the immediate operand of a backward-compatible DUPN or SWAPN instruction (EIP-8024) +// https://eips.ethereum.org/EIPS/eip-8024 func decodeSingle(x byte) int { - if x <= 90 { - return int(x) + 17 - } - return int(x) - 20 + // Depths 1-16 are already covered by the legacy opcodes. The forbidden byte range [91, 127] removes + // 37 values from the 256 possible immediates, leaving 219 usable values, so this encoding covers depths + // 17 through 235. The immediate is encoded as (x + 111) % 256, where 111 is chosen so that these values + // avoid the forbidden range. Decoding is simply the modular inverse (i.e. 111+145=256). + return (int(x) + 145) % 256 } +// decodePair decodes the immediate operand of a backward-compatible EXCHANGE +// instruction (EIP-8024) into stack indices (n, m) where 1 <= n < m +// and n + m <= 30. The forbidden byte range [82, 127] removes 46 values from +// the 256 possible immediates, leaving exactly 210 usable bytes. +// https://eips.ethereum.org/EIPS/eip-8024 func decodePair(x byte) (int, int) { - var k int - if x <= 79 { - k = int(x) - } else { - k = int(x) - 48 - } + // XOR with 143 remaps the forbidden bytes [82, 127] to an unused corner + // of the 16x16 grid below. + k := int(x ^ 143) + // Split into row q and column r of a 16x16 grid. The 210 valid pairs + // occupy two triangles within this grid. q, r := k/16, k%16 + // Upper triangle (q < r): pairs where m <= 16, encoded directly as + // (q+1, r+1). if q < r { return q + 1, r + 1 } + // Lower triangle: pairs where m > 16, recovered as (r+1, 29-q). return r + 1, 29 - q } @@ -1034,8 +1044,8 @@ func opExchange(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } // This range is excluded both to preserve compatibility with existing opcodes - // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–79, 128–255). - if x > 79 && x < 128 { + // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–81, 128–255). + if x > 81 && x < 128 { return nil, &ErrInvalidOpCode{opcode: OpCode(x)} } n, m := decodePair(x) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 3f776146f1..4c6d093d2e 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -1022,16 +1022,7 @@ func TestEIP8024_Execution(t *testing.T) { }{ { name: "DUPN", - codeHex: "60016000808080808080808080808080808080e600", - wantVals: []uint64{ - 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, - }, - }, - { - name: "DUPN_MISSING_IMMEDIATE", - codeHex: "60016000808080808080808080808080808080e6", + codeHex: "60016000808080808080808080808080808080e680", wantVals: []uint64{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1040,7 +1031,7 @@ func TestEIP8024_Execution(t *testing.T) { }, { name: "SWAPN", - codeHex: "600160008080808080808080808080808080806002e700", + codeHex: "600160008080808080808080808080808080806002e780", wantVals: []uint64{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1048,22 +1039,23 @@ func TestEIP8024_Execution(t *testing.T) { }, }, { - name: "SWAPN_MISSING_IMMEDIATE", - codeHex: "600160008080808080808080808080808080806002e7", + name: "EXCHANGE_MISSING_IMMEDIATE", + codeHex: "600260008080808080600160008080808080808080e8", wantVals: []uint64{ - 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, // 10th from top + 0, 0, 0, 0, 0, 0, + 1, // bottom }, }, { name: "EXCHANGE", - codeHex: "600060016002e801", + codeHex: "600060016002e88e", wantVals: []uint64{2, 0, 1}, }, { - name: "EXCHANGE_MISSING_IMMEDIATE", - codeHex: "600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060016002e8", + name: "EXCHANGE", + codeHex: "600080808080808080808080808080808080808080808080808080808060016002e88f", wantVals: []uint64{ 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1077,68 +1069,31 @@ func TestEIP8024_Execution(t *testing.T) { wantOpcode: SWAPN, }, { - name: "JUMP over INVALID_DUPN", + name: "JUMP_OVER_INVALID_DUPN", codeHex: "600456e65b", wantErr: nil, }, { - name: "UNDERFLOW_DUPN_1", - codeHex: "6000808080808080808080808080808080e600", + name: "EXCHANGE", + codeHex: "60008080e88e15", + wantVals: []uint64{1, 0, 0}, + }, + { + name: "INVALID_EXCHANGE", + codeHex: "e852", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "UNDERFLOW_DUPN", + codeHex: "6000808080808080808080808080808080e680", wantErr: &ErrStackUnderflow{}, wantOpcode: DUPN, }, // Additional test cases - { - name: "INVALID_DUPN_LOW", - codeHex: "e65b", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: DUPN, - }, - { - name: "INVALID_EXCHANGE_LOW", - codeHex: "e850", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: EXCHANGE, - }, - { - name: "INVALID_DUPN_HIGH", - codeHex: "e67f", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: DUPN, - }, - { - name: "INVALID_SWAPN_HIGH", - codeHex: "e77f", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: SWAPN, - }, - { - name: "INVALID_EXCHANGE_HIGH", - codeHex: "e87f", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: EXCHANGE, - }, - { - name: "UNDERFLOW_DUPN_2", - codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe600", // (n=17, need 17 items, have 16) - wantErr: &ErrStackUnderflow{}, - wantOpcode: DUPN, - }, - { - name: "UNDERFLOW_SWAPN", - codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe700", // (n=17, need 18 items, have 17) - wantErr: &ErrStackUnderflow{}, - wantOpcode: SWAPN, - }, - { - name: "UNDERFLOW_EXCHANGE", - codeHex: "60016002e801", // (n,m)=(1,2), need 3 items, have 2 - wantErr: &ErrStackUnderflow{}, - wantOpcode: EXCHANGE, - }, { name: "PC_INCREMENT", - codeHex: "600060006000e80115", + codeHex: "600060006000e88e15", wantVals: []uint64{1, 0, 0}, }, } From dd202d4283750d1672272aa3d55482e32f057289 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 4 Mar 2026 18:17:47 +0800 Subject: [PATCH 04/52] core, ethdb, triedb: add batch close (#33708) Pebble maintains a batch pool to recycle the batch object. Unfortunately batch object must be explicitly returned via `batch.Close` function. This PR extends the batch interface by adding the close function and also invoke batch.Close in some critical code paths. Memory allocation must be measured before merging this change. What's more, it's an open question that whether we should apply batch.Close as much as possible in every invocation. --- core/blockchain.go | 6 ++++++ core/rawdb/table.go | 5 +++++ core/state/statedb.go | 1 + ethdb/batch.go | 3 +++ ethdb/leveldb/leveldb.go | 3 +++ ethdb/memorydb/memorydb.go | 3 +++ ethdb/pebble/pebble.go | 6 ++++++ trie/trie_test.go | 1 + triedb/pathdb/buffer.go | 2 ++ 9 files changed, 30 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index ed186ccf5e..858d24bad7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1283,6 +1283,8 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { func (bc *BlockChain) writeHeadBlock(block *types.Block) { // Add the block to the canonical chain number scheme and mark as the head batch := bc.db.NewBatch() + defer batch.Close() + rawdb.WriteHeadHeaderHash(batch, block.Hash()) rawdb.WriteHeadFastBlockHash(batch, block.Hash()) rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) @@ -1657,6 +1659,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. batch = bc.db.NewBatch() start = time.Now() ) + defer batch.Close() + rawdb.WriteBlock(batch, block) rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) rawdb.WritePreimages(batch, statedb.Preimages()) @@ -2666,6 +2670,8 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. batch := bc.db.NewBatch() + defer batch.Close() + for _, tx := range types.HashDifference(deletedTxs, rebirthTxs) { rawdb.DeleteTxLookupEntry(batch, tx) } diff --git a/core/rawdb/table.go b/core/rawdb/table.go index d38afdaa35..407a619c9f 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -253,6 +253,11 @@ func (b *tableBatch) Reset() { b.batch.Reset() } +// Close closes the batch and releases all associated resources. +func (b *tableBatch) Close() { + b.batch.Close() +} + // tableReplayer is a wrapper around a batch replayer which truncates // the added prefix. type tableReplayer struct { diff --git a/core/state/statedb.go b/core/state/statedb.go index 3a2d9c9ac2..bf38bdf09d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1342,6 +1342,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag if err := batch.Write(); err != nil { return nil, err } + batch.Close() } if !ret.empty() { // If snapshotting is enabled, update the snapshot tree with this new version diff --git a/ethdb/batch.go b/ethdb/batch.go index 45b3781cb0..b93636c865 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -37,6 +37,9 @@ type Batch interface { // Replay replays the batch contents. Replay(w KeyValueWriter) error + + // Close closes the batch and releases all associated resources. + Close() } // Batcher wraps the NewBatch method of a backing data store. diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index b6c93907b1..c235d5f445 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -518,6 +518,9 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { return b.b.Replay(&replayer{writer: w}) } +// Close closes the batch and releases all associated resources. +func (b *batch) Close() {} + // replayer is a small wrapper to implement the correct replay methods. type replayer struct { writer ethdb.KeyValueWriter diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 200ad60245..29ed0aaea1 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -338,6 +338,9 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { return nil } +// Close closes the batch and releases all associated resources. +func (b *batch) Close() {} + // iterator can walk over the (potentially partial) keyspace of a memory key // value store. Internally it is a deep copy of the entire iterated state, // sorted by keys. diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 6b549f40d9..7654d582c4 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -731,6 +731,12 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } } +// Close closes the batch and releases all associated resources. After it is +// closed, any subsequent operations on this batch are undefined. +func (b *batch) Close() { + b.b.Close() +} + // pebbleIterator is a wrapper of underlying iterator in storage engine. // The purpose of this structure is to implement the missing APIs. // diff --git a/trie/trie_test.go b/trie/trie_test.go index 3423cde59c..3661933e22 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -880,6 +880,7 @@ func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil } +func (b *spongeBatch) Close() {} // TestCommitSequence tests that the trie.Commit operation writes the elements // of the trie in the expected order. diff --git a/triedb/pathdb/buffer.go b/triedb/pathdb/buffer.go index 853e1090b3..5d3099285f 100644 --- a/triedb/pathdb/buffer.go +++ b/triedb/pathdb/buffer.go @@ -180,6 +180,8 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezers []ethd b.flushErr = err return } + batch.Close() + commitBytesMeter.Mark(int64(size)) commitNodesMeter.Mark(int64(nodes)) commitAccountsMeter.Mark(int64(accounts)) From 6d0dd0886000a2011bc79872b74ccfe9c672c40d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 4 Mar 2026 11:18:18 +0100 Subject: [PATCH 05/52] core: implement eip-7778: block gas accounting without refunds (#33593) Implements https://eips.ethereum.org/EIPS/eip-7778 --------- Co-authored-by: Gary Rong --- cmd/evm/internal/t8ntool/execution.go | 12 +- core/chain_makers.go | 6 +- core/error.go | 4 + core/gaspool.go | 80 +++++-- core/state_prefetcher.go | 2 +- core/state_processor.go | 38 ++-- core/state_transition.go | 26 ++- core/tracing/hooks.go | 2 +- eth/catalyst/simulated_beacon.go | 12 +- eth/gasestimator/gasestimator.go | 3 +- eth/state_accessor.go | 2 +- eth/tracers/api.go | 12 +- eth/tracers/api_test.go | 2 +- .../internal/tracetest/calltrace_test.go | 6 +- .../internal/tracetest/erc7562_tracer_test.go | 2 +- .../internal/tracetest/flat_calltrace_test.go | 2 +- .../internal/tracetest/prestate_test.go | 2 +- .../tracetest/selfdestruct_state_test.go | 3 +- eth/tracers/tracers_test.go | 2 +- internal/ethapi/api.go | 20 +- internal/ethapi/api_test.go | 4 +- internal/ethapi/simulate.go | 76 +++++-- internal/ethapi/simulate_test.go | 6 +- miner/stress/main.go | 210 ++++++++++++++++++ miner/worker.go | 22 +- tests/state_test_util.go | 4 +- 26 files changed, 433 insertions(+), 127 deletions(-) create mode 100644 miner/stress/main.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 1979a8226e..b3fb79bc4a 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -149,15 +149,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, isEIP4762 = chainConfig.IsVerkle(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp) statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) signer = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) - gaspool = new(core.GasPool) + gaspool = core.NewGasPool(pre.Env.GasLimit) blockHash = common.Hash{0x13, 0x37} rejectedTxs []*rejectedTx includedTxs types.Transactions - gasUsed = uint64(0) blobGasUsed = uint64(0) receipts = make(types.Receipts, 0) ) - gaspool.AddGas(pre.Env.GasLimit) vmContext := vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -258,14 +256,14 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, statedb.SetTxContext(tx.Hash(), len(receipts)) var ( snapshot = statedb.Snapshot() - prevGas = gaspool.Gas() + gp = gaspool.Snapshot() ) - receipt, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, &gasUsed, evm) + receipt, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, evm) if err != nil { statedb.RevertToSnapshot(snapshot) log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) - gaspool.SetGas(prevGas) + gaspool.Set(gp) continue } if receipt.Logs == nil { @@ -352,7 +350,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, Receipts: receipts, Rejected: rejectedTxs, Difficulty: (*math.HexOrDecimal256)(vmContext.Difficulty), - GasUsed: (math.HexOrDecimal64)(gasUsed), + GasUsed: (math.HexOrDecimal64)(gaspool.Used()), BaseFee: (*math.HexOrDecimal256)(vmContext.BaseFee), } if pre.Env.Withdrawals != nil { diff --git a/core/chain_makers.go b/core/chain_makers.go index 7ce86b14e9..5264336aaa 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -63,7 +63,7 @@ func (b *BlockGen) SetCoinbase(addr common.Address) { panic("coinbase can only be set once") } b.header.Coinbase = addr - b.gasPool = new(GasPool).AddGas(b.header.GasLimit) + b.gasPool = NewGasPool(b.header.GasLimit) } // SetExtra sets the extra data field of the generated block. @@ -117,10 +117,12 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) + receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx) if err != nil { panic(err) } + b.header.GasUsed = b.gasPool.Used() + // Merge the tx-local access event into the "block-local" one, in order to collect // all values, so that the witness can be built. if b.statedb.Database().TrieDB().IsVerkle() { diff --git a/core/error.go b/core/error.go index 635d802863..4610842cee 100644 --- a/core/error.go +++ b/core/error.go @@ -58,6 +58,10 @@ var ( // by a transaction is higher than what's left in the block. ErrGasLimitReached = errors.New("gas limit reached") + // ErrGasLimitOverflow is returned by the gas pool if the remaining gas + // exceeds the maximum value of uint64. + ErrGasLimitOverflow = errors.New("gas limit overflow") + // ErrInsufficientFundsForTransfer is returned if the transaction sender doesn't // have enough funds for transfer(topmost call only). ErrInsufficientFundsForTransfer = errors.New("insufficient funds for transfer") diff --git a/core/gaspool.go b/core/gaspool.go index 767222674f..14f5abd93c 100644 --- a/core/gaspool.go +++ b/core/gaspool.go @@ -21,39 +21,87 @@ import ( "math" ) -// GasPool tracks the amount of gas available during execution of the transactions -// in a block. The zero value is a pool with zero gas available. -type GasPool uint64 +// GasPool tracks the amount of gas available for transaction execution +// within a block, along with the cumulative gas consumed. +type GasPool struct { + remaining uint64 + initial uint64 + cumulativeUsed uint64 +} -// AddGas makes gas available for execution. -func (gp *GasPool) AddGas(amount uint64) *GasPool { - if uint64(*gp) > math.MaxUint64-amount { - panic("gas pool pushed above uint64") +// NewGasPool initializes the gasPool with the given amount. +func NewGasPool(amount uint64) *GasPool { + return &GasPool{ + remaining: amount, + initial: amount, } - *(*uint64)(gp) += amount - return gp } // SubGas deducts the given amount from the pool if enough gas is // available and returns an error otherwise. func (gp *GasPool) SubGas(amount uint64) error { - if uint64(*gp) < amount { + if gp.remaining < amount { return ErrGasLimitReached } - *(*uint64)(gp) -= amount + gp.remaining -= amount + return nil +} + +// ReturnGas adds the refunded gas back to the pool and updates +// the cumulative gas usage accordingly. +func (gp *GasPool) ReturnGas(returned uint64, gasUsed uint64) error { + if gp.remaining > math.MaxUint64-returned { + return fmt.Errorf("%w: remaining: %d, returned: %d", ErrGasLimitOverflow, gp.remaining, returned) + } + // The returned gas calculation differs across forks. + // + // - Pre-Amsterdam: + // returned = purchased - remaining (refund included) + // + // - Post-Amsterdam: + // returned = purchased - gasUsed (refund excluded) + gp.remaining += returned + + // gasUsed = max(txGasUsed - gasRefund, calldataFloorGasCost) + // regardless of Amsterdam is activated or not. + gp.cumulativeUsed += gasUsed return nil } // Gas returns the amount of gas remaining in the pool. func (gp *GasPool) Gas() uint64 { - return uint64(*gp) + return gp.remaining } -// SetGas sets the amount of gas with the provided number. -func (gp *GasPool) SetGas(gas uint64) { - *(*uint64)(gp) = gas +// CumulativeUsed returns the amount of cumulative consumed gas (refunded included). +func (gp *GasPool) CumulativeUsed() uint64 { + return gp.cumulativeUsed +} + +// Used returns the amount of consumed gas. +func (gp *GasPool) Used() uint64 { + if gp.initial < gp.remaining { + panic("gas used underflow") + } + return gp.initial - gp.remaining +} + +// Snapshot returns the deep-copied object as the snapshot. +func (gp *GasPool) Snapshot() *GasPool { + return &GasPool{ + initial: gp.initial, + remaining: gp.remaining, + cumulativeUsed: gp.cumulativeUsed, + } +} + +// Set sets the content of gasPool with the provided one. +func (gp *GasPool) Set(other *GasPool) { + gp.initial = other.initial + gp.remaining = other.remaining + gp.cumulativeUsed = other.cumulativeUsed } func (gp *GasPool) String() string { - return fmt.Sprintf("%d", *gp) + return fmt.Sprintf("initial: %d, remaining: %d, cumulative used: %d", gp.initial, gp.remaining, gp.cumulativeUsed) } diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 1c738c1e38..c91d40d94f 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -107,7 +107,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c // We attempt to apply a transaction. The goal is not to execute // the transaction successfully, rather to warm up touched data slots. - if _, err := ApplyMessage(evm, msg, new(GasPool).AddGas(block.GasLimit())); err != nil { + if _, err := ApplyMessage(evm, msg, nil); err != nil { fails.Add(1) return nil // Ugh, something went horribly wrong, bail out } diff --git a/core/state_processor.go b/core/state_processor.go index 6eea74bdd8..998f180571 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -63,14 +63,12 @@ func (p *StateProcessor) Process(ctx context.Context, block *types.Block, stated var ( config = p.chainConfig() receipts types.Receipts - usedGas = new(uint64) header = block.Header() blockHash = block.Hash() blockNumber = block.Number() allLogs []*types.Log - gp = new(GasPool).AddGas(block.GasLimit()) + gp = NewGasPool(block.GasLimit()) ) - var tracingStateDB = vm.StateDB(statedb) if hooks := cfg.Tracer; hooks != nil { tracingStateDB = state.NewHookedState(statedb, hooks) @@ -107,13 +105,15 @@ func (p *StateProcessor) Process(ctx context.Context, block *types.Block, stated telemetry.StringAttribute("tx.hash", tx.Hash().Hex()), telemetry.Int64Attribute("tx.index", int64(i)), ) - receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, usedGas, evm) - spanEnd(&err) + + receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, evm) if err != nil { return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) + + spanEnd(&err) } requests, err := postExecution(ctx, config, block, allLogs, evm) if err != nil { @@ -127,7 +127,7 @@ func (p *StateProcessor) Process(ctx context.Context, block *types.Block, stated Receipts: receipts, Requests: requests, Logs: allLogs, - GasUsed: *usedGas, + GasUsed: gp.Used(), }, nil } @@ -159,7 +159,7 @@ func postExecution(ctx context.Context, config *params.ChainConfig, block *types // ApplyTransactionWithEVM attempts to apply a transaction to the given state database // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. -func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { +func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, evm *vm.EVM) (receipt *types.Receipt, err error) { if hooks := evm.Config.Tracer; hooks != nil { if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) @@ -180,27 +180,31 @@ func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, } else { root = statedb.IntermediateRoot(evm.ChainConfig().IsEIP158(blockNumber)).Bytes() } - *usedGas += result.UsedGas - // Merge the tx-local access event into the "block-local" one, in order to collect // all values, so that the witness can be built. if statedb.Database().TrieDB().IsVerkle() { statedb.AccessEvents().Merge(evm.AccessEvents) } - return MakeReceipt(evm, result, statedb, blockNumber, blockHash, blockTime, tx, *usedGas, root), nil + return MakeReceipt(evm, result, statedb, blockNumber, blockHash, blockTime, tx, gp.CumulativeUsed(), root), nil } // MakeReceipt generates the receipt object for a transaction given its execution result. -func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, usedGas uint64, root []byte) *types.Receipt { - // Create a new receipt for the transaction, storing the intermediate root and gas used - // by the tx. - receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: usedGas} +func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, cumulativeGas uint64, root []byte) *types.Receipt { + // Create a new receipt for the transaction, storing the intermediate root + // and gas used by the tx. + // + // The cumulative gas used equals the sum of gasUsed across all preceding + // txs with refunded gas deducted. + receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: cumulativeGas} if result.Failed() { receipt.Status = types.ReceiptStatusFailed } else { receipt.Status = types.ReceiptStatusSuccessful } receipt.TxHash = tx.Hash() + + // GasUsed = max(tx_gas_used - gas_refund, calldata_floor_gas_cost), unchanged + // in the Amsterdam fork. receipt.GasUsed = result.UsedGas if tx.Type() == types.BlobTxType { @@ -224,15 +228,15 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // ApplyTransaction attempts to apply a transaction to the given state database // and uses the input parameters for its environment. It returns the receipt -// for the transaction, gas used and an error if the transaction failed, +// for the transaction and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { +func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction) (*types.Receipt, error) { msg, err := TransactionToMessage(tx, types.MakeSigner(evm.ChainConfig(), header.Number, header.Time), header.BaseFee) if err != nil { return nil, err } // Create a new context to be used in the EVM environment - return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), header.Time, tx, usedGas, evm) + return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), header.Time, tx, evm) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root diff --git a/core/state_transition.go b/core/state_transition.go index 62474b5f5b..76a5147363 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -34,7 +34,7 @@ import ( // ExecutionResult includes all output after executing given evm // message no matter the execution itself is successful or not. type ExecutionResult struct { - UsedGas uint64 // Total used gas, not including the refunded gas + UsedGas uint64 // Total used gas, refunded gas is deducted MaxUsedGas uint64 // Maximum gas consumed during execution, excluding gas refunds. Err error // Any error encountered during the execution(listed in core/vm/errors.go) ReturnData []byte // Returned data from evm(function result or data supplied with revert opcode) @@ -210,6 +210,11 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In // indicates a core error meaning that the message would always fail for that particular // state and would never be accepted within a block. func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, error) { + // Do not panic if the gas pool is nil. This is allowed when executing + // a single message via RPC invocation. + if gp == nil { + gp = NewGasPool(msg.GasLimit) + } evm.SetTxContext(NewEVMTxContext(msg)) return newStateTransition(evm, msg, gp).execute() } @@ -300,8 +305,8 @@ func (st *stateTransition) buyGas() error { st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, tracing.GasChangeTxInitialBalance) } st.gasRemaining = st.msg.GasLimit - st.initialGas = st.msg.GasLimit + mgvalU256, _ := uint256.FromBig(mgval) st.state.SubBalance(st.msg.From, mgvalU256, tracing.BalanceDecreaseGasBuy) return nil @@ -542,8 +547,20 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { peakGasUsed = floorDataGas } } + // Return gas to the user st.returnGas() + // Return gas to the gas pool + if rules.IsAmsterdam { + // Refund is excluded for returning + err = st.gp.ReturnGas(st.initialGas-peakGasUsed, st.gasUsed()) + } else { + // Refund is included for returning + err = st.gp.ReturnGas(st.gasRemaining, st.gasUsed()) + } + if err != nil { + return nil, err + } effectiveTip := msg.GasPrice if rules.IsLondon { effectiveTip = new(big.Int).Sub(msg.GasPrice, st.evm.Context.BaseFee) @@ -564,7 +581,6 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { st.evm.AccessEvents.AddAccount(st.evm.Context.Coinbase, true, math.MaxUint64) } } - return &ExecutionResult{ UsedGas: st.gasUsed(), MaxUsedGas: peakGasUsed, @@ -660,10 +676,6 @@ func (st *stateTransition) returnGas() { if st.evm.Config.Tracer != nil && st.evm.Config.Tracer.OnGasChange != nil && st.gasRemaining > 0 { st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, tracing.GasChangeTxLeftOverReturned) } - - // Also return remaining gas to the block gas counter so it is - // available for the next transaction. - st.gp.AddGas(st.gasRemaining) } // gasUsed returns the amount of gas used up by the state transition. diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index c85abe6482..6d0131ce70 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -358,7 +358,7 @@ const ( // this generates an increase in gas. There is at most one of such gas change per transaction. GasChangeTxRefunds GasChangeReason = 3 // GasChangeTxLeftOverReturned is the amount of gas left over at the end of transaction's execution that will be returned - // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + // to the account. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. // There is at most one of such gas change per transaction. GasChangeTxLeftOverReturned GasChangeReason = 4 diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index ed3fa76a57..452902c78c 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -103,6 +103,8 @@ type SimulatedBeacon struct { func payloadVersion(config *params.ChainConfig, time uint64) engine.PayloadVersion { switch config.LatestFork(time) { + case forks.Amsterdam: + return engine.PayloadV4 case forks.BPO5, forks.BPO4, forks.BPO3, forks.BPO2, forks.BPO1, forks.Osaka, forks.Prague, forks.Cancun: return engine.PayloadV3 case forks.Paris, forks.Shanghai: @@ -198,13 +200,19 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u var random [32]byte rand.Read(random[:]) - fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, &engine.PayloadAttributes{ + + attribute := &engine.PayloadAttributes{ Timestamp: timestamp, SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, BeaconRoot: &common.Hash{}, - }, version, false) + } + if c.eth.BlockChain().Config().LatestFork(timestamp) == forks.Amsterdam { + slotNumber := uint64(0) + attribute.SlotNumber = &slotNumber + } + fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, attribute, version, false) if err != nil { return err } diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index 6e79fbd62b..80aeb3d3b2 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "math" "math/big" "github.com/ethereum/go-ethereum/common" @@ -268,7 +267,7 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio evm.Cancel() }() // Execute the call, returning a wrapped error or the result - result, err := core.ApplyMessage(evm, call, new(core.GasPool).AddGas(math.MaxUint64)) + result, err := core.ApplyMessage(evm, call, nil) if vmerr := dirtyState.Error(); vmerr != nil { return nil, vmerr } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 1261320b58..871f2c9269 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -265,7 +265,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, // Not yet the searched for transaction, execute on top of the current state statedb.SetTxContext(tx.Hash(), idx) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 5f2f16627a..eed404622e 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -551,7 +551,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) // We intentionally don't return the error here: if we do, then the RPC server will not // return the roots. Most likely, the caller already knows that a certain transaction fails to @@ -707,7 +707,7 @@ txloop: // Generate the next state snapshot fast without tracing msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { failed = err break txloop } @@ -792,7 +792,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) if txHash != (common.Hash{}) && tx.Hash() != txHash { // Process the tx to update state, but don't trace it. - _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + _, err := core.ApplyMessage(evm, msg, nil) if err != nil { return dumps, err } @@ -827,7 +827,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block if tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + _, err = core.ApplyMessage(evm, msg, nil) if writer != nil { writer.Flush() } @@ -1011,7 +1011,6 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor tracer *Tracer err error timeout = defaultTraceTimeout - usedGas uint64 ) if config == nil { config = &TraceConfig{} @@ -1055,7 +1054,8 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, err = core.ApplyTransactionWithEVM(message, new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, vmctx.Time, tx, &usedGas, evm) + + _, err = core.ApplyTransactionWithEVM(message, core.NewGasPool(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, vmctx.Time, tx, evm) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index f76c35a1d5..1d5024ad08 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -188,7 +188,7 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block return tx, context, statedb, release, nil } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 08bdafd91f..85eaef32ce 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -132,7 +132,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } @@ -224,7 +224,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + _, err = core.ApplyMessage(evm, msg, nil) if err != nil { b.Fatalf("failed to execute transaction: %v", err) } @@ -374,7 +374,7 @@ func TestInternals(t *testing.T) { t.Fatalf("test %v: failed to create message: %v", tc.name, err) } tc.tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("test %v: failed to execute transaction: %v", tc.name, err) } diff --git a/eth/tracers/internal/tracetest/erc7562_tracer_test.go b/eth/tracers/internal/tracetest/erc7562_tracer_test.go index f6e81f5886..02377b8dcb 100644 --- a/eth/tracers/internal/tracetest/erc7562_tracer_test.go +++ b/eth/tracers/internal/tracetest/erc7562_tracer_test.go @@ -124,7 +124,7 @@ func TestErc7562Tracer(t *testing.T) { } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 1882ef315e..37a05966ee 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -113,7 +113,7 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { return fmt.Errorf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 456d962c69..23216fa78c 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -105,7 +105,7 @@ func testPrestateTracer(tracerName string, dirPath string, t *testing.T) { } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/selfdestruct_state_test.go b/eth/tracers/internal/tracetest/selfdestruct_state_test.go index 2c714b6dce..bb1a3d9f18 100644 --- a/eth/tracers/internal/tracetest/selfdestruct_state_test.go +++ b/eth/tracers/internal/tracetest/selfdestruct_state_test.go @@ -620,8 +620,7 @@ func TestSelfdestructStateTracer(t *testing.T) { } context := core.NewEVMBlockContext(block.Header(), blockchain, nil) evm := vm.NewEVM(context, hookedState, tt.genesis.Config, vm.Config{Tracer: tracer.Hooks()}) - usedGas := uint64(0) - _, err = core.ApplyTransactionWithEVM(msg, new(core.GasPool).AddGas(tx.Gas()), statedb, block.Number(), block.Hash(), block.Time(), tx, &usedGas, evm) + _, err = core.ApplyTransactionWithEVM(msg, core.NewGasPool(msg.GasLimit), statedb, block.Number(), block.Hash(), block.Time(), tx, evm) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 06edeaf698..24f9b3701e 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -91,7 +91,7 @@ func BenchmarkTransactionTraceV2(b *testing.B) { evm.Config.Tracer = tracer snap := state.StateDB.Snapshot() - _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + _, err := core.ApplyMessage(evm, msg, nil) if err != nil { b.Fatal(err) } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 4f3071cb03..ff6797f67b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -741,11 +741,10 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S // Make sure the context is cancelled when the call has completed // this makes sure resources are cleaned up. defer cancel() - gp := new(core.GasPool) + + gp := core.NewGasPool(globalGasCap) if globalGasCap == 0 { - gp.AddGas(gomath.MaxUint64) - } else { - gp.AddGas(globalGasCap) + gp = core.NewGasPool(gomath.MaxUint64) } return applyMessage(ctx, b, args, state, header, timeout, gp, &blockCtx, &vm.Config{NoBaseFee: true}, precompiles) } @@ -855,12 +854,11 @@ func (api *BlockChainAPI) SimulateV1(ctx context.Context, opts simOpts, blockNrO gasCap = gomath.MaxUint64 } sim := &simulator{ - b: api.b, - state: state, - base: base, - chainConfig: api.b.ChainConfig(), - // Each tx and all the series of txes shouldn't consume more gas than cap - gp: new(core.GasPool).AddGas(gasCap), + b: api.b, + state: state, + base: base, + chainConfig: api.b.ChainConfig(), + gasRemaining: gasCap, traceTransfers: opts.TraceTransfers, validate: opts.Validation, fullTx: opts.ReturnFullTransactions, @@ -1369,7 +1367,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { evm.Context.BlobBaseFee = new(big.Int) } - res, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + res, err := core.ApplyMessage(evm, msg, nil) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.ToTransaction(types.LegacyTxType).Hash(), err) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 2f0c07694d..a82df440e6 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -2507,7 +2507,7 @@ func TestSimulateV1ChainLinkage(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gp: new(core.GasPool).AddGas(math.MaxUint64), + gasRemaining: math.MaxUint64, traceTransfers: false, validate: false, fullTx: false, @@ -2592,7 +2592,7 @@ func TestSimulateV1TxSender(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gp: new(core.GasPool).AddGas(math.MaxUint64), + gasRemaining: math.MaxUint64, traceTransfers: false, validate: false, fullTx: true, diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index c9396cd327..325ee6d5bb 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -156,7 +156,7 @@ type simulator struct { state *state.StateDB base *types.Header chainConfig *params.ChainConfig - gp *core.GasPool + gasRemaining uint64 traceTransfers bool validate bool fullTx bool @@ -200,7 +200,13 @@ func (sim *simulator) execute(ctx context.Context, blocks []simBlock) ([]*simBlo return nil, err } headers[bi] = result.Header() - results[bi] = &simBlockResult{fullTx: sim.fullTx, chainConfig: sim.chainConfig, Block: result, Calls: callResults, senders: senders} + results[bi] = &simBlockResult{ + fullTx: sim.fullTx, + chainConfig: sim.chainConfig, + Block: result, + Calls: callResults, + senders: senders, + } parent = result.Header() } return results, nil @@ -234,15 +240,19 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, blockContext.BlobBaseFee = block.BlockOverrides.BlobBaseFee.ToInt() } precompiles := sim.activePrecompiles(header) + // State overrides are applied prior to execution of a block if err := block.StateOverrides.Apply(sim.state, precompiles); err != nil { return nil, nil, nil, err } var ( - gasUsed, blobGasUsed uint64 - txes = make([]*types.Transaction, len(block.Calls)) - callResults = make([]simCallResult, len(block.Calls)) - receipts = make([]*types.Receipt, len(block.Calls)) + gp = core.NewGasPool(blockContext.GasLimit) + blobGasUsed uint64 + + txes = make([]*types.Transaction, len(block.Calls)) + callResults = make([]simCallResult, len(block.Calls)) + receipts = make([]*types.Receipt, len(block.Calls)) + // Block hash will be repaired after execution. tracer = newTracer(sim.traceTransfers, blockContext.BlockNumber.Uint64(), blockContext.Time, common.Hash{}, common.Hash{}, 0) vmConfig = &vm.Config{ @@ -272,10 +282,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } var allLogs []*types.Log for i, call := range block.Calls { + // Terminate if the context is cancelled if err := ctx.Err(); err != nil { return nil, nil, nil, err } - if err := sim.sanitizeCall(&call, sim.state, header, blockContext, &gasUsed); err != nil { + if err := sim.sanitizeCall(&call, sim.state, header, gp); err != nil { return nil, nil, nil, err } var ( @@ -285,10 +296,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, txes[i] = tx senders[txHash] = call.from() tracer.reset(txHash, uint(i)) - sim.state.SetTxContext(txHash, i) + // EoA check is always skipped, even in validation mode. + sim.state.SetTxContext(txHash, i) msg := call.ToMessage(header.BaseFee, !sim.validate) - result, err := applyMessageWithEVM(ctx, evm, msg, timeout, sim.gp) + result, err := applyMessageWithEVM(ctx, evm, msg, timeout, gp) if err != nil { txErr := txValidationError(err) return nil, nil, nil, txErr @@ -300,9 +312,16 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } else { root = sim.state.IntermediateRoot(sim.chainConfig.IsEIP158(blockContext.BlockNumber)).Bytes() } - gasUsed += result.UsedGas - receipts[i] = core.MakeReceipt(evm, result, sim.state, blockContext.BlockNumber, common.Hash{}, blockContext.Time, tx, gasUsed, root) + receipts[i] = core.MakeReceipt(evm, result, sim.state, blockContext.BlockNumber, common.Hash{}, blockContext.Time, tx, gp.CumulativeUsed(), root) blobGasUsed += receipts[i].BlobGasUsed + + // Make sure the gas cap is still enforced. It's only for + // internally protection. + if sim.gasRemaining < result.UsedGas { + return nil, nil, nil, fmt.Errorf("gas cap reached, required: %d, remaining: %d", result.UsedGas, sim.gasRemaining) + } + sim.gasRemaining -= result.UsedGas + logs := tracer.Logs() callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas)} if result.Failed() { @@ -320,12 +339,14 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } callResults[i] = callRes } - header.GasUsed = gasUsed + // Assign total consumed gas to the header + header.GasUsed = gp.Used() if sim.chainConfig.IsCancun(header.Number, header.Time) { header.BlobGasUsed = &blobGasUsed } - var requests [][]byte + // Process EIP-7685 requests + var requests [][]byte if sim.chainConfig.IsPrague(header.Number, header.Time) { requests = [][]byte{} // EIP-6110 @@ -345,7 +366,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, reqHash := types.CalcRequestsHash(requests) header.RequestsHash = &reqHash } - blockBody := &types.Body{Transactions: txes, Withdrawals: *block.BlockOverrides.Withdrawals} + + blockBody := &types.Body{ + Transactions: txes, + Withdrawals: *block.BlockOverrides.Withdrawals, + } chainHeadReader := &simChainHeadReader{ctx, sim.b} b, err := sim.b.Engine().FinalizeAndAssemble(chainHeadReader, header, sim.state, blockBody, receipts) if err != nil { @@ -366,23 +391,20 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, gp *core.GasPool) error { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) } // Let the call run wild unless explicitly specified. + remaining := gp.Gas() if call.Gas == nil { - remaining := blockContext.GasLimit - *gasUsed call.Gas = (*hexutil.Uint64)(&remaining) } - if *gasUsed+uint64(*call.Gas) > blockContext.GasLimit { - return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: %d >= %d", *gasUsed, blockContext.GasLimit)} + if remaining < uint64(*call.Gas) { + return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: remaining: %d, required: %d", remaining, *call.Gas)} } - if err := call.CallDefaults(sim.gp.Gas(), header.BaseFee, sim.chainConfig.ChainID); err != nil { - return err - } - return nil + return call.CallDefaults(0, header.BaseFee, sim.chainConfig.ChainID) } func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContracts { @@ -473,12 +495,14 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { } overrides := block.BlockOverrides - var withdrawalsHash *common.Hash number := overrides.Number.ToInt() timestamp := (uint64)(*overrides.Time) + + var withdrawalsHash *common.Hash if sim.chainConfig.IsShanghai(number, timestamp) { withdrawalsHash = &types.EmptyWithdrawalsHash } + var parentBeaconRoot *common.Hash if sim.chainConfig.IsCancun(number, timestamp) { parentBeaconRoot = &common.Hash{} @@ -508,7 +532,11 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { } func (sim *simulator) newSimulatedChainContext(ctx context.Context, headers []*types.Header) *ChainContext { - return NewChainContext(ctx, &simBackend{base: sim.base, b: sim.b, headers: headers}) + return NewChainContext(ctx, &simBackend{ + base: sim.base, + b: sim.b, + headers: headers, + }) } type simBackend struct { diff --git a/internal/ethapi/simulate_test.go b/internal/ethapi/simulate_test.go index c747b76477..b6037a8f35 100644 --- a/internal/ethapi/simulate_test.go +++ b/internal/ethapi/simulate_test.go @@ -17,6 +17,7 @@ package ethapi import ( + "math" "math/big" "testing" @@ -80,7 +81,10 @@ func TestSimulateSanitizeBlockOrder(t *testing.T) { err: "block timestamps must be in order: 72 <= 72", }, } { - sim := &simulator{base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}} + sim := &simulator{ + base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}, + gasRemaining: math.MaxUint64, + } res, err := sim.sanitizeChain(tc.blocks) if err != nil { if err.Error() == tc.err { diff --git a/miner/stress/main.go b/miner/stress/main.go new file mode 100644 index 0000000000..aaf0993c37 --- /dev/null +++ b/miner/stress/main.go @@ -0,0 +1,210 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains a miner stress test based on the Engine API flow. +package main + +import ( + "crypto/ecdsa" + "math/big" + "math/rand" + "os" + "os/signal" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/catalyst" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" +) + +var refundContract = common.HexToAddress("0x1000000000000000000000000000000000000001") + +func main() { + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + // Create a post-merge network where blocks are built/inserted through + // engine API calls driven by a simulated beacon client. + genesis := makeGenesis(faucets) + + // Handle interrupts. + interruptCh := make(chan os.Signal, 5) + signal.Notify(interruptCh, os.Interrupt) + + // Start one node that accepts transactions and builds/inserts blocks via + // Engine API (through the simulated beacon driver). + stack, backend, beacon, err := makeNode(genesis) + if err != nil { + panic(err) + } + defer stack.Close() + defer beacon.Stop() + + // Start injecting transactions from the faucet like crazy + var ( + sent uint64 + nonces = make([]uint64, len(faucets)) + signer = types.LatestSigner(genesis.Config) + refundSet = true // slot 0 starts as non-zero in genesis + ) + for { + // Stop when interrupted. + select { + case <-interruptCh: + return + default: + } + + var ( + tx *types.Transaction + err error + ) + // Every third tx targets a contract path that alternates set/clear. + // Clearing a previously non-zero slot triggers gas refund. + if sent%3 == 0 { + var data []byte + if refundSet { + data = nil // empty calldata => clear slot to zero (refund path) + } else { + data = []byte{0x01} // non-empty calldata => set slot to one + } + tx, err = types.SignTx(types.NewTransaction(nonces[0], refundContract, new(big.Int), 50000, big.NewInt(100000000000), data), signer, faucets[0]) + if err != nil { + panic(err) + } + nonces[0]++ + refundSet = !refundSet + } else { + index := 1 + rand.Intn(len(faucets)-1) + tx, err = types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), signer, faucets[index]) + if err != nil { + panic(err) + } + nonces[index]++ + } + errs := backend.TxPool().Add([]*types.Transaction{tx}, true) + for _, err := range errs { + if err != nil { + panic(err) + } + } + sent++ + + // Create and import blocks through the engine API path. + if sent%256 == 0 { + beacon.Commit() + } + + // Wait if we're too saturated + if pend, _ := backend.TxPool().Stats(); pend > 4096 { + beacon.Commit() + time.Sleep(50 * time.Millisecond) + } + } +} + +// makeGenesis creates a post-merge genesis block. +func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { + config := *params.AllDevChainProtocolChanges + config.ChainID = big.NewInt(18) + + blockZero := uint64(0) + config.AmsterdamTime = &blockZero + config.BlobScheduleConfig.Amsterdam = ¶ms.BlobConfig{ + Target: 14, + Max: 21, + UpdateFraction: 13739630, + } + + genesis := &core.Genesis{ + Config: &config, + GasLimit: 25000000, + Alloc: types.GenesisAlloc{}, + } + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = types.Account{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + // Runtime code: + // - empty calldata: SSTORE(0,0) + // - non-empty calldata: SSTORE(0,1) + // Slot 0 is initialized to 1 so the first clear includes gas refund. + genesis.Alloc[refundContract] = types.Account{ + Code: common.FromHex("0x3615600b576001600055005b600060005500"), + Storage: map[common.Hash]common.Hash{ + common.Hash{}: common.BigToHash(big.NewInt(1)), + }, + } + return genesis +} + +func makeNode(genesis *core.Genesis) (*node.Node, *eth.Ethereum, *catalyst.SimulatedBeacon, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := os.MkdirTemp("", "") + + config := &node.Config{ + Name: "geth", + DataDir: datadir, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, nil, nil, err + } + // Create and register the backend + ethBackend, err := eth.New(stack, ðconfig.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: legacypool.DefaultConfig, + GPO: ethconfig.Defaults.GPO, + Miner: ethconfig.Defaults.Miner, + SlowBlockThreshold: time.Second, + }) + if err != nil { + return nil, nil, nil, err + } + + if err := stack.Start(); err != nil { + return nil, nil, nil, err + } + driver, err := catalyst.NewSimulatedBeacon(0, common.Address{}, ethBackend) + if err != nil { + return nil, nil, nil, err + } + if err := driver.Start(); err != nil { + return nil, nil, nil, err + } + return stack, ethBackend, driver, nil +} diff --git a/miner/worker.go b/miner/worker.go index e924ca9c86..bfeeaa248b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -146,9 +146,6 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay // If forceOverrides is true and overrideTxs is not empty, commit the override transactions // otherwise, fill the block with the current transactions from the txpool if genParam.forceOverrides && len(genParam.overrideTxs) > 0 { - if work.gasPool == nil { - work.gasPool = new(core.GasPool).AddGas(work.header.GasLimit) - } for _, tx := range genParam.overrideTxs { work.state.SetTxContext(tx.Hash(), work.tcount) if err := miner.commitTransaction(work, tx); err != nil { @@ -326,6 +323,7 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase state: state, size: uint64(header.Size()), coinbase: coinbase, + gasPool: core.NewGasPool(header.GasLimit), header: header, witness: state.Witness(), evm: vm.NewEVM(core.NewEVMBlockContext(header, miner.chain, &coinbase), state, miner.chainConfig, vm.Config{}), @@ -379,24 +377,20 @@ func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transactio func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { var ( snap = env.state.Snapshot() - gp = env.gasPool.Gas() + gp = env.gasPool.Snapshot() ) - receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx) if err != nil { env.state.RevertToSnapshot(snap) - env.gasPool.SetGas(gp) + env.gasPool.Set(gp) + return nil, err } - return receipt, err + env.header.GasUsed = env.gasPool.Used() + return receipt, nil } func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { - var ( - isCancun = miner.chainConfig.IsCancun(env.header.Number, env.header.Time) - gasLimit = env.header.GasLimit - ) - if env.gasPool == nil { - env.gasPool = new(core.GasPool).AddGas(gasLimit) - } + isCancun := miner.chainConfig.IsCancun(env.header.Number, env.header.Time) for { // Check interruption signal and abort building if it's fired. if interrupt != nil { diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 7525081f84..3c7ee1c31d 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -342,9 +342,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh } // Execute the message. snapshot := st.StateDB.Snapshot() - gaspool := new(core.GasPool) - gaspool.AddGas(block.GasLimit()) - vmRet, err := core.ApplyMessage(evm, msg, gaspool) + vmRet, err := core.ApplyMessage(evm, msg, core.NewGasPool(block.GasLimit())) if err != nil { st.StateDB.RevertToSnapshot(snapshot) if tracer := evm.Config.Tracer; tracer != nil && tracer.OnTxEnd != nil { From 28dad943f62d5182edcf40b6249f3161ca9281b5 Mon Sep 17 00:00:00 2001 From: Jonny Rhea <5555162+jrhea@users.noreply.github.com> Date: Wed, 4 Mar 2026 04:21:11 -0600 Subject: [PATCH 06/52] cmd/geth: set default cache to 4096 (#33836) Mainnet was already overriding --cache to 4096. This PR just makes this the default. --- cmd/geth/main.go | 13 ------------- cmd/utils/flags.go | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ca75775be2..b72cbb9885 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -22,7 +22,6 @@ import ( "os" "slices" "sort" - "strconv" "time" "github.com/ethereum/go-ethereum/accounts" @@ -316,18 +315,6 @@ func prepare(ctx *cli.Context) { case !ctx.IsSet(utils.NetworkIdFlag.Name): log.Info("Starting Geth on Ethereum mainnet...") } - // If we're a full node on mainnet without --cache specified, bump default cache allowance - if !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) { - // Make sure we're not on any supported preconfigured testnet either - if !ctx.IsSet(utils.HoleskyFlag.Name) && - !ctx.IsSet(utils.SepoliaFlag.Name) && - !ctx.IsSet(utils.HoodiFlag.Name) && - !ctx.IsSet(utils.DeveloperFlag.Name) { - // Nope, we're really on mainnet. Bump that cache up! - log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) - ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) - } - } } // geth is the main entry point into the system if no special subcommand is run. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 75b5b4785a..b0d6ee5203 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -491,7 +491,7 @@ var ( CacheFlag = &cli.IntFlag{ Name: "cache", Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", - Value: 1024, + Value: 4096, Category: flags.PerfCategory, } CacheDatabaseFlag = &cli.IntFlag{ From 402c71f2e2e6a26cea7920e70a660d9027d44b82 Mon Sep 17 00:00:00 2001 From: Jonny Rhea <5555162+jrhea@users.noreply.github.com> Date: Wed, 4 Mar 2026 04:47:10 -0600 Subject: [PATCH 07/52] internal/telemetry: fix undersized span queue causing dropped spans (#33927) The BatchSpanProcessor queue size was incorrectly set to DefaultMaxExportBatchSize (512) instead of DefaultMaxQueueSize (2048). I noticed the issue on bloatnet when analyzing the block building traces. During a particular run, the miner was including 1000 transactions in a single block. When telemetry is enabled, the miner creates a span for each transaction added to the block. With the queue capped at 512, spans were silently dropped when production outpaced the span export, resulting in incomplete traces with orphaned spans. While this doesn't eliminate the possibility of drops under extreme load, using the correct default restores the 4x buffer between queue capacity and export batch size that the SDK was designed around. --- internal/telemetry/tracesetup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/telemetry/tracesetup/setup.go b/internal/telemetry/tracesetup/setup.go index 9637ca1a9b..444416dd26 100644 --- a/internal/telemetry/tracesetup/setup.go +++ b/internal/telemetry/tracesetup/setup.go @@ -113,7 +113,7 @@ func SetupTelemetry(cfg node.OpenTelemetryConfig, stack *node.Node) error { // Define batch span processor options batchOpts := []sdktrace.BatchSpanProcessorOption{ // The maximum number of spans that can be queued before dropping - sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxExportBatchSize), + sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxQueueSize), // The maximum number of spans to export in a single batch sdktrace.WithMaxExportBatchSize(sdktrace.DefaultMaxExportBatchSize), // How long an export operation can take before timing out From fc8c10476d5a43bd892cafb19b86d20642384c6c Mon Sep 17 00:00:00 2001 From: J Date: Wed, 4 Mar 2026 04:37:47 -0700 Subject: [PATCH 08/52] internal/ethapi: add MaxUsedGas field to eth_simulateV1 response (#32789) closes #32741 --- internal/ethapi/simulate.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 325ee6d5bb..63513f8d66 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -59,6 +59,7 @@ type simCallResult struct { ReturnValue hexutil.Bytes `json:"returnData"` Logs []*types.Log `json:"logs"` GasUsed hexutil.Uint64 `json:"gasUsed"` + MaxUsedGas hexutil.Uint64 `json:"maxUsedGas"` Status hexutil.Uint64 `json:"status"` Error *callError `json:"error,omitempty"` } @@ -323,7 +324,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, sim.gasRemaining -= result.UsedGas logs := tracer.Logs() - callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas)} + callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas), MaxUsedGas: hexutil.Uint64(result.MaxUsedGas)} if result.Failed() { callRes.Status = hexutil.Uint64(types.ReceiptStatusFailed) if errors.Is(result.Err, vm.ErrExecutionReverted) { From ce64ab44ed5209e7ac9c9ac5af383309483d155b Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Thu, 5 Mar 2026 02:09:07 +0100 Subject: [PATCH 09/52] internal/ethapi: fix gas cap for eth_simulateV1 (#33952) Fixes a regression in #33593 where a block gas limit > gasCap resulted in more execution than the gas cap. --- internal/ethapi/api.go | 6 +---- internal/ethapi/api_test.go | 5 ++-- internal/ethapi/simulate.go | 45 +++++++++++++++++++++++++++++--- internal/ethapi/simulate_test.go | 5 ++-- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index ff6797f67b..41d165a423 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -849,16 +849,12 @@ func (api *BlockChainAPI) SimulateV1(ctx context.Context, opts simOpts, blockNrO if state == nil || err != nil { return nil, err } - gasCap := api.b.RPCGasCap() - if gasCap == 0 { - gasCap = gomath.MaxUint64 - } sim := &simulator{ b: api.b, state: state, base: base, chainConfig: api.b.ChainConfig(), - gasRemaining: gasCap, + budget: newGasBudget(api.b.RPCGasCap()), traceTransfers: opts.TraceTransfers, validate: opts.Validation, fullTx: opts.ReturnFullTransactions, diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index a82df440e6..62e9979d3d 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -24,7 +24,6 @@ import ( "encoding/json" "errors" "fmt" - "math" "math/big" "os" "path/filepath" @@ -2507,7 +2506,7 @@ func TestSimulateV1ChainLinkage(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gasRemaining: math.MaxUint64, + budget: newGasBudget(0), traceTransfers: false, validate: false, fullTx: false, @@ -2592,7 +2591,7 @@ func TestSimulateV1TxSender(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gasRemaining: math.MaxUint64, + budget: newGasBudget(0), traceTransfers: false, validate: false, fullTx: true, diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 63513f8d66..f0c69e39a4 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" "time" @@ -150,6 +151,39 @@ func (m *simChainHeadReader) GetHeaderByHash(hash common.Hash) *types.Header { return header } +// gasBudget tracks the remaining gas allowed across all simulated blocks. +// It enforces the RPC-level gas cap to prevent DoS. +type gasBudget struct { + remaining uint64 +} + +// newGasBudget creates a gas budget with the given cap. +// A cap of 0 is treated as unlimited. +func newGasBudget(cap uint64) *gasBudget { + if cap == 0 { + cap = math.MaxUint64 + } + return &gasBudget{remaining: cap} +} + +// cap returns the given gas value clamped to the remaining budget. +func (b *gasBudget) cap(gas uint64) uint64 { + if gas > b.remaining { + return b.remaining + } + return gas +} + +// consume deducts the given amount from the budget. +// Returns an error if the amount exceeds the remaining budget. +func (b *gasBudget) consume(amount uint64) error { + if amount > b.remaining { + return fmt.Errorf("RPC gas cap exhausted: need %d, remaining %d", amount, b.remaining) + } + b.remaining -= amount + return nil +} + // simulator is a stateful object that simulates a series of blocks. // it is not safe for concurrent use. type simulator struct { @@ -157,7 +191,7 @@ type simulator struct { state *state.StateDB base *types.Header chainConfig *params.ChainConfig - gasRemaining uint64 + budget *gasBudget traceTransfers bool validate bool fullTx bool @@ -318,10 +352,9 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, // Make sure the gas cap is still enforced. It's only for // internally protection. - if sim.gasRemaining < result.UsedGas { - return nil, nil, nil, fmt.Errorf("gas cap reached, required: %d, remaining: %d", result.UsedGas, sim.gasRemaining) + if err := sim.budget.consume(result.UsedGas); err != nil { + return nil, nil, nil, err } - sim.gasRemaining -= result.UsedGas logs := tracer.Logs() callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas), MaxUsedGas: hexutil.Uint64(result.MaxUsedGas)} @@ -405,6 +438,10 @@ func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, head if remaining < uint64(*call.Gas) { return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: remaining: %d, required: %d", remaining, *call.Gas)} } + // Clamp to the cross-block gas budget. + gas := sim.budget.cap(uint64(*call.Gas)) + call.Gas = (*hexutil.Uint64)(&gas) + return call.CallDefaults(0, header.BaseFee, sim.chainConfig.ChainID) } diff --git a/internal/ethapi/simulate_test.go b/internal/ethapi/simulate_test.go index b6037a8f35..6a83e744de 100644 --- a/internal/ethapi/simulate_test.go +++ b/internal/ethapi/simulate_test.go @@ -17,7 +17,6 @@ package ethapi import ( - "math" "math/big" "testing" @@ -82,8 +81,8 @@ func TestSimulateSanitizeBlockOrder(t *testing.T) { }, } { sim := &simulator{ - base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}, - gasRemaining: math.MaxUint64, + base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}, + budget: newGasBudget(0), } res, err := sim.sanitizeChain(tc.blocks) if err != nil { From 344ce84a431613014c09cb88cdb14514a5a778cf Mon Sep 17 00:00:00 2001 From: Bosul Mun Date: Thu, 5 Mar 2026 12:48:44 +0900 Subject: [PATCH 10/52] eth/fetcher: fix flaky test by improving event unsubscription (#33950) Eth currently has a flaky test, related to the tx fetcher. The issue seems to happen when Unsubscribe is called while sub is nil. It seems that chain.Stop() may be invoked before the loop starts in some tests, but the exact cause is still under investigation through repeated runs. I think this change will at least prevent the error. --- eth/fetcher/tx_fetcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index bc422e6abe..5817dfbcf5 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -442,7 +442,9 @@ func (f *TxFetcher) loop() { if f.chain != nil { headEventCh = make(chan core.ChainEvent, 10) sub := f.chain.SubscribeChainEvent(headEventCh) - defer sub.Unsubscribe() + if sub != nil { + defer sub.Unsubscribe() + } } for { From a0fb8102fefd524dcb1ad884f99ad310f7fe4fe2 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:43:31 +0100 Subject: [PATCH 11/52] trie/bintrie: fix overflow management in slot key computation (#33951) The computation of `MAIN_STORAGE_OFFSET` was incorrect, causing the last byte of the stem to be dropped. This means that there would be a collision in the hash computation (at the preimage level, not a hash collision of course) if two keys were only differing at byte 31. --- core/genesis_test.go | 2 +- trie/bintrie/key_encoding.go | 50 ++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/core/genesis_test.go b/core/genesis_test.go index ba00581b06..2b08b36690 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -308,7 +308,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.FromHex("b94812c1674dcf4f2bc98f4503d15f4cc674265135bcf3be6e4417b60881042a") + expected := common.FromHex("1fd154971d9a386c4ec75fe7138c17efb569bfc2962e46e94a376ba997e3fadc") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/trie/bintrie/key_encoding.go b/trie/bintrie/key_encoding.go index 94a22d52d0..9b98bee491 100644 --- a/trie/bintrie/key_encoding.go +++ b/trie/bintrie/key_encoding.go @@ -47,13 +47,26 @@ var ( ) func GetBinaryTreeKey(addr common.Address, key []byte) []byte { + return getBinaryTreeKey(addr, key, false) +} + +func getBinaryTreeKey(addr common.Address, offset []byte, overflow bool) []byte { hasher := sha256.New() hasher.Write(zeroHash[:12]) hasher.Write(addr[:]) - hasher.Write(key[:31]) - hasher.Write([]byte{0}) + var buf [32]byte + // key is big endian, hashed value is little endian + for i := range offset[:31] { + buf[i] = offset[30-i] + } + if overflow { + // Overflow detected when adding MAIN_STORAGE_OFFSET, + // reporting it in the shifter 32 byte value. + buf[31] = 1 + } + hasher.Write(buf[:]) k := hasher.Sum(nil) - k[31] = key[31] + k[31] = offset[31] return k } @@ -69,24 +82,29 @@ func GetBinaryTreeKeyCodeHash(addr common.Address) []byte { return GetBinaryTreeKey(addr, k[:]) } -func GetBinaryTreeKeyStorageSlot(address common.Address, key []byte) []byte { - var k [32]byte +func GetBinaryTreeKeyStorageSlot(address common.Address, slotnum []byte) []byte { + var offset [32]byte // Case when the key belongs to the account header - if bytes.Equal(key[:31], zeroHash[:31]) && key[31] < 64 { - k[31] = 64 + key[31] - return GetBinaryTreeKey(address, k[:]) + if bytes.Equal(slotnum[:31], zeroHash[:31]) && slotnum[31] < 64 { + offset[31] = 64 + slotnum[31] + return GetBinaryTreeKey(address, offset[:]) } - // Set the main storage offset - // note that the first 64 bytes of the main offset storage - // are unreachable, which is consistent with the spec and - // what verkle does. - k[0] = 1 // 1 << 248 - copy(k[1:], key[:31]) - k[31] = key[31] + // Set the main storage offset offset = MAIN_STORAGE_OFFSET + slotnum + // * Note that MAIN_STORAGE_OFFSET is 1 << 248, so the number + // can overflow into a 33rd byte, but since the value is + // shifted by one byte in getBinaryTreeKey, this only takes + // note of the overflow, and the value will be added after + // the shift, in order to avoid allocating an extra byte. + // * Note that the first 64 bytes of the main offset storage + // are unreachable, which is consistent with the spec. + // * Note that `slotnum` is big-endian + overflow := slotnum[0] == 255 + copy(offset[:], slotnum) + offset[0] += 1 // 1 << 248, handle overflow out of band - return GetBinaryTreeKey(address, k[:]) + return getBinaryTreeKey(address, offset[:], overflow) } func GetBinaryTreeKeyCodeChunk(address common.Address, chunknr *uint256.Int) []byte { From 3f1871524fda4e8da31fbbad927e554993228575 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:06:24 +0100 Subject: [PATCH 12/52] trie/bintrie: cache hashes of clean nodes so as not to rehash the whole tree (#33961) This is an optimization that existed for verkle and the MPT, but that got dropped during the rebase. Mark the nodes that were modified as needing recomputation, and skip the hash computation if this is not needed. Otherwise, the whole tree is hashed, which kills performance. --- trie/bintrie/binary_node.go | 28 +++++++++++++++------ trie/bintrie/empty.go | 14 ++++++----- trie/bintrie/hashed_node.go | 2 +- trie/bintrie/internal_node.go | 29 ++++++++++++++++------ trie/bintrie/internal_node_test.go | 8 +++--- trie/bintrie/iterator.go | 2 +- trie/bintrie/stem_node.go | 39 +++++++++++++++++++++--------- trie/bintrie/stem_node_test.go | 1 + trie/bintrie/trie.go | 2 +- 9 files changed, 86 insertions(+), 39 deletions(-) diff --git a/trie/bintrie/binary_node.go b/trie/bintrie/binary_node.go index 690489b2aa..a7392ec958 100644 --- a/trie/bintrie/binary_node.go +++ b/trie/bintrie/binary_node.go @@ -90,8 +90,18 @@ func SerializeNode(node BinaryNode) []byte { var invalidSerializedLength = errors.New("invalid serialized node length") -// DeserializeNode deserializes a binary trie node from a byte slice. +// DeserializeNode deserializes a binary trie node from a byte slice. The +// hash will be recomputed from the deserialized data. func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { + return deserializeNode(serialized, depth, common.Hash{}, true) +} + +// DeserializeNodeWithHash deserializes a binary trie node from a byte slice, using the provided hash. +func DeserializeNodeWithHash(serialized []byte, depth int, hn common.Hash) (BinaryNode, error) { + return deserializeNode(serialized, depth, hn, false) +} + +func deserializeNode(serialized []byte, depth int, hn common.Hash, mustRecompute bool) (BinaryNode, error) { if len(serialized) == 0 { return Empty{}, nil } @@ -102,9 +112,11 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { return nil, invalidSerializedLength } return &InternalNode{ - depth: depth, - left: HashedNode(common.BytesToHash(serialized[1:33])), - right: HashedNode(common.BytesToHash(serialized[33:65])), + depth: depth, + left: HashedNode(common.BytesToHash(serialized[1:33])), + right: HashedNode(common.BytesToHash(serialized[33:65])), + hash: hn, + mustRecompute: mustRecompute, }, nil case nodeTypeStem: if len(serialized) < 64 { @@ -124,9 +136,11 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { } } return &StemNode{ - Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], - Values: values[:], - depth: depth, + Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], + Values: values[:], + depth: depth, + hash: hn, + mustRecompute: mustRecompute, }, nil default: return nil, errors.New("invalid node type") diff --git a/trie/bintrie/empty.go b/trie/bintrie/empty.go index 7cfe373b35..252146a4a7 100644 --- a/trie/bintrie/empty.go +++ b/trie/bintrie/empty.go @@ -32,9 +32,10 @@ func (e Empty) Insert(key []byte, value []byte, _ NodeResolverFn, depth int) (Bi var values [256][]byte values[key[31]] = value return &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values[:], - depth: depth, + Stem: slices.Clone(key[:31]), + Values: values[:], + depth: depth, + mustRecompute: true, }, nil } @@ -53,9 +54,10 @@ func (e Empty) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error) { func (e Empty) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, depth int) (BinaryNode, error) { return &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values, - depth: depth, + Stem: slices.Clone(key[:31]), + Values: values, + depth: depth, + mustRecompute: true, }, nil } diff --git a/trie/bintrie/hashed_node.go b/trie/bintrie/hashed_node.go index e4d8c2e7ac..e44c6d1e8a 100644 --- a/trie/bintrie/hashed_node.go +++ b/trie/bintrie/hashed_node.go @@ -64,7 +64,7 @@ func (h HashedNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver No } // Step 3: Deserialize the resolved data into a concrete node - node, err := DeserializeNode(data, depth) + node, err := DeserializeNodeWithHash(data, depth, common.Hash(h)) if err != nil { return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) } diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index 0a7bece521..2d02e240be 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -40,6 +40,9 @@ func keyToPath(depth int, key []byte) ([]byte, error) { type InternalNode struct { left, right BinaryNode depth int + + mustRecompute bool // true if the hash needs to be recomputed + hash common.Hash // cached hash when mustRecompute == false } // GetValuesAtStem retrieves the group of values located at the given stem key. @@ -59,7 +62,7 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } @@ -77,7 +80,7 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } @@ -108,14 +111,20 @@ func (bt *InternalNode) Insert(key []byte, value []byte, resolver NodeResolverFn // Copy creates a deep copy of the node. func (bt *InternalNode) Copy() BinaryNode { return &InternalNode{ - left: bt.left.Copy(), - right: bt.right.Copy(), - depth: bt.depth, + left: bt.left.Copy(), + right: bt.right.Copy(), + depth: bt.depth, + mustRecompute: bt.mustRecompute, + hash: bt.hash, } } // Hash returns the hash of the node. func (bt *InternalNode) Hash() common.Hash { + if !bt.mustRecompute { + return bt.hash + } + h := sha256.New() if bt.left != nil { h.Write(bt.left.Hash().Bytes()) @@ -127,7 +136,9 @@ func (bt *InternalNode) Hash() common.Hash { } else { h.Write(zero[:]) } - return common.BytesToHash(h.Sum(nil)) + bt.hash = common.BytesToHash(h.Sum(nil)) + bt.mustRecompute = false + return bt.hash } // InsertValuesAtStem inserts a full value group at the given stem in the internal node. @@ -149,7 +160,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve if err != nil { return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) } @@ -157,6 +168,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve } bt.left, err = bt.left.InsertValuesAtStem(stem, values, resolver, depth+1) + bt.mustRecompute = true return bt, err } @@ -173,7 +185,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve if err != nil { return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) } @@ -181,6 +193,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve } bt.right, err = bt.right.InsertValuesAtStem(stem, values, resolver, depth+1) + bt.mustRecompute = true return bt, err } diff --git a/trie/bintrie/internal_node_test.go b/trie/bintrie/internal_node_test.go index 158d8b7147..69097483fd 100644 --- a/trie/bintrie/internal_node_test.go +++ b/trie/bintrie/internal_node_test.go @@ -239,6 +239,7 @@ func TestInternalNodeHash(t *testing.T) { // Changing a child should change the hash node.left = HashedNode(common.HexToHash("0x3333")) + node.mustRecompute = true hash3 := node.Hash() if hash1 == hash3 { t.Error("Hash didn't change after modifying left child") @@ -246,9 +247,10 @@ func TestInternalNodeHash(t *testing.T) { // Test with nil children (should use zero hash) nodeWithNil := &InternalNode{ - depth: 0, - left: nil, - right: HashedNode(common.HexToHash("0x4444")), + depth: 0, + left: nil, + right: HashedNode(common.HexToHash("0x4444")), + mustRecompute: true, } hashWithNil := nodeWithNil.Hash() if hashWithNil == (common.Hash{}) { diff --git a/trie/bintrie/iterator.go b/trie/bintrie/iterator.go index 9b863ed1e3..917f82efc9 100644 --- a/trie/bintrie/iterator.go +++ b/trie/bintrie/iterator.go @@ -123,7 +123,7 @@ func (it *binaryNodeIterator) Next(descend bool) bool { if err != nil { panic(err) } - it.current, err = DeserializeNode(data, len(it.stack)-1) + it.current, err = DeserializeNodeWithHash(data, len(it.stack)-1, common.Hash(node)) if err != nil { panic(err) } diff --git a/trie/bintrie/stem_node.go b/trie/bintrie/stem_node.go index 60856b42ce..f1ae2361ff 100644 --- a/trie/bintrie/stem_node.go +++ b/trie/bintrie/stem_node.go @@ -31,6 +31,9 @@ type StemNode struct { Stem []byte // Stem path to get to StemNodeWidth values Values [][]byte // All values, indexed by the last byte of the key. depth int // Depth of the node + + mustRecompute bool // true if the hash needs to be recomputed + hash common.Hash // cached hash when mustRecompute == false } // Get retrieves the value for the given key. @@ -43,7 +46,7 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - n := &InternalNode{depth: bt.depth} + n := &InternalNode{depth: bt.depth, mustRecompute: true} bt.depth++ var child, other *BinaryNode if bitStem == 0 { @@ -68,9 +71,10 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int var values [StemNodeWidth][]byte values[key[StemSize]] = value *other = &StemNode{ - Stem: slices.Clone(key[:StemSize]), - Values: values[:], - depth: depth + 1, + Stem: slices.Clone(key[:StemSize]), + Values: values[:], + depth: depth + 1, + mustRecompute: true, } } return n, nil @@ -79,6 +83,7 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int return bt, errors.New("invalid insertion: value length") } bt.Values[key[StemSize]] = value + bt.mustRecompute = true return bt, nil } @@ -89,9 +94,11 @@ func (bt *StemNode) Copy() BinaryNode { values[i] = slices.Clone(v) } return &StemNode{ - Stem: slices.Clone(bt.Stem), - Values: values[:], - depth: bt.depth, + Stem: slices.Clone(bt.Stem), + Values: values[:], + depth: bt.depth, + hash: bt.hash, + mustRecompute: bt.mustRecompute, } } @@ -102,6 +109,10 @@ func (bt *StemNode) GetHeight() int { // Hash returns the hash of the node. func (bt *StemNode) Hash() common.Hash { + if !bt.mustRecompute { + return bt.hash + } + var data [StemNodeWidth]common.Hash for i, v := range bt.Values { if v != nil { @@ -130,7 +141,9 @@ func (bt *StemNode) Hash() common.Hash { h.Write(bt.Stem) h.Write([]byte{0}) h.Write(data[0][:]) - return common.BytesToHash(h.Sum(nil)) + bt.hash = common.BytesToHash(h.Sum(nil)) + bt.mustRecompute = false + return bt.hash } // CollectNodes collects all child nodes at a given path, and flushes it @@ -154,7 +167,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - n := &InternalNode{depth: bt.depth} + n := &InternalNode{depth: bt.depth, mustRecompute: true} bt.depth++ var child, other *BinaryNode if bitStem == 0 { @@ -177,9 +190,10 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv *other = Empty{} } else { *other = &StemNode{ - Stem: slices.Clone(key[:StemSize]), - Values: values, - depth: n.depth + 1, + Stem: slices.Clone(key[:StemSize]), + Values: values, + depth: n.depth + 1, + mustRecompute: true, } } return n, nil @@ -189,6 +203,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv for i, v := range values { if v != nil { bt.Values[i] = v + bt.mustRecompute = true } } return bt, nil diff --git a/trie/bintrie/stem_node_test.go b/trie/bintrie/stem_node_test.go index d8d6844427..92c1b49e02 100644 --- a/trie/bintrie/stem_node_test.go +++ b/trie/bintrie/stem_node_test.go @@ -220,6 +220,7 @@ func TestStemNodeHash(t *testing.T) { // Changing a value should change the hash node.Values[1] = common.HexToHash("0x0202").Bytes() + node.mustRecompute = true hash3 := node.Hash() if hash1 == hash3 { t.Error("Hash didn't change after modifying values") diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index a509c471b8..6c29239a87 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -143,7 +143,7 @@ func NewBinaryTrie(root common.Hash, db database.NodeDatabase) (*BinaryTrie, err if err != nil { return nil, err } - node, err := DeserializeNode(blob, 0) + node, err := DeserializeNodeWithHash(blob, 0, root) if err != nil { return nil, err } From ecee64ecdc798ee1ec7edb3346373e5ce705f41b Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:03:05 +0100 Subject: [PATCH 13/52] core: fix TestProcessVerkle flaky test (#33971) `GenerateChain` commits trie nodes asynchronously, and it can happen that some nodes aren't making it to the db in time for `GenerateChain` to open it and find the data it is looking for. --- core/chain_makers.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index 5264336aaa..e4b5cf964f 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -481,13 +481,14 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, if genesis.Config != nil && genesis.Config.IsVerkle(genesis.Config.ChainID, 0) { triedbConfig = triedb.VerkleDefaults } - triedb := triedb.NewDatabase(db, triedbConfig) - defer triedb.Close() - _, err := genesis.Commit(db, triedb, nil) + genesisTriedb := triedb.NewDatabase(db, triedbConfig) + block, err := genesis.Commit(db, genesisTriedb, nil) if err != nil { + genesisTriedb.Close() panic(err) } - blocks, receipts := GenerateChain(genesis.Config, genesis.ToBlock(), engine, db, n, gen) + genesisTriedb.Close() + blocks, receipts := GenerateChain(genesis.Config, block, engine, db, n, gen) return db, blocks, receipts } From 0d043d071e7a6bb63a49c7d8499c339f531c83ad Mon Sep 17 00:00:00 2001 From: marukai67 Date: Fri, 6 Mar 2026 21:50:30 +0100 Subject: [PATCH 14/52] signer/core: prevent nil pointer panics in keystore operations (#33829) Add nil checks to prevent potential panics when keystore backend is unavailable in the Clef signer API. --- signer/core/uiapi.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/signer/core/uiapi.go b/signer/core/uiapi.go index 2f511c7e19..09ee4b492f 100644 --- a/signer/core/uiapi.go +++ b/signer/core/uiapi.go @@ -73,8 +73,9 @@ type rawWallet struct { // Example call // {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5} func (api *UIServerAPI) ListWallets() []rawWallet { - wallets := make([]rawWallet, 0) // return [] instead of nil if empty - for _, wallet := range api.am.Wallets() { + allWallets := api.am.Wallets() + wallets := make([]rawWallet, 0, len(allWallets)) // return [] instead of nil if empty + for _, wallet := range allWallets { status, failure := wallet.Status() raw := rawWallet{ @@ -130,8 +131,12 @@ func (api *UIServerAPI) ImportRawKey(privkey string, password string) (accounts. if err := ValidatePasswordFormat(password); err != nil { return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err) } + ks := fetchKeystore(api.am) + if ks == nil { + return accounts.Account{}, errors.New("password based accounts not supported") + } // No error - return fetchKeystore(api.am).ImportECDSA(key, password) + return ks.ImportECDSA(key, password) } // OpenWallet initiates a hardware wallet opening procedure, establishing a USB From e15d4ccc0195d0725926614c2fec4396976f259e Mon Sep 17 00:00:00 2001 From: cui Date: Sat, 7 Mar 2026 21:31:36 +0800 Subject: [PATCH 15/52] core/types: reduce alloc in hot code path (#33523) Reduce allocations in calculation of tx cost. --------- Co-authored-by: weixie.cui Co-authored-by: Sina M <1591639+s1na@users.noreply.github.com> --- core/types/transaction.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 21f858ecfa..e9bf08daef 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -317,11 +317,15 @@ func (tx *Transaction) To() *common.Address { // Cost returns (gas * gasPrice) + (blobGas * blobGasPrice) + value. func (tx *Transaction) Cost() *big.Int { - total := new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas())) - if tx.Type() == BlobTxType { - total.Add(total, new(big.Int).Mul(tx.BlobGasFeeCap(), new(big.Int).SetUint64(tx.BlobGas()))) + // Avoid allocating copies via tx.GasPrice()/tx.Value(); use inner values directly. + total := new(big.Int).SetUint64(tx.inner.gas()) + total.Mul(total, tx.inner.gasPrice()) + if blobtx, ok := tx.inner.(*BlobTx); ok { + tmp := new(big.Int).SetUint64(blobtx.blobGas()) + tmp.Mul(tmp, blobtx.BlobFeeCap.ToBig()) + total.Add(total, tmp) } - total.Add(total, tx.Value()) + total.Add(total, tx.inner.value()) return total } From 00540f94699c099f3e4aee823fc9a19888d7a4a7 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sun, 8 Mar 2026 11:44:29 +0100 Subject: [PATCH 16/52] go.mod: update go-eth-kzg (#33963) Updates go-eth-kzg to https://github.com/crate-crypto/go-eth-kzg/releases/tag/v1.5.0 Significantly reduces the allocations in VerifyCellProofBatch which is around ~5% of all allocations on my node --------- Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- cmd/keeper/go.mod | 2 +- cmd/keeper/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index abf5d4c7a1..8303b4ab2e 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -13,7 +13,7 @@ require ( github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/gnark-crypto v0.18.1 // indirect - github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect + github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/emicklei/dot v1.6.2 // indirect diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index 2c28c6a2ec..a162537c88 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -28,8 +28,8 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAK github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= -github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= -github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= diff --git a/go.mod b/go.mod index 81c00719bd..bfe2df8c0c 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/cloudflare/cloudflare-go v0.114.0 github.com/cockroachdb/pebble v1.1.5 github.com/consensys/gnark-crypto v0.18.1 - github.com/crate-crypto/go-eth-kzg v1.4.0 + github.com/crate-crypto/go-eth-kzg v1.5.0 github.com/davecgh/go-spew v1.1.1 github.com/dchest/siphash v1.2.3 github.com/deckarep/golang-set/v2 v2.6.0 diff --git a/go.sum b/go.sum index 72ae43c24f..b8c0558c8c 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,8 @@ github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDd github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= -github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From b08aac1dbce8138980ff3a4ca60d97f4e7aa734a Mon Sep 17 00:00:00 2001 From: Muzry Date: Mon, 9 Mar 2026 18:22:58 +0800 Subject: [PATCH 17/52] eth/catalyst: allow getPayloadV2 for pre-shanghai payloads (#33932) I observed failing tests in Hive `engine-withdrawals`: - https://hive.ethpandaops.io/#/test/generic/1772351960-ad3e3e460605c670efe1b4f4178eb422?testnumber=146 - https://hive.ethpandaops.io/#/test/generic/1772351960-ad3e3e460605c670efe1b4f4178eb422?testnumber=147 ```shell DEBUG (Withdrawals Fork on Block 2): NextPayloadID before getPayloadV2: id=0x01487547e54e8abe version=1 >> engine_getPayloadV2("0x01487547e54e8abe") << error: {"code":-38005,"message":"Unsupported fork"} FAIL: Expected no error on EngineGetPayloadV2: error=Unsupported fork ``` The same failure pattern occurred for Block 3. Per Shanghai engine_getPayloadV2 spec, pre-Shanghai payloads should be accepted via V2 and returned as ExecutionPayloadV1: - executionPayload: ExecutionPayloadV1 | ExecutionPayloadV2 - ExecutionPayloadV1 MUST be returned if payload timestamp < Shanghai timestamp - ExecutionPayloadV2 MUST be returned if payload timestamp >= Shanghai timestamp Reference: - https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_getpayloadv2 Current implementation only allows GetPayloadV2 on the Shanghai fork window (`[]forks.Fork{forks.Shanghai}`), so pre-Shanghai payloads are rejected with Unsupported fork. If my interpretation of the spec is incorrect, please let me know and I can adjust accordingly. --------- Co-authored-by: muzry.li --- eth/catalyst/api.go | 2 +- eth/catalyst/api_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 1e019ffb15..c64039b690 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -435,7 +435,7 @@ func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.Execu payloadID, false, []engine.PayloadVersion{engine.PayloadV1, engine.PayloadV2}, - []forks.Fork{forks.Shanghai}, + []forks.Fork{forks.Paris, forks.Shanghai}, ) } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 7eb26065dc..db0505101f 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -1219,6 +1219,11 @@ func TestNilWithdrawals(t *testing.T) { Random: test.blockParams.Random, Version: payloadVersion, }).Id() + if !shanghai { + if _, err := api.GetPayloadV2(payloadID); err != nil { + t.Fatalf("GetPayloadV2 rejected pre-shanghai payload: %v", err) + } + } execData, err := api.getPayload(payloadID, false, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) From b8a3fa7d063de6b37b0242121e4185a7b17353a6 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 9 Mar 2026 23:18:18 +0800 Subject: [PATCH 18/52] cmd/utils, eth/ethconfig: change default cache settings (#33975) This PR fixes a regression introduced in https://github.com/ethereum/go-ethereum/pull/33836/changes Before PR 33836, running mainnet would automatically bump the cache size to 4GB and trigger a cache re-calculation, specifically setting the key-value database cache to 2GB. After PR 33836, this logic was removed, and the cache value is no longer recomputed if no command line flags are specified. The default key-value database cache is 512MB. This PR bumps the default key-value database cache size alongside the default cache size for other components (such as snapshot) accordingly. --- cmd/utils/flags.go | 8 +------- eth/ethconfig/config.go | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b0d6ee5203..d5d2bfbf1c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -218,17 +218,11 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } - TopFlag = &cli.IntFlag{ - Name: "top", - Usage: "Print the top N results", - Value: 5, - } OutputFileFlag = &cli.StringFlag{ Name: "output", Usage: "Writes the result in json to the output", Value: "", } - SnapshotFlag = &cli.BoolFlag{ Name: "snapshot", Usage: `Enables snapshot-database mode (default = enable)`, @@ -490,7 +484,7 @@ var ( // Performance tuning settings CacheFlag = &cli.IntFlag{ Name: "cache", - Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", + Usage: "Megabytes of memory allocated to internal caching", Value: 4096, Category: flags.PerfCategory, } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 8aa6e4ef09..01aaaa751b 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -59,11 +59,11 @@ var Defaults = Config{ StateHistory: pathdb.Defaults.StateHistory, TrienodeHistory: pathdb.Defaults.TrienodeHistory, NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieDirtyCache: 256, + DatabaseCache: 2048, + TrieCleanCache: 614, + TrieDirtyCache: 1024, + SnapshotCache: 409, TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, FilterLogCacheSize: 32, LogQueryLimit: 1000, Miner: miner.DefaultConfig, From 91cec92bf364525e03b9c449de26ab0b15cfe9b1 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 10 Mar 2026 15:29:21 +0800 Subject: [PATCH 19/52] core, miner, tests: introduce codedb and simplify cachingDB (#33816) --- core/blockchain.go | 33 +++-- core/blockchain_reader.go | 18 +-- core/blockchain_sethead_test.go | 2 - core/blockchain_stats.go | 46 +++--- core/blockchain_test.go | 2 +- core/state/database.go | 108 ++++++++------ core/state/database_code.go | 231 ++++++++++++++++++++++++++++++ core/state/database_history.go | 26 ++-- core/state/iterator.go | 5 +- core/state/reader.go | 242 ++++++++------------------------ core/state/reader_stater.go | 82 +++++++++++ core/state/state_object.go | 10 +- core/state/statedb.go | 46 ++---- core/state/statedb_fuzz_test.go | 2 +- core/state/statedb_test.go | 4 +- core/state/stateupdate.go | 6 +- core/state/sync_test.go | 20 +-- miner/miner_test.go | 2 +- tests/state_test_util.go | 2 +- triedb/hashdb/database.go | 3 + 20 files changed, 525 insertions(+), 365 deletions(-) create mode 100644 core/state/database_code.go create mode 100644 core/state/reader_stater.go diff --git a/core/blockchain.go b/core/blockchain.go index 858d24bad7..126ff1f666 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -93,9 +93,7 @@ var ( accountReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/account/single/reads", nil) storageReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/storage/single/reads", nil) codeReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/code/single/reads", nil) - - snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) - triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) blockInsertTimer = metrics.NewRegisteredResettingTimer("chain/inserts", nil) blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil) @@ -326,7 +324,7 @@ type BlockChain struct { lastWrite uint64 // Last block when the state was flushed flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state triedb *triedb.Database // The database handler for maintaining trie nodes. - statedb *state.CachingDB // State database to reuse between imports (contains state cache) + codedb *state.CodeDB // The database handler for maintaining contract codes. txIndexer *txIndexer // Transaction indexer, might be nil if not enabled hc *HeaderChain @@ -408,6 +406,7 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine, cfg: cfg, db: db, triedb: triedb, + codedb: state.NewCodeDB(db), triegc: prque.New[int64, common.Hash](nil), chainmu: syncx.NewClosableMutex(), bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), @@ -424,7 +423,6 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine, return nil, err } bc.flushInterval.Store(int64(cfg.TrieTimeLimit)) - bc.statedb = state.NewDatabase(bc.triedb, nil) bc.validator = NewBlockValidator(chainConfig, bc) bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc) bc.processor = NewStateProcessor(bc.hc) @@ -601,9 +599,6 @@ func (bc *BlockChain) setupSnapshot() { AsyncBuild: !bc.cfg.SnapshotWait, } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) - - // Re-initialize the state database with snapshot - bc.statedb = state.NewDatabase(bc.triedb, bc.snaps) } } @@ -2124,11 +2119,12 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, startTime = time.Now() statedb *state.StateDB interrupt atomic.Bool + sdb = state.NewDatabase(bc.triedb, bc.codedb).WithSnapshot(bc.snaps) ) defer interrupt.Store(true) // terminate the prefetch at the end if bc.cfg.NoPrefetch { - statedb, err = state.New(parentRoot, bc.statedb) + statedb, err = state.New(parentRoot, sdb) if err != nil { return nil, err } @@ -2138,23 +2134,27 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, // // Note: the main processor and prefetcher share the same reader with a local // cache for mitigating the overhead of state access. - prefetch, process, err := bc.statedb.ReadersWithCacheStats(parentRoot) + prefetch, process, err := sdb.ReadersWithCacheStats(parentRoot) if err != nil { return nil, err } - throwaway, err := state.NewWithReader(parentRoot, bc.statedb, prefetch) + throwaway, err := state.NewWithReader(parentRoot, sdb, prefetch) if err != nil { return nil, err } - statedb, err = state.NewWithReader(parentRoot, bc.statedb, process) + statedb, err = state.NewWithReader(parentRoot, sdb, process) if err != nil { return nil, err } // Upload the statistics of reader at the end defer func() { if result != nil { - result.stats.StatePrefetchCacheStats = prefetch.GetStats() - result.stats.StateReadCacheStats = process.GetStats() + if stater, ok := prefetch.(state.ReaderStater); ok { + result.stats.StatePrefetchCacheStats = stater.GetStats() + } + if stater, ok := process.(state.ReaderStater); ok { + result.stats.StateReadCacheStats = stater.GetStats() + } } }() go func(start time.Time, throwaway *state.StateDB, block *types.Block) { @@ -2305,9 +2305,8 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, // Update the metrics touched during block commit stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them - stats.SnapshotCommit = statedb.SnapshotCommits // Snapshot commits are complete, we can mark them - stats.TrieDBCommit = statedb.TrieDBCommits // Trie database commits are complete, we can mark them - stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits + stats.DatabaseCommit = statedb.DatabaseCommits // Database commits are complete, we can mark them + stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.DatabaseCommits } // Report the collected witness statistics if witnessStats != nil { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index ee15c152c4..f1b40d0d0c 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -371,7 +371,7 @@ func (bc *BlockChain) TxIndexDone() bool { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { - _, err := bc.statedb.OpenTrie(hash) + _, err := bc.triedb.NodeReader(hash) return err == nil } @@ -403,7 +403,7 @@ func (bc *BlockChain) stateRecoverable(root common.Hash) bool { func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) []byte { // TODO(rjl493456442) The associated account address is also required // in Verkle scheme. Fix it once snap-sync is supported for Verkle. - return bc.statedb.ContractCodeWithPrefix(common.Address{}, hash) + return bc.codedb.Reader().CodeWithPrefix(common.Address{}, hash) } // State returns a new mutable state based on the current HEAD block. @@ -413,14 +413,14 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return state.New(root, bc.statedb) + return state.New(root, state.NewDatabase(bc.triedb, bc.codedb).WithSnapshot(bc.snaps)) } // HistoricState returns a historic state specified by the given root. // Live states are not available and won't be served, please use `State` // or `StateAt` instead. func (bc *BlockChain) HistoricState(root common.Hash) (*state.StateDB, error) { - return state.New(root, state.NewHistoricDatabase(bc.db, bc.triedb)) + return state.New(root, state.NewHistoricDatabase(bc.triedb, bc.codedb)) } // Config retrieves the chain's fork configuration. @@ -444,11 +444,6 @@ func (bc *BlockChain) Processor() Processor { return bc.processor } -// StateCache returns the caching database underpinning the blockchain instance. -func (bc *BlockChain) StateCache() state.Database { - return bc.statedb -} - // GasLimit returns the gas limit of the current HEAD block. func (bc *BlockChain) GasLimit() uint64 { return bc.CurrentBlock().GasLimit @@ -492,6 +487,11 @@ func (bc *BlockChain) TrieDB() *triedb.Database { return bc.triedb } +// CodeDB retrieves the low level contract code database used for data storage. +func (bc *BlockChain) CodeDB() *state.CodeDB { + return bc.codedb +} + // HeaderChain returns the underlying header chain. func (bc *BlockChain) HeaderChain() *HeaderChain { return bc.hc diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 72ca15d7f6..f2fbc003f1 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" @@ -2041,7 +2040,6 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme dbconfig.HashDB = hashdb.Defaults } chain.triedb = triedb.NewDatabase(chain.db, dbconfig) - chain.statedb = state.NewDatabase(chain.triedb, chain.snaps) // Force run a freeze cycle type freezer interface { diff --git a/core/blockchain_stats.go b/core/blockchain_stats.go index adc66266c4..d753b0b700 100644 --- a/core/blockchain_stats.go +++ b/core/blockchain_stats.go @@ -52,8 +52,7 @@ type ExecuteStats struct { Execution time.Duration // Time spent on the EVM execution Validation time.Duration // Time spent on the block validation CrossValidation time.Duration // Optional, time spent on the block cross validation - SnapshotCommit time.Duration // Time spent on snapshot commit - TrieDBCommit time.Duration // Time spent on database commit + DatabaseCommit time.Duration // Time spent on database commit BlockWrite time.Duration // Time spent on block write TotalTime time.Duration // The total time spent on block execution MgasPerSecond float64 // The million gas processed per second @@ -87,22 +86,21 @@ func (s *ExecuteStats) reportMetrics() { blockExecutionTimer.Update(s.Execution) // The time spent on EVM processing blockValidationTimer.Update(s.Validation) // The time spent on block validation blockCrossValidationTimer.Update(s.CrossValidation) // The time spent on stateless cross validation - snapshotCommitTimer.Update(s.SnapshotCommit) // Snapshot commits are complete, we can mark them - triedbCommitTimer.Update(s.TrieDBCommit) // Trie database commits are complete, we can mark them + triedbCommitTimer.Update(s.DatabaseCommit) // Trie database commits are complete, we can mark them blockWriteTimer.Update(s.BlockWrite) // The time spent on block write blockInsertTimer.Update(s.TotalTime) // The total time spent on block execution chainMgaspsMeter.Update(time.Duration(s.MgasPerSecond)) // TODO(rjl493456442) generalize the ResettingTimer // Cache hit rates - accountCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.AccountCacheHit) - accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.AccountCacheMiss) - storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StorageCacheHit) - storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StorageCacheMiss) + accountCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheHit) + accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheMiss) + storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheHit) + storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheMiss) - accountCacheHitMeter.Mark(s.StateReadCacheStats.AccountCacheHit) - accountCacheMissMeter.Mark(s.StateReadCacheStats.AccountCacheMiss) - storageCacheHitMeter.Mark(s.StateReadCacheStats.StorageCacheHit) - storageCacheMissMeter.Mark(s.StateReadCacheStats.StorageCacheMiss) + accountCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheHit) + accountCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheMiss) + storageCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheHit) + storageCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheMiss) } // slowBlockLog represents the JSON structure for slow block logging. @@ -177,14 +175,6 @@ type slowBlockCodeCacheEntry struct { MissBytes int64 `json:"miss_bytes"` } -// calculateHitRate computes the cache hit rate as a percentage (0-100). -func calculateHitRate(hits, misses int64) float64 { - if total := hits + misses; total > 0 { - return float64(hits) / float64(total) * 100.0 - } - return 0.0 -} - // durationToMs converts a time.Duration to milliseconds as a float64 // with sub-millisecond precision for accurate cross-client metrics. func durationToMs(d time.Duration) float64 { @@ -216,7 +206,7 @@ func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Durat ExecutionMs: durationToMs(s.Execution), StateReadMs: durationToMs(s.AccountReads + s.StorageReads + s.CodeReads), StateHashMs: durationToMs(s.AccountHashes + s.AccountUpdates + s.StorageUpdates), - CommitMs: durationToMs(max(s.AccountCommits, s.StorageCommits) + s.TrieDBCommit + s.SnapshotCommit + s.BlockWrite), + CommitMs: durationToMs(max(s.AccountCommits, s.StorageCommits) + s.DatabaseCommit + s.BlockWrite), TotalMs: durationToMs(s.TotalTime), }, Throughput: slowBlockThru{ @@ -238,19 +228,19 @@ func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Durat }, Cache: slowBlockCache{ Account: slowBlockCacheEntry{ - Hits: s.StateReadCacheStats.AccountCacheHit, - Misses: s.StateReadCacheStats.AccountCacheMiss, - HitRate: calculateHitRate(s.StateReadCacheStats.AccountCacheHit, s.StateReadCacheStats.AccountCacheMiss), + Hits: s.StateReadCacheStats.StateStats.AccountCacheHit, + Misses: s.StateReadCacheStats.StateStats.AccountCacheMiss, + HitRate: s.StateReadCacheStats.StateStats.AccountCacheHitRate(), }, Storage: slowBlockCacheEntry{ - Hits: s.StateReadCacheStats.StorageCacheHit, - Misses: s.StateReadCacheStats.StorageCacheMiss, - HitRate: calculateHitRate(s.StateReadCacheStats.StorageCacheHit, s.StateReadCacheStats.StorageCacheMiss), + Hits: s.StateReadCacheStats.StateStats.StorageCacheHit, + Misses: s.StateReadCacheStats.StateStats.StorageCacheMiss, + HitRate: s.StateReadCacheStats.StateStats.StorageCacheHitRate(), }, Code: slowBlockCodeCacheEntry{ Hits: s.StateReadCacheStats.CodeStats.CacheHit, Misses: s.StateReadCacheStats.CodeStats.CacheMiss, - HitRate: calculateHitRate(s.StateReadCacheStats.CodeStats.CacheHit, s.StateReadCacheStats.CodeStats.CacheMiss), + HitRate: s.StateReadCacheStats.CodeStats.HitRate(), HitBytes: s.StateReadCacheStats.CodeStats.CacheHitBytes, MissBytes: s.StateReadCacheStats.CodeStats.CacheMissBytes, }, diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 13ce690518..ce592f0267 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -157,7 +157,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.statedb) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), state.NewDatabase(blockchain.triedb, blockchain.codedb)) if err != nil { return err } diff --git a/core/state/database.go b/core/state/database.go index 4a5547d075..002ce57fbc 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -20,13 +20,13 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/transitiontrie" @@ -34,14 +34,6 @@ import ( "github.com/ethereum/go-ethereum/triedb" ) -const ( - // Number of codehash->size associations to keep. - codeSizeCacheSize = 1_000_000 // 4 megabytes in total - - // Cache size granted for caching clean code. - codeCacheSize = 256 * 1024 * 1024 -) - // Database wraps access to tries and contract code. type Database interface { // Reader returns a state reader associated with the specified state root. @@ -58,6 +50,11 @@ type Database interface { // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree + + // Commit flushes all pending writes and finalizes the state transition, + // committing the changes to the underlying storage. It returns an error + // if the commit fails. + Commit(update *stateUpdate) error } // Trie is a Ethereum Merkle Patricia trie. @@ -149,32 +146,34 @@ type Trie interface { // state snapshot to provide functionalities for state access. It's meant to be a // long-live object and has a few caches inside for sharing between blocks. type CachingDB struct { - disk ethdb.KeyValueStore - triedb *triedb.Database - snap *snapshot.Tree - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] - - // Transition-specific fields - TransitionStatePerRoot *lru.Cache[common.Hash, *overlay.TransitionState] + triedb *triedb.Database + codedb *CodeDB + snap *snapshot.Tree } // NewDatabase creates a state database with the provided data sources. -func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { +func NewDatabase(triedb *triedb.Database, codedb *CodeDB) *CachingDB { + if codedb == nil { + codedb = NewCodeDB(triedb.Disk()) + } return &CachingDB{ - disk: triedb.Disk(), - triedb: triedb, - snap: snap, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - TransitionStatePerRoot: lru.NewCache[common.Hash, *overlay.TransitionState](1000), + triedb: triedb, + codedb: codedb, } } // NewDatabaseForTesting is similar to NewDatabase, but it initializes the caching // db by using an ephemeral memory db with default config for testing. func NewDatabaseForTesting() *CachingDB { - return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil) + db := rawdb.NewMemoryDatabase() + return NewDatabase(triedb.NewDatabase(db, nil), NewCodeDB(db)) +} + +// WithSnapshot configures the provided contract code cache. Note that this +// registration must be performed before the cachingDB is used. +func (db *CachingDB) WithSnapshot(snapshot *snapshot.Tree) *CachingDB { + db.snap = snapshot + return db } // StateReader returns a state reader associated with the specified state root. @@ -218,21 +217,20 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), sr), nil + return newReader(db.codedb.Reader(), sr), nil } // ReadersWithCacheStats creates a pair of state readers that share the same // underlying state reader and internal state cache, while maintaining separate // statistics respectively. -func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithStats, ReaderWithStats, error) { +func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (Reader, Reader, error) { r, err := db.StateReader(stateRoot) if err != nil { return nil, nil, err } sr := newStateReaderWithCache(r) - - ra := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache)) - rb := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache)) + ra := newReader(db.codedb.Reader(), newStateReaderWithStats(sr)) + rb := newReader(db.codedb.Reader(), newStateReaderWithStats(sr)) return ra, rb, nil } @@ -268,22 +266,6 @@ func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Addre return tr, nil } -// ContractCodeWithPrefix retrieves a particular contract's code. If the -// code can't be found in the cache, then check the existence with **new** -// db scheme. -func (db *CachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) []byte { - code, _ := db.codeCache.Get(codeHash) - if len(code) > 0 { - return code - } - code = rawdb.ReadCodeWithPrefix(db.disk, codeHash) - if len(code) > 0 { - db.codeCache.Add(codeHash, code) - db.codeSizeCache.Add(codeHash, len(code)) - } - return code -} - // TrieDB retrieves any intermediate trie-node caching layer. func (db *CachingDB) TrieDB() *triedb.Database { return db.triedb @@ -294,6 +276,40 @@ func (db *CachingDB) Snapshot() *snapshot.Tree { return db.snap } +// Commit flushes all pending writes and finalizes the state transition, +// committing the changes to the underlying storage. It returns an error +// if the commit fails. +func (db *CachingDB) Commit(update *stateUpdate) error { + // Short circuit if nothing to commit + if update.empty() { + return nil + } + // Commit dirty contract code if any exists + if len(update.codes) > 0 { + batch := db.codedb.NewBatchWithSize(len(update.codes)) + for _, code := range update.codes { + batch.Put(code.hash, code.blob) + } + if err := batch.Commit(); err != nil { + return err + } + } + // If snapshotting is enabled, update the snapshot tree with this new version + if db.snap != nil && db.snap.Snapshot(update.originRoot) != nil { + if err := db.snap.Update(update.root, update.originRoot, update.accounts, update.storages); err != nil { + log.Warn("Failed to update snapshot tree", "from", update.originRoot, "to", update.root, "err", err) + } + // Keep 128 diff layers in the memory, persistent layer is 129th. + // - head layer is paired with HEAD state + // - head-1 layer is paired with HEAD-1 state + // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state + if err := db.snap.Cap(update.root, TriesInMemory); err != nil { + log.Warn("Failed to cap snapshot tree", "root", update.root, "layers", TriesInMemory, "err", err) + } + } + return db.triedb.Update(update.root, update.originRoot, update.blockNumber, update.nodes, update.stateSet()) +} + // mustCopyTrie returns a deep-copied trie. func mustCopyTrie(t Trie) Trie { switch t := t.(type) { diff --git a/core/state/database_code.go b/core/state/database_code.go new file mode 100644 index 0000000000..820c9c1168 --- /dev/null +++ b/core/state/database_code.go @@ -0,0 +1,231 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +const ( + // Number of codeHash->size associations to keep. + codeSizeCacheSize = 1_000_000 + + // Cache size granted for caching clean code. + codeCacheSize = 256 * 1024 * 1024 +) + +// CodeCache maintains cached contract code that is shared across blocks, enabling +// fast access for external calls such as RPCs and state transitions. +// +// It is thread-safe and has a bounded size. +type codeCache struct { + codeCache *lru.SizeConstrainedCache[common.Hash, []byte] + codeSizeCache *lru.Cache[common.Hash, int] +} + +// newCodeCache initializes the contract code cache with the predefined capacity. +func newCodeCache() *codeCache { + return &codeCache{ + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + } +} + +// Get returns the contract code associated with the provided code hash. +func (c *codeCache) Get(hash common.Hash) ([]byte, bool) { + return c.codeCache.Get(hash) +} + +// GetSize returns the contract code size associated with the provided code hash. +func (c *codeCache) GetSize(hash common.Hash) (int, bool) { + return c.codeSizeCache.Get(hash) +} + +// Put adds the provided contract code along with its size information into the cache. +func (c *codeCache) Put(hash common.Hash, code []byte) { + c.codeCache.Add(hash, code) + c.codeSizeCache.Add(hash, len(code)) +} + +// CodeReader implements state.ContractCodeReader, accessing contract code either in +// local key-value store or the shared code cache. +// +// Reader is safe for concurrent access. +type CodeReader struct { + db ethdb.KeyValueReader + cache *codeCache + + // Cache statistics + hit atomic.Int64 // Number of code lookups found in the cache + miss atomic.Int64 // Number of code lookups not found in the cache + hitBytes atomic.Int64 // Total number of bytes read from cache + missBytes atomic.Int64 // Total number of bytes read from database +} + +// newCodeReader constructs the code reader with provided key value store and the cache. +func newCodeReader(db ethdb.KeyValueReader, cache *codeCache) *CodeReader { + return &CodeReader{ + db: db, + cache: cache, + } +} + +// Has returns the flag indicating whether the contract code with +// specified address and hash exists or not. +func (r *CodeReader) Has(addr common.Address, codeHash common.Hash) bool { + return len(r.Code(addr, codeHash)) > 0 +} + +// Code implements state.ContractCodeReader, retrieving a particular contract's code. +// Null is returned if the contract code is not present. +func (r *CodeReader) Code(addr common.Address, codeHash common.Hash) []byte { + code, _ := r.cache.Get(codeHash) + if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) + return code + } + r.miss.Add(1) + + code = rawdb.ReadCode(r.db, codeHash) + if len(code) > 0 { + r.cache.Put(codeHash, code) + r.missBytes.Add(int64(len(code))) + } + return code +} + +// CodeSize implements state.ContractCodeReader, retrieving a particular contract +// code's size. Zero is returned if the contract code is not present. +func (r *CodeReader) CodeSize(addr common.Address, codeHash common.Hash) int { + if cached, ok := r.cache.GetSize(codeHash); ok { + r.hit.Add(1) + return cached + } + return len(r.Code(addr, codeHash)) +} + +// CodeWithPrefix retrieves the contract code for the specified account address +// and code hash. It is almost identical to Code, but uses rawdb.ReadCodeWithPrefix +// for database lookups. The intention is to gradually deprecate the old +// contract code scheme. +func (r *CodeReader) CodeWithPrefix(addr common.Address, codeHash common.Hash) []byte { + code, _ := r.cache.Get(codeHash) + if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) + return code + } + r.miss.Add(1) + + code = rawdb.ReadCodeWithPrefix(r.db, codeHash) + if len(code) > 0 { + r.cache.Put(codeHash, code) + r.missBytes.Add(int64(len(code))) + } + return code +} + +// GetCodeStats implements ContractCodeReaderStater, returning the statistics +// of the code reader. +func (r *CodeReader) GetCodeStats() ContractCodeReaderStats { + return ContractCodeReaderStats{ + CacheHit: r.hit.Load(), + CacheMiss: r.miss.Load(), + CacheHitBytes: r.hitBytes.Load(), + CacheMissBytes: r.missBytes.Load(), + } +} + +type CodeBatch struct { + db *CodeDB + codes [][]byte + codeHashes []common.Hash +} + +// newCodeBatch constructs the batch for writing contract code. +func newCodeBatch(db *CodeDB) *CodeBatch { + return &CodeBatch{ + db: db, + } +} + +// newCodeBatchWithSize constructs the batch with a pre-allocated capacity. +func newCodeBatchWithSize(db *CodeDB, size int) *CodeBatch { + return &CodeBatch{ + db: db, + codes: make([][]byte, 0, size), + codeHashes: make([]common.Hash, 0, size), + } +} + +// Put inserts the given contract code into the writer, waiting for commit. +func (b *CodeBatch) Put(codeHash common.Hash, code []byte) { + b.codes = append(b.codes, code) + b.codeHashes = append(b.codeHashes, codeHash) +} + +// Commit flushes the accumulated dirty contract code into the database and +// also place them in the cache. +func (b *CodeBatch) Commit() error { + batch := b.db.db.NewBatch() + for i, code := range b.codes { + rawdb.WriteCode(batch, b.codeHashes[i], code) + b.db.cache.Put(b.codeHashes[i], code) + } + if err := batch.Write(); err != nil { + return err + } + b.codes = b.codes[:0] + b.codeHashes = b.codeHashes[:0] + return nil +} + +// CodeDB is responsible for managing the contract code and provides the access +// to it. It can be used as a global object, sharing it between multiple entities. +type CodeDB struct { + db ethdb.KeyValueStore + cache *codeCache +} + +// NewCodeDB constructs the contract code database with the provided key value store. +func NewCodeDB(db ethdb.KeyValueStore) *CodeDB { + return &CodeDB{ + db: db, + cache: newCodeCache(), + } +} + +// Reader returns the contract code reader. +func (d *CodeDB) Reader() *CodeReader { + return newCodeReader(d.db, d.cache) +} + +// NewBatch returns the batch for flushing contract codes. +func (d *CodeDB) NewBatch() *CodeBatch { + return newCodeBatch(d) +} + +// NewBatchWithSize returns the batch with pre-allocated capacity. +func (d *CodeDB) NewBatchWithSize(size int) *CodeBatch { + return newCodeBatchWithSize(d, size) +} diff --git a/core/state/database_history.go b/core/state/database_history.go index 7a2be8fe4f..c25c4eae4b 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -17,15 +17,14 @@ package state import ( + "errors" "fmt" "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" @@ -221,19 +220,15 @@ func (r *historicalTrieReader) Storage(addr common.Address, key common.Hash) (co // HistoricDB is the implementation of Database interface, with the ability to // access historical state. type HistoricDB struct { - disk ethdb.KeyValueStore - triedb *triedb.Database - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] + triedb *triedb.Database + codedb *CodeDB } // NewHistoricDatabase creates a historic state database. -func NewHistoricDatabase(disk ethdb.KeyValueStore, triedb *triedb.Database) *HistoricDB { +func NewHistoricDatabase(triedb *triedb.Database, codedb *CodeDB) *HistoricDB { return &HistoricDB{ - disk: disk, - triedb: triedb, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + triedb: triedb, + codedb: codedb, } } @@ -258,7 +253,7 @@ func (db *HistoricDB) Reader(stateRoot common.Hash) (Reader, error) { if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil + return newReader(db.codedb.Reader(), combined), nil } // OpenTrie opens the main account trie. It's not supported by historic database. @@ -298,3 +293,10 @@ func (db *HistoricDB) TrieDB() *triedb.Database { func (db *HistoricDB) Snapshot() *snapshot.Tree { return nil } + +// Commit flushes all pending writes and finalizes the state transition, +// committing the changes to the underlying storage. It returns an error +// if the commit fails. +func (db *HistoricDB) Commit(update *stateUpdate) error { + return errors.New("not implemented") +} diff --git a/core/state/iterator.go b/core/state/iterator.go index 0abae091d9..0050a840d8 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -144,10 +144,7 @@ func (it *nodeIterator) step() error { } if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { it.codeHash = common.BytesToHash(account.CodeHash) - it.code, err = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) - if err != nil { - return fmt.Errorf("code %x: %v", account.CodeHash, err) - } + it.code = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) if len(it.code) == 0 { return fmt.Errorf("code is not found: %x", account.CodeHash) } diff --git a/core/state/reader.go b/core/state/reader.go index 35b732173b..49375c467c 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -18,17 +18,13 @@ package state import ( "errors" - "fmt" "sync" "sync/atomic" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/overlay" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/bintrie" @@ -38,55 +34,26 @@ import ( ) // ContractCodeReader defines the interface for accessing contract code. +// +// ContractCodeReader is supposed to be thread-safe. type ContractCodeReader interface { // Has returns the flag indicating whether the contract code with // specified address and hash exists or not. Has(addr common.Address, codeHash common.Hash) bool - // Code retrieves a particular contract's code. - // - // - Returns nil code along with nil error if the requested contract code - // doesn't exist - // - Returns an error only if an unexpected issue occurs - Code(addr common.Address, codeHash common.Hash) ([]byte, error) + // Code retrieves a particular contract's code. Returns nil code if the + // requested contract code doesn't exist. + Code(addr common.Address, codeHash common.Hash) []byte - // CodeSize retrieves a particular contracts code's size. - // - // - Returns zero code size along with nil error if the requested contract code - // doesn't exist - // - Returns an error only if an unexpected issue occurs - CodeSize(addr common.Address, codeHash common.Hash) (int, error) -} - -// ContractCodeReaderStats aggregates statistics for the contract code reader. -type ContractCodeReaderStats struct { - CacheHit int64 // Number of cache hits - CacheMiss int64 // Number of cache misses - CacheHitBytes int64 // Total bytes served from cache - CacheMissBytes int64 // Total bytes read on cache misses -} - -// HitRate returns the cache hit rate. -func (s ContractCodeReaderStats) HitRate() float64 { - if s.CacheHit == 0 { - return 0 - } - return float64(s.CacheHit) / float64(s.CacheHit+s.CacheMiss) -} - -// ContractCodeReaderWithStats extends ContractCodeReader by adding GetStats to -// expose statistics of code reader. -type ContractCodeReaderWithStats interface { - ContractCodeReader - - GetStats() ContractCodeReaderStats + // CodeSize retrieves a particular contracts code's size. Returns zero code + // size if the requested contract code doesn't exist. + CodeSize(addr common.Address, codeHash common.Hash) int } // StateReader defines the interface for accessing accounts and storage slots // associated with a specific state. // -// StateReader is assumed to be thread-safe and implementation must take care -// of the concurrency issue by themselves. +// StateReader is supposed to be thread-safe. type StateReader interface { // Account retrieves the account associated with a particular address. // @@ -114,119 +81,6 @@ type Reader interface { StateReader } -// ReaderStats wraps the statistics of reader. -type ReaderStats struct { - AccountCacheHit int64 - AccountCacheMiss int64 - StorageCacheHit int64 - StorageCacheMiss int64 - CodeStats ContractCodeReaderStats -} - -// String implements fmt.Stringer, returning string format statistics. -func (s ReaderStats) String() string { - var ( - accountCacheHitRate float64 - storageCacheHitRate float64 - ) - if s.AccountCacheHit > 0 { - accountCacheHitRate = float64(s.AccountCacheHit) / float64(s.AccountCacheHit+s.AccountCacheMiss) * 100 - } - if s.StorageCacheHit > 0 { - storageCacheHitRate = float64(s.StorageCacheHit) / float64(s.StorageCacheHit+s.StorageCacheMiss) * 100 - } - msg := fmt.Sprintf("Reader statistics\n") - msg += fmt.Sprintf("account: hit: %d, miss: %d, rate: %.2f\n", s.AccountCacheHit, s.AccountCacheMiss, accountCacheHitRate) - msg += fmt.Sprintf("storage: hit: %d, miss: %d, rate: %.2f\n", s.StorageCacheHit, s.StorageCacheMiss, storageCacheHitRate) - msg += fmt.Sprintf("code: hit: %d(%v), miss: %d(%v), rate: %.2f\n", s.CodeStats.CacheHit, common.StorageSize(s.CodeStats.CacheHitBytes), s.CodeStats.CacheMiss, common.StorageSize(s.CodeStats.CacheMissBytes), s.CodeStats.HitRate()) - return msg -} - -// ReaderWithStats wraps the additional method to retrieve the reader statistics from. -type ReaderWithStats interface { - Reader - GetStats() ReaderStats -} - -// cachingCodeReader implements ContractCodeReader, accessing contract code either in -// local key-value store or the shared code cache. -// -// cachingCodeReader is safe for concurrent access. -type cachingCodeReader struct { - db ethdb.KeyValueReader - - // These caches could be shared by multiple code reader instances, - // they are natively thread-safe. - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] - - // Cache statistics - hit atomic.Int64 // Number of code lookups found in the cache - miss atomic.Int64 // Number of code lookups not found in the cache - hitBytes atomic.Int64 // Total number of bytes read from cache - missBytes atomic.Int64 // Total number of bytes read from database -} - -// newCachingCodeReader constructs the code reader. -func newCachingCodeReader(db ethdb.KeyValueReader, codeCache *lru.SizeConstrainedCache[common.Hash, []byte], codeSizeCache *lru.Cache[common.Hash, int]) *cachingCodeReader { - return &cachingCodeReader{ - db: db, - codeCache: codeCache, - codeSizeCache: codeSizeCache, - } -} - -// Code implements ContractCodeReader, retrieving a particular contract's code. -// If the contract code doesn't exist, no error will be returned. -func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]byte, error) { - code, _ := r.codeCache.Get(codeHash) - if len(code) > 0 { - r.hit.Add(1) - r.hitBytes.Add(int64(len(code))) - return code, nil - } - r.miss.Add(1) - - code = rawdb.ReadCode(r.db, codeHash) - if len(code) > 0 { - r.codeCache.Add(codeHash, code) - r.codeSizeCache.Add(codeHash, len(code)) - r.missBytes.Add(int64(len(code))) - } - return code, nil -} - -// CodeSize implements ContractCodeReader, retrieving a particular contracts code's size. -// If the contract code doesn't exist, no error will be returned. -func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) (int, error) { - if cached, ok := r.codeSizeCache.Get(codeHash); ok { - r.hit.Add(1) - return cached, nil - } - code, err := r.Code(addr, codeHash) - if err != nil { - return 0, err - } - return len(code), nil -} - -// Has returns the flag indicating whether the contract code with -// specified address and hash exists or not. -func (r *cachingCodeReader) Has(addr common.Address, codeHash common.Hash) bool { - code, _ := r.Code(addr, codeHash) - return len(code) > 0 -} - -// GetStats returns the statistics of the code reader. -func (r *cachingCodeReader) GetStats() ContractCodeReaderStats { - return ContractCodeReaderStats{ - CacheHit: r.hit.Load(), - CacheMiss: r.miss.Load(), - CacheHitBytes: r.hitBytes.Load(), - CacheMissBytes: r.missBytes.Load(), - } -} - // flatReader wraps a database state reader and is safe for concurrent access. type flatReader struct { reader database.StateReader @@ -495,20 +349,6 @@ func (r *multiStateReader) Storage(addr common.Address, slot common.Hash) (commo return common.Hash{}, errors.Join(errs...) } -// reader is the wrapper of ContractCodeReader and StateReader interface. -type reader struct { - ContractCodeReader - StateReader -} - -// newReader constructs a reader with the supplied code reader and state reader. -func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { - return &reader{ - ContractCodeReader: codeReader, - StateReader: stateReader, - } -} - // stateReaderWithCache is a wrapper around StateReader that maintains additional // state caches to support concurrent state access. type stateReaderWithCache struct { @@ -619,9 +459,10 @@ func (r *stateReaderWithCache) Storage(addr common.Address, slot common.Hash) (c return value, err } -type readerWithStats struct { +// stateReaderWithStats is a wrapper over the stateReaderWithCache, tracking +// the cache hit statistics of the reader. +type stateReaderWithStats struct { *stateReaderWithCache - ContractCodeReaderWithStats accountCacheHit atomic.Int64 accountCacheMiss atomic.Int64 @@ -629,11 +470,10 @@ type readerWithStats struct { storageCacheMiss atomic.Int64 } -// newReaderWithStats constructs the reader with additional statistics tracked. -func newReaderWithStats(sr *stateReaderWithCache, cr ContractCodeReaderWithStats) *readerWithStats { - return &readerWithStats{ - stateReaderWithCache: sr, - ContractCodeReaderWithStats: cr, +// newReaderWithStats constructs the state reader with additional statistics tracked. +func newStateReaderWithStats(sr *stateReaderWithCache) *stateReaderWithStats { + return &stateReaderWithStats{ + stateReaderWithCache: sr, } } @@ -641,7 +481,7 @@ func newReaderWithStats(sr *stateReaderWithCache, cr ContractCodeReaderWithStats // The returned account might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithStats) Account(addr common.Address) (*types.StateAccount, error) { +func (r *stateReaderWithStats) Account(addr common.Address) (*types.StateAccount, error) { account, incache, err := r.stateReaderWithCache.account(addr) if err != nil { return nil, err @@ -659,7 +499,7 @@ func (r *readerWithStats) Account(addr common.Address) (*types.StateAccount, err // existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { +func (r *stateReaderWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { value, incache, err := r.stateReaderWithCache.storage(addr, slot) if err != nil { return common.Hash{}, err @@ -672,13 +512,51 @@ func (r *readerWithStats) Storage(addr common.Address, slot common.Hash) (common return value, nil } -// GetStats implements ReaderWithStats, returning the statistics of state reader. -func (r *readerWithStats) GetStats() ReaderStats { - return ReaderStats{ +// GetStateStats implements StateReaderStater, returning the statistics of the +// state reader. +func (r *stateReaderWithStats) GetStateStats() StateReaderStats { + return StateReaderStats{ AccountCacheHit: r.accountCacheHit.Load(), AccountCacheMiss: r.accountCacheMiss.Load(), StorageCacheHit: r.storageCacheHit.Load(), StorageCacheMiss: r.storageCacheMiss.Load(), - CodeStats: r.ContractCodeReaderWithStats.GetStats(), + } +} + +// reader aggregates a code reader and a state reader into a single object. +type reader struct { + ContractCodeReader + StateReader +} + +// newReader constructs a reader with the supplied code reader and state reader. +func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { + return &reader{ + ContractCodeReader: codeReader, + StateReader: stateReader, + } +} + +// GetCodeStats returns the statistics of code access. +func (r *reader) GetCodeStats() ContractCodeReaderStats { + if stater, ok := r.ContractCodeReader.(ContractCodeReaderStater); ok { + return stater.GetCodeStats() + } + return ContractCodeReaderStats{} +} + +// GetStateStats returns the statistics of state access. +func (r *reader) GetStateStats() StateReaderStats { + if stater, ok := r.StateReader.(StateReaderStater); ok { + return stater.GetStateStats() + } + return StateReaderStats{} +} + +// GetStats returns the aggregated statistics for both state and code access. +func (r *reader) GetStats() ReaderStats { + return ReaderStats{ + CodeStats: r.GetCodeStats(), + StateStats: r.GetStateStats(), } } diff --git a/core/state/reader_stater.go b/core/state/reader_stater.go new file mode 100644 index 0000000000..5294275953 --- /dev/null +++ b/core/state/reader_stater.go @@ -0,0 +1,82 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +// ContractCodeReaderStats aggregates statistics for the contract code reader. +type ContractCodeReaderStats struct { + CacheHit int64 // Number of cache hits + CacheMiss int64 // Number of cache misses + CacheHitBytes int64 // Total bytes served from cache + CacheMissBytes int64 // Total bytes read on cache misses +} + +// HitRate returns the cache hit rate in percentage. +func (s ContractCodeReaderStats) HitRate() float64 { + total := s.CacheHit + s.CacheMiss + if total == 0 { + return 0 + } + return float64(s.CacheHit) / float64(total) * 100 +} + +// ContractCodeReaderStater wraps the method to retrieve the statistics of +// contract code reader. +type ContractCodeReaderStater interface { + GetCodeStats() ContractCodeReaderStats +} + +// StateReaderStats aggregates statistics for the state reader. +type StateReaderStats struct { + AccountCacheHit int64 // Number of account cache hits + AccountCacheMiss int64 // Number of account cache misses + StorageCacheHit int64 // Number of storage cache hits + StorageCacheMiss int64 // Number of storage cache misses +} + +// AccountCacheHitRate returns the cache hit rate of account requests in percentage. +func (s StateReaderStats) AccountCacheHitRate() float64 { + total := s.AccountCacheHit + s.AccountCacheMiss + if total == 0 { + return 0 + } + return float64(s.AccountCacheHit) / float64(total) * 100 +} + +// StorageCacheHitRate returns the cache hit rate of storage requests in percentage. +func (s StateReaderStats) StorageCacheHitRate() float64 { + total := s.StorageCacheHit + s.StorageCacheMiss + if total == 0 { + return 0 + } + return float64(s.StorageCacheHit) / float64(total) * 100 +} + +// StateReaderStater wraps the method to retrieve the statistics of state reader. +type StateReaderStater interface { + GetStateStats() StateReaderStats +} + +// ReaderStats wraps the statistics of reader. +type ReaderStats struct { + CodeStats ContractCodeReaderStats + StateStats StateReaderStats +} + +// ReaderStater defines the capability to retrieve aggregated statistics. +type ReaderStater interface { + GetStats() ReaderStats +} diff --git a/core/state/state_object.go b/core/state/state_object.go index f7109bddee..dd30bb64a5 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -564,10 +564,7 @@ func (s *stateObject) Code() []byte { s.db.CodeLoadBytes += len(s.code) }(time.Now()) - code, err := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) - if err != nil { - s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) - } + code := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) if len(code) == 0 { s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) } @@ -590,10 +587,7 @@ func (s *stateObject) CodeSize() int { s.db.CodeReads += time.Since(start) }(time.Now()) - size, err := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) - if err != nil { - s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) - } + size := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) if size == 0 { s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) } diff --git a/core/state/statedb.go b/core/state/statedb.go index bf38bdf09d..2477242eb5 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,7 +28,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" @@ -148,8 +147,7 @@ type StateDB struct { StorageReads time.Duration StorageUpdates time.Duration StorageCommits time.Duration - SnapshotCommits time.Duration - TrieDBCommits time.Duration + DatabaseCommits time.Duration CodeReads time.Duration AccountLoaded int // Number of accounts retrieved from the database during the state transition @@ -1333,42 +1331,14 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } } - // Commit dirty contract code if any exists - if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { - batch := db.NewBatch() - for _, code := range ret.codes { - rawdb.WriteCode(batch, code.hash, code.blob) - } - if err := batch.Write(); err != nil { - return nil, err - } - batch.Close() - } - if !ret.empty() { - // If snapshotting is enabled, update the snapshot tree with this new version - if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { - start := time.Now() - if err := snap.Update(ret.root, ret.originRoot, ret.accounts, ret.storages); err != nil { - log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) - } - // Keep 128 diff layers in the memory, persistent layer is 129th. - // - head layer is paired with HEAD state - // - head-1 layer is paired with HEAD-1 state - // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state - if err := snap.Cap(ret.root, TriesInMemory); err != nil { - log.Warn("Failed to cap snapshot tree", "root", ret.root, "layers", TriesInMemory, "err", err) - } - s.SnapshotCommits += time.Since(start) - } - // If trie database is enabled, commit the state update as a new layer - if db := s.db.TrieDB(); db != nil { - start := time.Now() - if err := db.Update(ret.root, ret.originRoot, block, ret.nodes, ret.stateSet()); err != nil { - return nil, err - } - s.TrieDBCommits += time.Since(start) - } + start := time.Now() + if err := s.db.Commit(ret); err != nil { + return nil, err } + s.DatabaseCommits = time.Since(start) + + // The reader update must be performed as the final step, otherwise, + // the new state would not be visible before db.commit. s.reader, _ = s.db.Reader(s.originalRoot) return ret, err } diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 8b6ac0ba64..3582185344 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -209,7 +209,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[len(roots)-1] } - state, err := New(root, NewDatabase(tdb, snaps)) + state, err := New(root, NewDatabase(tdb, nil).WithSnapshot(snaps)) if err != nil { panic(err) } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 661d17bb7b..8d1f93ca1b 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1276,7 +1276,7 @@ func TestDeleteStorage(t *testing.T) { disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, nil) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) - db = NewDatabase(tdb, snaps) + db = NewDatabase(tdb, nil).WithSnapshot(snaps) state, _ = New(types.EmptyRootHash, db) addr = common.HexToAddress("0x1") ) @@ -1290,7 +1290,7 @@ func TestDeleteStorage(t *testing.T) { } root, _ := state.Commit(0, true, false) // Init phase done, create two states, one with snap and one without - fastState, _ := New(root, NewDatabase(tdb, snaps)) + fastState, _ := New(root, NewDatabase(tdb, nil).WithSnapshot(snaps)) slowState, _ := New(root, NewDatabase(tdb, nil)) obj := fastState.getOrNewStateObject(addr) diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index 0c1b76b4f8..1c171cbd5e 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -211,9 +211,9 @@ func (sc *stateUpdate) deriveCodeFields(reader ContractCodeReader) error { cache := make(map[common.Hash]bool) for addr, code := range sc.codes { if code.originHash != types.EmptyCodeHash { - blob, err := reader.Code(addr, code.originHash) - if err != nil { - return err + blob := reader.Code(addr, code.originHash) + if len(blob) == 0 { + return fmt.Errorf("original code of %x is empty", addr) } code.originBlob = blob } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index cae0e0a936..e5e22deae5 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -222,8 +222,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s codeResults = make([]trie.CodeSyncResult, len(codeElements)) ) for i, element := range codeElements { - data, err := cReader.Code(common.Address{}, element.code) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, element.code) + if len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -346,8 +346,8 @@ func testIterativeDelayedStateSync(t *testing.T, scheme string) { if len(codeElements) > 0 { codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) for i, element := range codeElements[:len(codeResults)] { - data, err := cReader.Code(common.Address{}, element.code) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, element.code) + if len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -452,8 +452,8 @@ func testIterativeRandomStateSync(t *testing.T, count int, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -551,8 +551,8 @@ func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { for hash := range codeQueue { delete(codeQueue, hash) - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -671,8 +671,8 @@ func testIncompleteStateSync(t *testing.T, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) diff --git a/miner/miner_test.go b/miner/miner_test.go index 575ee4d0fd..13475a19b6 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -155,7 +155,7 @@ func createMiner(t *testing.T) *Miner { if err != nil { t.Fatalf("can't create new chain %v", err) } - statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache()) + statedb, _ := state.New(bc.Genesis().Root(), state.NewDatabase(bc.TrieDB(), bc.CodeDB())) blockchain := &testBlockChain{bc.Genesis().Root(), chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 3c7ee1c31d..1dd1bf6a04 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -544,7 +544,7 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bo } snaps, _ = snapshot.New(snapconfig, db, triedb, root) } - sdb = state.NewDatabase(triedb, snaps) + sdb = state.NewDatabase(triedb, nil).WithSnapshot(snaps) statedb, _ = state.New(root, sdb) return StateTestState{statedb, triedb, snaps} } diff --git a/triedb/hashdb/database.go b/triedb/hashdb/database.go index 38392aa519..90d0514290 100644 --- a/triedb/hashdb/database.go +++ b/triedb/hashdb/database.go @@ -612,6 +612,9 @@ func (db *Database) Close() error { // NodeReader returns a reader for accessing trie nodes within the specified state. // An error will be returned if the specified state is not available. func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) { + if root == types.EmptyRootHash { + return &reader{db: db}, nil + } if _, err := db.node(root); err != nil { return nil, fmt.Errorf("state %#x is not available, %v", root, err) } From aa417b03a6f9ef4f58c0e05c7eb1fde6a8db4894 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Tue, 10 Mar 2026 16:53:21 +0100 Subject: [PATCH 20/52] core/tracing: fix nonce revert edge case (#33978) We got a report for a bug in the tracing journal which has the responsibility to emit events for all state that must be reverted. The edge case is as follows: on CREATE operations the nonce is incremented. When a create frame reverts, the nonce increment associated with it does **not** revert. This works fine on master. Now one step further: if the parent frame reverts tho, the nonce **should** revert and there is the bug. --- core/tracing/journal.go | 16 ++++++++++++---- core/tracing/journal_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/core/tracing/journal.go b/core/tracing/journal.go index 62a70d6c27..560c937115 100644 --- a/core/tracing/journal.go +++ b/core/tracing/journal.go @@ -155,10 +155,18 @@ func (j *journal) OnBalanceChange(addr common.Address, prev, new *big.Int, reaso } func (j *journal) OnNonceChangeV2(addr common.Address, prev, new uint64, reason NonceChangeReason) { - // When a contract is created, the nonce of the creator is incremented. - // This change is not reverted when the creation fails. - if reason != NonceChangeContractCreator { - j.entries = append(j.entries, nonceChange{addr: addr, prev: prev, new: new}) + j.entries = append(j.entries, nonceChange{addr: addr, prev: prev, new: new}) + if reason == NonceChangeContractCreator { + // When a contract is created via CREATE/CREATE2, the creator's nonce is + // incremented. The EVM does not revert this when the CREATE frame itself + // fails (the nonce change happens before the EVM snapshot). However, if + // a parent frame reverts, the nonce must be reverted along with everything + // else. + // + // To achieve this, advance the current frame's revision point past this + // entry. The CREATE frame's revert won't touch it (it's below the revision), + // but a parent frame's revert will (it's above the parent's revision). + j.revisions[len(j.revisions)-1] = len(j.entries) } if j.hooks.OnNonceChangeV2 != nil { j.hooks.OnNonceChangeV2(addr, prev, new, reason) diff --git a/core/tracing/journal_test.go b/core/tracing/journal_test.go index e00447f5f3..488d192502 100644 --- a/core/tracing/journal_test.go +++ b/core/tracing/journal_test.go @@ -219,6 +219,42 @@ func TestNonceIncOnCreate(t *testing.T) { } } +// TestNonceIncOnCreateParentReverts checks that the creator's nonce increment +// from CREATE survives the CREATE frame's own revert but is properly reverted +// when the parent call frame reverts. +func TestNonceIncOnCreateParentReverts(t *testing.T) { + const opCREATE = 0xf0 + + tr := &testTracer{t: t} + wr, err := WrapWithJournal(&Hooks{OnNonceChange: tr.OnNonceChange}) + if err != nil { + t.Fatalf("failed to wrap test tracer: %v", err) + } + + addr := common.HexToAddress("0x1234") + { + // Parent call frame + wr.OnEnter(0, 0, addr, addr, nil, 1000, big.NewInt(0)) + { + // CREATE frame — creator nonce incremented, then CREATE reverts + wr.OnEnter(1, opCREATE, addr, addr, nil, 1000, big.NewInt(0)) + wr.OnNonceChangeV2(addr, 0, 1, NonceChangeContractCreator) + wr.OnExit(1, nil, 100, errors.New("revert"), true) + } + // After CREATE reverts, nonce should still be 1 + if tr.nonce != 1 { + t.Fatalf("nonce after CREATE revert: got %v, want 1", tr.nonce) + } + // Parent frame also reverts + wr.OnExit(0, nil, 150, errors.New("revert"), true) + } + + // After parent reverts, nonce should be back to 0 + if tr.nonce != 0 { + t.Fatalf("nonce after parent revert: got %v, want 0", tr.nonce) + } +} + func TestOnNonceChangeV2(t *testing.T) { tr := &testTracer{t: t} wr, err := WrapWithJournal(&Hooks{OnNonceChangeV2: tr.OnNonceChangeV2}) From 27c4ca9df0caf0a235585ea791850df40b0d3fa4 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 11 Mar 2026 11:23:00 +0800 Subject: [PATCH 21/52] eth: resolve finalized from disk if it's not recently announced (#33150) This PR contains two changes: Firstly, the finalized header will be resolved from local chain if it's not recently announced via the `engine_newPayload`. What's more importantly is, in the downloader, originally there are two code paths to push forward the pivot point block, one in the beacon header fetcher (`fetchHeaders`), and another one is in the snap content processer (`processSnapSyncContent`). Usually if there are new blocks and local pivot block becomes stale, it will firstly be detected by the `fetchHeaders`. `processSnapSyncContent` is fully driven by the beacon headers and will only detect the stale pivot block after synchronizing the corresponding chain segment. I think the detection here is redundant and useless. --- eth/catalyst/api.go | 9 ++++----- eth/downloader/downloader.go | 23 ----------------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index c64039b690..96d4570561 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -255,12 +255,9 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { return engine.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil } - // If the head hash is unknown (was not given to us in a newPayload request), - // we cannot resolve the header, so not much to do. This could be extended in - // the future to resolve from the `eth` network, but it's an unexpected case - // that should be fixed, not papered over. header := api.remoteBlocks.get(update.HeadBlockHash) if header == nil { + // The head hash is unknown locally, try to resolve it from the `eth` network log.Warn("Fetching the unknown forkchoice head from network", "hash", update.HeadBlockHash) retrievedHead, err := api.eth.Downloader().GetHeader(update.HeadBlockHash) if err != nil { @@ -273,7 +270,9 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // If the finalized hash is known, we can direct the downloader to move // potentially more data to the freezer from the get go. finalized := api.remoteBlocks.get(update.FinalizedBlockHash) - + if finalized == nil { + finalized = api.eth.BlockChain().GetHeaderByHash(update.FinalizedBlockHash) + } // Header advertised via a past newPayload request. Start syncing to it. context := []interface{}{"number", header.Number, "hash", header.Hash()} if update.FinalizedBlockHash != (common.Hash{}) { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index caeb3d64dd..1de0933842 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -959,29 +959,6 @@ func (d *Downloader) processSnapSyncContent() error { } else { // results already piled up, consume before handling pivot move results = append(append([]*fetchResult{oldPivot}, oldTail...), results...) } - // Split around the pivot block and process the two sides via snap/full sync - if !d.committed.Load() { - latest := results[len(results)-1].Header - // If the height is above the pivot block by 2 sets, it means the pivot - // became stale in the network, and it was garbage collected, move to a - // new pivot. - // - // Note, we have `reorgProtHeaderDelay` number of blocks withheld, Those - // need to be taken into account, otherwise we're detecting the pivot move - // late and will drop peers due to unavailable state!!! - if height := latest.Number.Uint64(); height >= pivot.Number.Uint64()+2*uint64(fsMinFullBlocks)-uint64(reorgProtHeaderDelay) { - log.Warn("Pivot became stale, moving", "old", pivot.Number.Uint64(), "new", height-uint64(fsMinFullBlocks)+uint64(reorgProtHeaderDelay)) - pivot = results[len(results)-1-fsMinFullBlocks+reorgProtHeaderDelay].Header // must exist as lower old pivot is uncommitted - - d.pivotLock.Lock() - d.pivotHeader = pivot - d.pivotLock.Unlock() - - // Write out the pivot into the database so a rollback beyond it will - // reenable snap sync - rawdb.WriteLastPivotNumber(d.stateDB, pivot.Number.Uint64()) - } - } P, beforeP, afterP := splitAroundPivot(pivot.Number.Uint64(), results) if err := d.commitSnapSyncData(beforeP, sync); err != nil { return err From f6068e3fb28d0dd90013131bf2ad39390bf807bb Mon Sep 17 00:00:00 2001 From: georgehao Date: Wed, 11 Mar 2026 11:46:49 +0800 Subject: [PATCH 22/52] eth/tracers: fix accessList StorageKeys return null (#33976) --- eth/tracers/logger/access_list_tracer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/eth/tracers/logger/access_list_tracer.go b/eth/tracers/logger/access_list_tracer.go index 2e51a9a907..749aade61b 100644 --- a/eth/tracers/logger/access_list_tracer.go +++ b/eth/tracers/logger/access_list_tracer.go @@ -85,11 +85,14 @@ func (al accessList) equal(other accessList) bool { func (al accessList) accessList() types.AccessList { acl := make(types.AccessList, 0, len(al)) for addr, slots := range al { - tuple := types.AccessTuple{Address: addr, StorageKeys: []common.Hash{}} - for slot := range slots { - tuple.StorageKeys = append(tuple.StorageKeys, slot) - } keys := slices.SortedFunc(maps.Keys(slots), common.Hash.Cmp) + // Ensure keys is never nil to avoid JSON serialization issues. + // When slots is empty, slices.SortedFunc returns nil, but JSON marshaling + // will serialize nil slice as null instead of [], which breaks clients + // that expect storageKeys to always be an array. + if keys == nil { + keys = []common.Hash{} + } acl = append(acl, types.AccessTuple{Address: addr, StorageKeys: keys}) } slices.SortFunc(acl, func(a, b types.AccessTuple) int { return a.Address.Cmp(b.Address) }) From 32f05d68a2ae09cf1ba023654a728a5cb73b8218 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Wed, 11 Mar 2026 02:41:43 -0400 Subject: [PATCH 23/52] core: end telemetry span for ApplyTransactionWithEVM if error is returned (#33955) --- core/state_processor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state_processor.go b/core/state_processor.go index 998f180571..85f106d58c 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -108,12 +108,12 @@ func (p *StateProcessor) Process(ctx context.Context, block *types.Block, stated receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, evm) if err != nil { + spanEnd(&err) return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) - - spanEnd(&err) + spanEnd(nil) } requests, err := postExecution(ctx, config, block, allLogs, evm) if err != nil { From 88f8549d37353fa9659134f7d9fe555395be7b87 Mon Sep 17 00:00:00 2001 From: bigbear <155267841+aso20455@users.noreply.github.com> Date: Wed, 11 Mar 2026 09:33:10 +0100 Subject: [PATCH 24/52] cmd/geth: correct misleading flag description in removedb command (#33984) The `--remove.chain` flag incorrectly described itself as selecting "state data" for removal, which could mislead operators into removing the wrong data category. This corrects the description to accurately reflect that the flag targets chain data (block bodies and receipts). --- cmd/geth/dbcmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 10b0c514ad..455dd05aca 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -53,7 +53,7 @@ var ( } removeChainDataFlag = &cli.BoolFlag{ Name: "remove.chain", - Usage: "If set, selects the state data for removal", + Usage: "If set, selects the chain data for removal", } inspectTrieTopFlag = &cli.IntFlag{ Name: "top", From 3c20e08cbae9bf370816d253c07831da534fb594 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Wed, 11 Mar 2026 12:47:42 +0100 Subject: [PATCH 25/52] cmd/geth: add Prague pruning points (#33657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR allows users to prune their nodes up to the Prague fork. It indirectly depends on #32157 and can't really be merged before eraE files are widely available for download. The `--history.chain` flag becomes mandatory for `prune-history` command. Here I've listed all the edge cases that can happen and how we behave: ## prune-history Behavior | From | To | Result | |-------------|--------------|--------------------------| | full | postmerge | ✅ prunes | | full | postprague | ✅ prunes | | postmerge | postprague | ✅ prunes further | | postprague | postmerge | ❌ can't unprune | | any | all | ❌ use import-history | ## Node Startup Behavior | DB State | Flag | Result | |-------------|--------------|----------------------------------------------------------------| | fresh | postprague | ✅ syncs from Prague | | full | postprague | ❌ "run prune-history first" | | postmerge | postprague | ❌ "run prune-history first" | | postprague | postmerge | ❌ "can't unprune, use import-history or fix flag" | | pruned | all | ✅ accepts known prune points | --- cmd/geth/chaincmd.go | 79 ++++++++++++++++++++++++++----------- cmd/utils/flags.go | 2 +- core/blockchain.go | 72 ++++++++++++++++++++++++--------- core/history/historymode.go | 50 ++++++++++++++++++++--- 4 files changed, 156 insertions(+), 47 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index f4e15afebe..7e14ec1c60 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -208,13 +208,19 @@ This command dumps out the state for a given block (or latest, if none provided) pruneHistoryCommand = &cli.Command{ Action: pruneHistory, Name: "prune-history", - Usage: "Prune blockchain history (block bodies and receipts) up to the merge block", + Usage: "Prune blockchain history (block bodies and receipts) up to a specified point", ArgsUsage: "", - Flags: utils.DatabaseFlags, + Flags: slices.Concat(utils.DatabaseFlags, []cli.Flag{ + utils.ChainHistoryFlag, + }), Description: ` The prune-history command removes historical block bodies and receipts from the -blockchain database up to the merge block, while preserving block headers. This -helps reduce storage requirements for nodes that don't need full historical data.`, +blockchain database up to a specified point, while preserving block headers. This +helps reduce storage requirements for nodes that don't need full historical data. + +The --history.chain flag is required to specify the pruning target: + - postmerge: Prune up to the merge block. The node will keep the merge block and everything thereafter. + - postprague: Prune up to the Prague (Pectra) upgrade block. The node will keep the prague block and everything thereafter.`, } downloadEraCommand = &cli.Command{ @@ -703,47 +709,74 @@ func hashish(x string) bool { } func pruneHistory(ctx *cli.Context) error { + // Parse and validate the history mode flag. + if !ctx.IsSet(utils.ChainHistoryFlag.Name) { + return errors.New("--history.chain flag is required") + } + var mode history.HistoryMode + if err := mode.UnmarshalText([]byte(ctx.String(utils.ChainHistoryFlag.Name))); err != nil { + return err + } + if mode == history.KeepAll { + return errors.New("--history.chain=all is not valid for pruning. To restore history, use 'geth import-history'") + } + stack, _ := makeConfigNode(ctx) defer stack.Close() - // Open the chain database + // Open the chain database. chain, chaindb := utils.MakeChain(ctx, stack, false) defer chaindb.Close() defer chain.Stop() - // Determine the prune point. This will be the first PoS block. - prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()] - if !ok || prunePoint == nil { - return errors.New("prune point not found") + // Determine the prune point based on the history mode. + genesisHash := chain.Genesis().Hash() + prunePoint := history.GetPrunePoint(genesisHash, mode) + if prunePoint == nil { + return fmt.Errorf("prune point for %q not found for this network", mode.String()) } var ( - mergeBlock = prunePoint.BlockNumber - mergeBlockHash = prunePoint.BlockHash.Hex() + targetBlock = prunePoint.BlockNumber + targetBlockHash = prunePoint.BlockHash ) - // Check we're far enough past merge to ensure all data is in freezer + // Check the current freezer tail to see if pruning is needed/possible. + freezerTail, _ := chaindb.Tail() + if freezerTail > 0 { + if freezerTail == targetBlock { + log.Info("Database already pruned to target block", "tail", freezerTail) + return nil + } + if freezerTail > targetBlock { + // Database is pruned beyond the target - can't unprune. + return fmt.Errorf("database is already pruned to block %d, which is beyond target %d. Cannot unprune. To restore history, use 'geth import-history'", freezerTail, targetBlock) + } + // freezerTail < targetBlock: we can prune further, continue below. + } + + // Check we're far enough past the target to ensure all data is in freezer. currentHeader := chain.CurrentHeader() if currentHeader == nil { return errors.New("current header not found") } - if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold { - return fmt.Errorf("chain not far enough past merge block, need %d more blocks", - mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) + if currentHeader.Number.Uint64() < targetBlock+params.FullImmutabilityThreshold { + return fmt.Errorf("chain not far enough past target block %d, need %d more blocks", + targetBlock, targetBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) } - // Double-check the prune block in db has the expected hash. - hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock) - if hash != common.HexToHash(mergeBlockHash) { - return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash) + // Double-check the target block in db has the expected hash. + hash := rawdb.ReadCanonicalHash(chaindb, targetBlock) + if hash != targetBlockHash { + return fmt.Errorf("target block hash mismatch: got %s, want %s", hash.Hex(), targetBlockHash.Hex()) } - log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash) + log.Info("Starting history pruning", "head", currentHeader.Number, "target", targetBlock, "targetHash", targetBlockHash.Hex()) start := time.Now() - rawdb.PruneTransactionIndex(chaindb, mergeBlock) - if _, err := chaindb.TruncateTail(mergeBlock); err != nil { + rawdb.PruneTransactionIndex(chaindb, targetBlock) + if _, err := chaindb.TruncateTail(targetBlock); err != nil { return fmt.Errorf("failed to truncate ancient data: %v", err) } - log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("History pruning completed", "tail", targetBlock, "elapsed", common.PrettyDuration(time.Since(start))) // TODO(s1na): what if there is a crash between the two prune operations? diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d5d2bfbf1c..792e0e55ab 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -319,7 +319,7 @@ var ( } ChainHistoryFlag = &cli.StringFlag{ Name: "history.chain", - Usage: `Blockchain history retention ("all" or "postmerge")`, + Usage: `Blockchain history retention ("all", "postmerge", or "postprague")`, Value: ethconfig.Defaults.HistoryMode.String(), Category: flags.StateCategory, } diff --git a/core/blockchain.go b/core/blockchain.go index 126ff1f666..8df2365072 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -715,8 +715,12 @@ func (bc *BlockChain) loadLastState() error { // initializeHistoryPruning sets bc.historyPrunePoint. func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { - freezerTail, _ := bc.db.Tail() - + var ( + freezerTail, _ = bc.db.Tail() + genesisHash = bc.genesisBlock.Hash() + mergePoint = history.MergePrunePoints[genesisHash] + praguePoint = history.PraguePrunePoints[genesisHash] + ) switch bc.cfg.ChainHistoryMode { case history.KeepAll: if freezerTail == 0 { @@ -724,33 +728,65 @@ func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { } // The database was pruned somehow, so we need to figure out if it's a known // configuration or an error. - predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] - if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber { - log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) - return errors.New("unexpected database tail") + if mergePoint != nil && freezerTail == mergePoint.BlockNumber { + bc.historyPrunePoint.Store(mergePoint) + return nil } - bc.historyPrunePoint.Store(predefinedPoint) - return nil + if praguePoint != nil && freezerTail == praguePoint.BlockNumber { + bc.historyPrunePoint.Store(praguePoint) + return nil + } + log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) + return errors.New("unexpected database tail") case history.KeepPostMerge: + if mergePoint == nil { + return errors.New("history pruning requested for unknown network") + } if freezerTail == 0 && latest != 0 { - // This is the case where a user is trying to run with --history.chain - // postmerge directly on an existing DB. We could just trigger the pruning - // here, but it'd be a bit dangerous since they may not have intended this - // action to happen. So just tell them how to do it. log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) - log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history.")) + log.Error("Run 'geth prune-history --history.chain postmerge' to prune pre-merge history.") return errors.New("history pruning requested via configuration") } - predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] - if predefinedPoint == nil { - log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) + // Check if DB is pruned further than requested (to Prague). + if praguePoint != nil && freezerTail == praguePoint.BlockNumber { + log.Error("Chain history database is pruned to Prague block, but postmerge mode was requested.") + log.Error("History cannot be unpruned. To restore history, use 'geth import-history'.") + log.Error("If you intended to keep post-Prague history, use '--history.chain postprague' instead.") + return errors.New("database pruned beyond requested history mode") + } + if freezerTail > 0 && freezerTail != mergePoint.BlockNumber { + return errors.New("chain history database pruned to unknown block") + } + bc.historyPrunePoint.Store(mergePoint) + return nil + + case history.KeepPostPrague: + if praguePoint == nil { return errors.New("history pruning requested for unknown network") - } else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber { + } + // Check if already at the prague prune point. + if freezerTail == praguePoint.BlockNumber { + bc.historyPrunePoint.Store(praguePoint) + return nil + } + // Check if database needs pruning. + if latest != 0 { + if freezerTail == 0 { + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) + log.Error("Run 'geth prune-history --history.chain postprague' to prune pre-Prague history.") + return errors.New("history pruning requested via configuration") + } + if mergePoint != nil && freezerTail == mergePoint.BlockNumber { + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is only pruned to merge block.", bc.cfg.ChainHistoryMode.String())) + log.Error("Run 'geth prune-history --history.chain postprague' to prune pre-Prague history.") + return errors.New("history pruning requested via configuration") + } log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) return errors.New("unexpected database tail") } - bc.historyPrunePoint.Store(predefinedPoint) + // Fresh database (latest == 0), will sync from prague point. + bc.historyPrunePoint.Store(praguePoint) return nil default: diff --git a/core/history/historymode.go b/core/history/historymode.go index e735222d37..bdaf07826d 100644 --- a/core/history/historymode.go +++ b/core/history/historymode.go @@ -32,10 +32,13 @@ const ( // KeepPostMerge sets the history pruning point to the merge activation block. KeepPostMerge + + // KeepPostPrague sets the history pruning point to the Prague (Pectra) activation block. + KeepPostPrague ) func (m HistoryMode) IsValid() bool { - return m <= KeepPostMerge + return m <= KeepPostPrague } func (m HistoryMode) String() string { @@ -44,6 +47,8 @@ func (m HistoryMode) String() string { return "all" case KeepPostMerge: return "postmerge" + case KeepPostPrague: + return "postprague" default: return fmt.Sprintf("invalid HistoryMode(%d)", m) } @@ -64,8 +69,10 @@ func (m *HistoryMode) UnmarshalText(text []byte) error { *m = KeepAll case "postmerge": *m = KeepPostMerge + case "postprague": + *m = KeepPostPrague default: - return fmt.Errorf(`unknown sync mode %q, want "all" or "postmerge"`, text) + return fmt.Errorf(`unknown history mode %q, want "all", "postmerge", or "postprague"`, text) } return nil } @@ -75,10 +82,10 @@ type PrunePoint struct { BlockHash common.Hash } -// PrunePoints the pre-defined history pruning cutoff blocks for known networks. +// MergePrunePoints contains the pre-defined history pruning cutoff blocks for known networks. // They point to the first post-merge block. Any pruning should truncate *up to* but excluding -// given block. -var PrunePoints = map[common.Hash]*PrunePoint{ +// the given block. +var MergePrunePoints = map[common.Hash]*PrunePoint{ // mainnet params.MainnetGenesisHash: { BlockNumber: 15537393, @@ -91,6 +98,39 @@ var PrunePoints = map[common.Hash]*PrunePoint{ }, } +// PraguePrunePoints contains the pre-defined history pruning cutoff blocks for the Prague +// (Pectra) upgrade. They point to the first post-Prague block. Any pruning should truncate +// *up to* but excluding the given block. +var PraguePrunePoints = map[common.Hash]*PrunePoint{ + // mainnet - first Prague block (May 7, 2025) + params.MainnetGenesisHash: { + BlockNumber: 22431084, + BlockHash: common.HexToHash("0x50c8cab760b2948349c590461b166773c45d8f4858cccf5a43025ab2960152e8"), + }, + // sepolia - first Prague block (March 5, 2025) + params.SepoliaGenesisHash: { + BlockNumber: 7836331, + BlockHash: common.HexToHash("0xe6571beb68bf24dbd8a6ba354518996920c55a3f8d8fdca423e391b8ad071f22"), + }, +} + +// PrunePoints is an alias for MergePrunePoints for backward compatibility. +// Deprecated: Use GetPrunePoint or MergePrunePoints directly. +var PrunePoints = MergePrunePoints + +// GetPrunePoint returns the prune point for the given genesis hash and history mode. +// Returns nil if no prune point is defined for the given combination. +func GetPrunePoint(genesisHash common.Hash, mode HistoryMode) *PrunePoint { + switch mode { + case KeepPostMerge: + return MergePrunePoints[genesisHash] + case KeepPostPrague: + return PraguePrunePoints[genesisHash] + default: + return nil + } +} + // PrunedHistoryError is returned by APIs when the requested history is pruned. type PrunedHistoryError struct{} From 59512b1849f700cc48e2fdaa264b247b07ca9300 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:18:42 +0100 Subject: [PATCH 26/52] cmd/fetchpayload: add payload-building utility (#33919) This PR adds a cmd tool fetchpayload which connects to a node and gets all the information in order to create a serialized payload that can then be passed to the zkvm. --- cmd/fetchpayload/main.go | 177 +++++++++++++++++++++++++++++++++++++ core/stateless/encoding.go | 6 +- 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 cmd/fetchpayload/main.go diff --git a/cmd/fetchpayload/main.go b/cmd/fetchpayload/main.go new file mode 100644 index 0000000000..eafc05fbe8 --- /dev/null +++ b/cmd/fetchpayload/main.go @@ -0,0 +1,177 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// fetchpayload queries an Ethereum node over RPC, fetches a block and its +// execution witness, and writes the combined Payload (ChainID + Block + +// Witness) to disk in the format consumed by cmd/keeper. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "math/big" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/stateless" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" +) + +// Payload is duplicated from cmd/keeper/main.go (package main, not importable). +type Payload struct { + ChainID uint64 + Block *types.Block + Witness *stateless.Witness +} + +func main() { + var ( + rpcURL = flag.String("rpc", "http://localhost:8545", "RPC endpoint URL") + blockArg = flag.String("block", "latest", `Block number: decimal, 0x-hex, or "latest"`) + format = flag.String("format", "rlp", "Comma-separated output formats: rlp, hex, json") + outDir = flag.String("out", "", "Output directory (default: current directory)") + ) + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Parse block number (nil means "latest" in ethclient). + blockNum, err := parseBlockNumber(*blockArg) + if err != nil { + fatal("invalid block number %q: %v", *blockArg, err) + } + + // Connect to the node. + client, err := ethclient.DialContext(ctx, *rpcURL) + if err != nil { + fatal("failed to connect to %s: %v", *rpcURL, err) + } + defer client.Close() + + chainID, err := client.ChainID(ctx) + if err != nil { + fatal("failed to get chain ID: %v", err) + } + + // Fetch the block first so we have a concrete number for the witness call, + // avoiding a race where "latest" advances between the two RPCs. + block, err := client.BlockByNumber(ctx, blockNum) + if err != nil { + fatal("failed to fetch block: %v", err) + } + fmt.Printf("Fetched block %d (%#x)\n", block.NumberU64(), block.Hash()) + + // Fetch the execution witness via the debug namespace. + var extWitness stateless.ExtWitness + err = client.Client().CallContext(ctx, &extWitness, "debug_executionWitness", rpc.BlockNumber(block.NumberU64())) + if err != nil { + fatal("failed to fetch execution witness: %v", err) + } + + witness := new(stateless.Witness) + err = witness.FromExtWitness(&extWitness) + if err != nil { + fatal("failed to convert witness: %v", err) + } + + payload := Payload{ + ChainID: chainID.Uint64(), + Block: block, + Witness: witness, + } + + // Encode payload as RLP (shared by "rlp" and "hex" formats). + rlpBytes, err := rlp.EncodeToBytes(payload) + if err != nil { + fatal("failed to RLP-encode payload: %v", err) + } + + // Write one output file per requested format. + blockHex := fmt.Sprintf("%x", block.NumberU64()) + for f := range strings.SplitSeq(*format, ",") { + f = strings.TrimSpace(f) + outPath := filepath.Join(*outDir, fmt.Sprintf("%s_payload.%s", blockHex, f)) + + var data []byte + switch f { + case "rlp": + data = rlpBytes + case "hex": + data = []byte(hexutil.Encode(rlpBytes)) + case "json": + data, err = marshalJSONPayload(chainID, block, &extWitness) + if err != nil { + fatal("failed to JSON-encode payload: %v", err) + } + default: + fatal("unknown format %q (valid: rlp, hex, json)", f) + } + + if err := os.WriteFile(outPath, data, 0644); err != nil { + fatal("failed to write %s: %v", outPath, err) + } + fmt.Printf("Wrote %s (%d bytes)\n", outPath, len(data)) + } +} + +// parseBlockNumber converts a CLI string to *big.Int. +// Returns nil for "latest" (ethclient convention for the head block). +func parseBlockNumber(s string) (*big.Int, error) { + if strings.EqualFold(s, "latest") { + return nil, nil + } + n := new(big.Int) + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + if _, ok := n.SetString(s[2:], 16); !ok { + return nil, fmt.Errorf("invalid hex number") + } + return n, nil + } + if _, ok := n.SetString(s, 10); !ok { + return nil, fmt.Errorf("invalid decimal number") + } + return n, nil +} + +// jsonPayload is a JSON-friendly representation of Payload. It uses ExtWitness +// instead of the internal Witness (which has no JSON marshaling). +type jsonPayload struct { + ChainID uint64 `json:"chainId"` + Block *types.Block `json:"block"` + Witness *stateless.ExtWitness `json:"witness"` +} + +func marshalJSONPayload(chainID *big.Int, block *types.Block, ext *stateless.ExtWitness) ([]byte, error) { + return json.MarshalIndent(jsonPayload{ + ChainID: chainID.Uint64(), + Block: block, + Witness: ext, + }, "", " ") +} + +func fatal(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/core/stateless/encoding.go b/core/stateless/encoding.go index 5c43159e66..d559178892 100644 --- a/core/stateless/encoding.go +++ b/core/stateless/encoding.go @@ -40,8 +40,8 @@ func (w *Witness) ToExtWitness() *ExtWitness { return ext } -// fromExtWitness converts the consensus witness format into our internal one. -func (w *Witness) fromExtWitness(ext *ExtWitness) error { +// FromExtWitness converts the consensus witness format into our internal one. +func (w *Witness) FromExtWitness(ext *ExtWitness) error { w.Headers = ext.Headers w.Codes = make(map[string]struct{}, len(ext.Codes)) @@ -66,7 +66,7 @@ func (w *Witness) DecodeRLP(s *rlp.Stream) error { if err := s.Decode(&ext); err != nil { return err } - return w.fromExtWitness(&ext) + return w.FromExtWitness(&ext) } // ExtWitness is a witness RLP encoding for transferring across clients. From 7d13acd030e27b504db5ca549580bf24f8256ca4 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 12 Mar 2026 09:21:54 +0800 Subject: [PATCH 27/52] core/rawdb, triedb/pathdb: enable trienode history alongside existing data (#33934) Fixes https://github.com/ethereum/go-ethereum/issues/33907 Notably there is a behavioral change: - Previously Geth will refuse to restart if the existing trienode history is gapped with the state data - With this PR, the gapped trienode history will be entirely reset and being constructed from scratch --- core/rawdb/ancienttest/testsuite.go | 40 ++++++++++++++ core/rawdb/freezer.go | 25 +++++---- core/rawdb/freezer_memory.go | 15 +++++- core/rawdb/freezer_table.go | 84 ++++++++++++++++++++++------- core/rawdb/freezer_table_test.go | 60 +++++++++++++++++++-- core/rawdb/freezer_utils.go | 52 +++++++++++++++++- triedb/pathdb/history.go | 34 +++++++----- triedb/pathdb/reader.go | 2 +- 8 files changed, 261 insertions(+), 51 deletions(-) diff --git a/core/rawdb/ancienttest/testsuite.go b/core/rawdb/ancienttest/testsuite.go index 7512c1f44b..eb66645a3a 100644 --- a/core/rawdb/ancienttest/testsuite.go +++ b/core/rawdb/ancienttest/testsuite.go @@ -260,6 +260,46 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) { if err != nil { t.Fatalf("Failed to write ancient data %v", err) } + + // Write should work after truncating from tail but over the head + db.TruncateTail(200) + head, err := db.Ancients() + if err != nil { + t.Fatalf("Failed to retrieve head ancients %v", err) + } + tail, err := db.Tail() + if err != nil { + t.Fatalf("Failed to retrieve tail ancients %v", err) + } + if head != 200 || tail != 200 { + t.Fatalf("Ancient head and tail are not expected") + } + _, err = db.ModifyAncients(func(op ethdb.AncientWriteOp) error { + offset := uint64(200) + for i := 0; i < 100; i++ { + if err := op.AppendRaw("a", offset+uint64(i), dataA[i]); err != nil { + return err + } + if err := op.AppendRaw("b", offset+uint64(i), dataB[i]); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("Failed to write ancient data %v", err) + } + head, err = db.Ancients() + if err != nil { + t.Fatalf("Failed to retrieve head ancients %v", err) + } + tail, err = db.Tail() + if err != nil { + t.Fatalf("Failed to retrieve tail ancients %v", err) + } + if head != 300 || tail != 200 { + t.Fatalf("Ancient head and tail are not expected") + } } func nonMutable(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) { diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 42cd2a7999..0e2f86d6ed 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -59,7 +59,7 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000 // - The in-order data ensures that disk reads are always optimized. type Freezer struct { datadir string - frozen atomic.Uint64 // Number of items already frozen + head atomic.Uint64 // Number of items stored (including items removed from tail) tail atomic.Uint64 // Number of the first stored item in the freezer // This lock synchronizes writers and the truncate operation, as well as @@ -97,12 +97,12 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui return nil, errSymlinkDatadir } } + // Leveldb/Pebble uses LOCK as the filelock filename. To prevent the + // name collision, we use FLOCK as the lock name. flockFile := filepath.Join(datadir, "FLOCK") if err := os.MkdirAll(filepath.Dir(flockFile), 0755); err != nil { return nil, err } - // Leveldb uses LOCK as the filelock filename. To prevent the - // name collision, we use FLOCK as the lock name. lock := flock.New(flockFile) tryLock := lock.TryLock if readonly { @@ -213,7 +213,7 @@ func (f *Freezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, // Ancients returns the length of the frozen items. func (f *Freezer) Ancients() (uint64, error) { - return f.frozen.Load(), nil + return f.head.Load(), nil } // Tail returns the number of first stored item in the freezer. @@ -252,7 +252,7 @@ func (f *Freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize defer f.writeLock.Unlock() // Roll back all tables to the starting position in case of error. - prevItem := f.frozen.Load() + prevItem := f.head.Load() defer func() { if err != nil { // The write operation has failed. Go back to the previous item position. @@ -273,7 +273,7 @@ func (f *Freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize if err != nil { return 0, err } - f.frozen.Store(item) + f.head.Store(item) return writeSize, nil } @@ -286,7 +286,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { f.writeLock.Lock() defer f.writeLock.Unlock() - oitems := f.frozen.Load() + oitems := f.head.Load() if oitems <= items { return oitems, nil } @@ -295,7 +295,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { return 0, err } } - f.frozen.Store(items) + f.head.Store(items) return oitems, nil } @@ -320,6 +320,11 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) { } } f.tail.Store(tail) + + // Update the head if the requested tail exceeds the current head + if f.head.Load() < tail { + f.head.Store(tail) + } return old, nil } @@ -379,7 +384,7 @@ func (f *Freezer) validate() error { prunedTail = &tmp } - f.frozen.Store(head) + f.head.Store(head) f.tail.Store(*prunedTail) return nil } @@ -414,7 +419,7 @@ func (f *Freezer) repair() error { } } - f.frozen.Store(head) + f.head.Store(head) f.tail.Store(prunedTail) return nil } diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index a0d308f896..ec6d4b22e2 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -113,7 +113,7 @@ func (t *memoryTable) truncateTail(items uint64) error { return nil } if t.items < items { - return errors.New("truncation above head") + return t.reset(items) } for i := uint64(0); i < items-t.offset; i++ { if t.size > uint64(len(t.data[i])) { @@ -127,6 +127,16 @@ func (t *memoryTable) truncateTail(items uint64) error { return nil } +// reset clears the entire table and sets both the head and tail to the given +// value. It assumes the caller holds the lock and that tail > t.items. +func (t *memoryTable) reset(offset uint64) error { + t.size = 0 + t.data = nil + t.items = offset + t.offset = offset + return nil +} + // commit merges the given item batch into table. It's presumed that the // batch is ordered and continuous with table. func (t *memoryTable) commit(batch [][]byte) error { @@ -387,6 +397,9 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) { } } f.tail = tail + if f.items < tail { + f.items = tail + } return old, nil } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index aedb2d8eed..280f6e1aaa 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -707,12 +707,13 @@ func (t *freezerTable) truncateTail(items uint64) error { t.lock.Lock() defer t.lock.Unlock() - // Ensure the given truncate target falls in the correct range + // Short-circuit if the requested tail deletion points to a stale position if t.itemHidden.Load() >= items { return nil } + // If the requested tail exceeds the current head, reset the entire table if t.items.Load() < items { - return errors.New("truncation above head") + return t.resetTo(items) } // Load the new tail index by the given new tail position var ( @@ -822,10 +823,9 @@ func (t *freezerTable) truncateTail(items uint64) error { shorten := indexEntrySize * int64(newDeleted-deleted) if t.metadata.flushOffset <= shorten { return fmt.Errorf("invalid index flush offset: %d, shorten: %d", t.metadata.flushOffset, shorten) - } else { - if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { - return err - } + } + if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { + return err } // Retrieve the new size and update the total size counter newSize, err := t.sizeNolock() @@ -836,6 +836,59 @@ func (t *freezerTable) truncateTail(items uint64) error { return nil } +// resetTo clears the entire table and sets both the head and tail to the given +// value. It assumes the caller holds the lock and that tail > t.items. +func (t *freezerTable) resetTo(tail uint64) error { + // Sync the entire table before resetting, eliminating the potential + // data corruption. + err := t.doSync() + if err != nil { + return err + } + // Update the index file to reflect the new offset + if err := t.index.Close(); err != nil { + return err + } + entry := &indexEntry{ + filenum: t.headId + 1, + offset: uint32(tail), + } + if err := reset(t.index.Name(), entry.append(nil)); err != nil { + return err + } + if err := t.metadata.setVirtualTail(tail, true); err != nil { + return err + } + if err := t.metadata.setFlushOffset(indexEntrySize, true); err != nil { + return err + } + t.index, err = openFreezerFileForAppend(t.index.Name()) + if err != nil { + return err + } + + // Purge all the existing data file + if err := t.head.Close(); err != nil { + return err + } + t.headId = t.headId + 1 + t.tailId = t.headId + t.headBytes = 0 + + t.head, err = t.openFile(t.headId, openFreezerFileTruncated) + if err != nil { + return err + } + t.releaseFilesBefore(t.headId, true) + + t.items.Store(tail) + t.itemOffset.Store(tail) + t.itemHidden.Store(tail) + t.sizeGauge.Update(0) + + return nil +} + // Close closes all opened files and finalizes the freezer table for use. // This operation must be completed before shutdown to prevent the loss of // recent writes. @@ -1247,25 +1300,20 @@ func (t *freezerTable) doSync() error { if t.index == nil || t.head == nil || t.metadata.file == nil { return errClosed } - var err error - trackError := func(e error) { - if e != nil && err == nil { - err = e - } + if err := t.index.Sync(); err != nil { + return err + } + if err := t.head.Sync(); err != nil { + return err } - trackError(t.index.Sync()) - trackError(t.head.Sync()) - // A crash may occur before the offset is updated, leaving the offset - // points to a old position. If so, the extra items above the offset + // points to an old position. If so, the extra items above the offset // will be truncated during the next run. stat, err := t.index.Stat() if err != nil { return err } - offset := stat.Size() - trackError(t.metadata.setFlushOffset(offset, true)) - return err + return t.metadata.setFlushOffset(stat.Size(), true) } func (t *freezerTable) dumpIndexStdout(start, stop int64) { diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index fc21ea6c63..3393f88e1a 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -1139,6 +1139,7 @@ const ( opTruncateHeadAll opTruncateTail opTruncateTailAll + opTruncateTailOverHead opCheckAll opMax // boundary value, not an actual op ) @@ -1226,6 +1227,11 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { step.target = deleted + uint64(len(items)) items = items[:0] deleted = step.target + case opTruncateTailOverHead: + newDeleted := deleted + uint64(len(items)) + 10 + step.target = newDeleted + deleted = newDeleted + items = items[:0] } steps = append(steps, step) } @@ -1268,7 +1274,7 @@ func runRandTest(rt randTest) bool { for i := 0; i < len(step.items); i++ { batch.AppendRaw(step.items[i], step.blobs[i]) } - batch.commit() + rt[i].err = batch.commit() values = append(values, step.blobs...) case opRetrieve: @@ -1290,24 +1296,28 @@ func runRandTest(rt randTest) bool { } case opTruncateHead: - f.truncateHead(step.target) + rt[i].err = f.truncateHead(step.target) length := f.items.Load() - f.itemHidden.Load() values = values[:length] case opTruncateHeadAll: - f.truncateHead(step.target) + rt[i].err = f.truncateHead(step.target) values = nil case opTruncateTail: prev := f.itemHidden.Load() - f.truncateTail(step.target) + rt[i].err = f.truncateTail(step.target) truncated := f.itemHidden.Load() - prev values = values[truncated:] case opTruncateTailAll: - f.truncateTail(step.target) + rt[i].err = f.truncateTail(step.target) + values = nil + + case opTruncateTailOverHead: + rt[i].err = f.truncateTail(step.target) values = nil } // Abort the test on error. @@ -1633,3 +1643,43 @@ func TestFreezerAncientBytes(t *testing.T) { }) } } + +func TestTruncateOverHead(t *testing.T) { + t.Parallel() + + fn := fmt.Sprintf("t-%d", rand.Uint64()) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, freezerTableConfig{noSnappy: true}, false) + if err != nil { + t.Fatal(err) + } + + // Tail truncation on an empty table + if err := f.truncateTail(10); err != nil { + t.Fatal(err) + } + batch := f.newBatch() + data := getChunk(10, 1) + require.NoError(t, batch.AppendRaw(uint64(10), data)) + require.NoError(t, batch.commit()) + + got, err := f.RetrieveItems(uint64(10), 1, 0) + require.NoError(t, err) + if !bytes.Equal(got[0], data) { + t.Fatalf("Unexpected bytes, want: %v, got: %v", data, got[0]) + } + + // Tail truncation on the non-empty table + if err := f.truncateTail(20); err != nil { + t.Fatal(err) + } + batch = f.newBatch() + data = getChunk(10, 1) + require.NoError(t, batch.AppendRaw(uint64(20), data)) + require.NoError(t, batch.commit()) + + got, err = f.RetrieveItems(uint64(20), 1, 0) + require.NoError(t, err) + if !bytes.Equal(got[0], data) { + t.Fatalf("Unexpected bytes, want: %v, got: %v", data, got[0]) + } +} diff --git a/core/rawdb/freezer_utils.go b/core/rawdb/freezer_utils.go index 752e95ba6a..7786b7a990 100644 --- a/core/rawdb/freezer_utils.go +++ b/core/rawdb/freezer_utils.go @@ -22,6 +22,19 @@ import ( "path/filepath" ) +func atomicRename(src, dest string) error { + if err := os.Rename(src, dest); err != nil { + return err + } + dir, err := os.Open(filepath.Dir(src)) + if err != nil { + return err + } + defer dir.Close() + + return dir.Sync() +} + // copyFrom copies data from 'srcPath' at offset 'offset' into 'destPath'. // The 'destPath' is created if it doesn't exist, otherwise it is overwritten. // Before the copy is executed, there is a callback can be registered to @@ -73,13 +86,48 @@ func copyFrom(srcPath, destPath string, offset uint64, before func(f *os.File) e return err } f = nil - return os.Rename(fname, destPath) + + return atomicRename(fname, destPath) +} + +// reset atomically replaces the file at the given path with the provided content. +func reset(path string, content []byte) error { + // Create a temp file in the same dir where we want it to wind up + f, err := os.CreateTemp(filepath.Dir(path), "*") + if err != nil { + return err + } + fname := f.Name() + + // Clean up the leftover file + defer func() { + if f != nil { + f.Close() + } + os.Remove(fname) + }() + + // Write the content into the temp file + _, err = f.Write(content) + if err != nil { + return err + } + // Permanently persist the content into disk + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + f = nil + + return atomicRename(fname, path) } // openFreezerFileForAppend opens a freezer table file and seeks to the end func openFreezerFileForAppend(filename string) (*os.File, error) { // Open the file without the O_APPEND flag - // because it has differing behaviour during Truncate operations + // because it has differing behavior during Truncate operations // on different OS's file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) if err != nil { diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 820c3c03bf..0a9f7091fa 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -412,28 +412,34 @@ func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint // Truncate excessive history entries in either the state history or // the trienode history, ensuring both histories remain aligned with // the state. - head, err := states.Ancients() + shead, err := states.Ancients() if err != nil { return nil, nil, err } - if stateID > head { - return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, head) + if stateID > shead { // Gap is not permitted in the state history + return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, shead) } + truncTo := min(shead, stateID) + if trienodes != nil { - th, err := trienodes.Ancients() + thead, err := trienodes.Ancients() if err != nil { return nil, nil, err } - if stateID > th { - return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, th) - } - if th != head { - log.Info("Histories are not aligned with each other", "state", head, "trienode", th) - head = min(head, th) + if stateID <= thead { + truncTo = min(truncTo, thead) + } else { + if thead == 0 { + _, err = trienodes.TruncateTail(stateID) + if err != nil { + return nil, nil, err + } + log.Warn("Initialized trienode history") + } else { + return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, thead) + } } } - head = min(head, stateID) - // Truncate the extra history elements above in freezer in case it's not // aligned with the state. It might happen after an unclean shutdown. truncate := func(store ethdb.AncientStore, typ historyType, nhead uint64) { @@ -448,7 +454,7 @@ func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint log.Warn("Truncated extra histories", "typ", typ, "number", pruned) } } - truncate(states, typeStateHistory, head) - truncate(trienodes, typeTrienodeHistory, head) + truncate(states, typeStateHistory, truncTo) + truncate(trienodes, typeTrienodeHistory, truncTo) return states, trienodes, nil } diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index aaa64e902c..e3cfbcba8a 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -349,7 +349,7 @@ func (db *Database) HistoricNodeReader(root common.Hash) (*HistoricalNodeReader, // are not accessible. meta, err := readTrienodeMetadata(db.trienodeFreezer, *id+1) if err != nil { - return nil, err // e.g., the referred trienode history has been pruned + return nil, fmt.Errorf("state %#x is not available", root) // e.g., the referred trienode history has been pruned } if meta.parent != root { return nil, fmt.Errorf("state %#x is not canonincal", root) From de0a452f7d2b3259ef7bb5eaf68fc5daf761df91 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:21:45 +0800 Subject: [PATCH 28/52] eth/filters: fix race in pending tx and new heads subscriptions (#33990) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TestSubscribePendingTxHashes` hangs indefinitely because pending tx events are permanently missed due to a race condition in `NewPendingTransactions` (and `NewHeads`). Both handlers called their event subscription functions (`SubscribePendingTxs`, `SubscribeNewHeads`) inside goroutines, so the RPC handler returned the subscription ID to the client before the filter was installed in the event loop. When the client then sent a transaction, the event fired but no filter existed to catch it — the event was silently lost. - Move `SubscribePendingTxs` and `SubscribeNewHeads` calls out of goroutines so filters are installed synchronously before the RPC response is sent, matching the pattern already used by `Logs` and `TransactionReceipts` --- 💬 We'd love your input! Share your thoughts on Copilot coding agent in our [2 minute survey](https://gh.io/copilot-coding-agent-survey). --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: s1na <1591639+s1na@users.noreply.github.com> --- eth/filters/api.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index f4bed35b26..2cb72dc114 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -187,11 +187,13 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } - rpcSub := notifier.CreateSubscription() + var ( + rpcSub = notifier.CreateSubscription() + txs = make(chan []*types.Transaction, 128) + pendingTxSub = api.events.SubscribePendingTxs(txs) + ) go func() { - txs := make(chan []*types.Transaction, 128) - pendingTxSub := api.events.SubscribePendingTxs(txs) defer pendingTxSub.Unsubscribe() chainConfig := api.sys.backend.ChainConfig() @@ -260,11 +262,13 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } - rpcSub := notifier.CreateSubscription() + var ( + rpcSub = notifier.CreateSubscription() + headers = make(chan *types.Header) + headersSub = api.events.SubscribeNewHeads(headers) + ) go func() { - headers := make(chan *types.Header) - headersSub := api.events.SubscribeNewHeads(headers) defer headersSub.Unsubscribe() for { From 95b9a2ed77e8b7330206373358fda3a3d3426bbd Mon Sep 17 00:00:00 2001 From: jvn Date: Thu, 12 Mar 2026 07:53:49 +0530 Subject: [PATCH 29/52] core: Implement eip-7954 increase Maximum Contract Size (#33832) Implement EIP7954, This PR raises the maximum contract code size to 32KiB and initcode size to 64KiB , following https://eips.ethereum.org/EIPS/eip-7954 --------- Co-authored-by: Marius van der Wijden --- cmd/evm/internal/t8ntool/transaction.go | 9 +++++-- core/state_transition.go | 6 +++-- core/txpool/validation.go | 7 ++++-- core/vm/common.go | 31 +++++++++++++++++++++++++ core/vm/evm.go | 4 ++-- core/vm/gas_table.go | 13 +++++------ core/vm/gas_table_test.go | 6 ++++- params/protocol_params.go | 6 +++-- 8 files changed, 64 insertions(+), 18 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 4ba7b5f130..3a457eeaec 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -27,7 +27,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" @@ -177,9 +179,12 @@ func Transaction(ctx *cli.Context) error { r.Error = errors.New("gas * maxFeePerGas exceeds 256 bits") } // Check whether the init code size has been exceeded. - if chainConfig.IsShanghai(new(big.Int), 0) && tx.To() == nil && len(tx.Data()) > params.MaxInitCodeSize { - r.Error = errors.New("max initcode size exceeded") + if tx.To() == nil { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(tx.Data()))); err != nil { + r.Error = err + } } + if chainConfig.IsOsaka(new(big.Int), 0) && tx.Gas() > params.MaxTxGas { r.Error = errors.New("gas limit exceeds maximum") } diff --git a/core/state_transition.go b/core/state_transition.go index 76a5147363..6a40b4f7ab 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -488,8 +488,10 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { } // Check whether the init code size has been exceeded. - if rules.IsShanghai && contractCreation && len(msg.Data) > params.MaxInitCodeSize { - return nil, fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, len(msg.Data), params.MaxInitCodeSize) + if contractCreation { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(msg.Data))); err != nil { + return nil, err + } } // Execute the preparatory steps for state transition which includes: diff --git a/core/txpool/validation.go b/core/txpool/validation.go index e0a333dfa5..13b1bfa312 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -86,8 +87,10 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return fmt.Errorf("%w: type %d rejected, pool not yet in Prague", core.ErrTxTypeNotSupported, tx.Type()) } // Check whether the init code size has been exceeded - if rules.IsShanghai && tx.To() == nil && len(tx.Data()) > params.MaxInitCodeSize { - return fmt.Errorf("%w: code size %v, limit %v", core.ErrMaxInitCodeSizeExceeded, len(tx.Data()), params.MaxInitCodeSize) + if tx.To() == nil { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(tx.Data()))); err != nil { + return err + } } if rules.IsOsaka && tx.Gas() > params.MaxTxGas { return fmt.Errorf("%w (cap: %d, tx: %d)", core.ErrGasLimitTooHigh, params.MaxTxGas, tx.Gas()) diff --git a/core/vm/common.go b/core/vm/common.go index 2990f58972..2d631f8a55 100644 --- a/core/vm/common.go +++ b/core/vm/common.go @@ -17,12 +17,43 @@ package vm import ( + "fmt" "math" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) +// CheckMaxInitCodeSize checks the size of contract initcode against the protocol-defined limit. +func CheckMaxInitCodeSize(rules *params.Rules, size uint64) error { + if rules.IsAmsterdam { + if size > params.MaxInitCodeSizeAmsterdam { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, size, params.MaxInitCodeSizeAmsterdam) + } + } else if rules.IsShanghai { + if size > params.MaxInitCodeSize { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, size, params.MaxInitCodeSize) + } + } + + return nil +} + +// CheckMaxCodeSize checks the size of contract code against the protocol-defined limit. +func CheckMaxCodeSize(rules *params.Rules, size uint64) error { + if rules.IsAmsterdam { + if size > params.MaxCodeSizeAmsterdam { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxCodeSizeExceeded, size, params.MaxCodeSizeAmsterdam) + } + } else if rules.IsEIP158 { + if size > params.MaxCodeSize { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxCodeSizeExceeded, size, params.MaxCodeSize) + } + } + return nil +} + // calcMemSize64 calculates the required memory size, and returns // the size and whether the result overflowed uint64 func calcMemSize64(off, l *uint256.Int) (uint64, bool) { diff --git a/core/vm/evm.go b/core/vm/evm.go index 97ae9468bf..5897dbd265 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -597,8 +597,8 @@ func (evm *EVM) initNewContract(contract *Contract, address common.Address) ([]b } // Check whether the max code size has been exceeded, assign err if the case. - if evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize { - return ret, ErrMaxCodeSizeExceeded + if err := CheckMaxCodeSize(&evm.chainRules, uint64(len(ret))); err != nil { + return ret, err } // Reject code starting with 0xEF if EIP-3541 is enabled. diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 23a2cbbf4d..aa1ad918bb 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -18,7 +18,6 @@ package vm import ( "errors" - "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" @@ -318,10 +317,10 @@ func gasCreateEip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, m if overflow { return 0, ErrGasUintOverflow } - if size > params.MaxInitCodeSize { - return 0, fmt.Errorf("%w: size %d", ErrMaxInitCodeSizeExceeded, size) + if err := CheckMaxInitCodeSize(&evm.chainRules, size); err != nil { + return 0, err } - // Since size <= params.MaxInitCodeSize, these multiplication cannot overflow + // Since size <= the protocol-defined maximum initcode size limit, these multiplication cannot overflow moreGas := params.InitCodeWordGas * ((size + 31) / 32) if gas, overflow = math.SafeAdd(gas, moreGas); overflow { return 0, ErrGasUintOverflow @@ -337,10 +336,10 @@ func gasCreate2Eip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, if overflow { return 0, ErrGasUintOverflow } - if size > params.MaxInitCodeSize { - return 0, fmt.Errorf("%w: size %d", ErrMaxInitCodeSizeExceeded, size) + if err := CheckMaxInitCodeSize(&evm.chainRules, size); err != nil { + return 0, err } - // Since size <= params.MaxInitCodeSize, these multiplication cannot overflow + // Since size <= the protocol-defined maximum initcode size limit, these multiplication cannot overflow moreGas := (params.InitCodeWordGas + params.Keccak256WordGas) * ((size + 31) / 32) if gas, overflow = math.SafeAdd(gas, moreGas); overflow { return 0, ErrGasUintOverflow diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 7fe76b0a63..436cc47f2e 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -148,11 +148,15 @@ func TestCreateGas(t *testing.T) { BlockNumber: big.NewInt(0), } config := Config{} + chainConfig := params.AllEthashProtocolChanges if tt.eip3860 { config.ExtraEips = []int{3860} + vmctx.Random = new(common.Hash) + + chainConfig = params.MergedTestChainConfig } - evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, config) + evm := NewEVM(vmctx, statedb, chainConfig, config) var startGas = uint64(testGas) ret, gas, err := evm.Call(common.Address{}, address, nil, startGas, new(uint256.Int)) if err != nil { diff --git a/params/protocol_params.go b/params/protocol_params.go index bb506af015..cebf5008c8 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -134,8 +134,10 @@ const ( DefaultElasticityMultiplier = 2 // Bounds the maximum gas limit an EIP-1559 block may have. InitialBaseFee = 1000000000 // Initial base fee for EIP-1559 blocks. - MaxCodeSize = 24576 // Maximum bytecode to permit for a contract - MaxInitCodeSize = 2 * MaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions + MaxCodeSize = 24576 // Maximum bytecode to permit for a contract + MaxInitCodeSize = 2 * MaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions + MaxCodeSizeAmsterdam = 32768 // Maximum bytecode to permit for a contract post Amsterdam + MaxInitCodeSizeAmsterdam = 2 * MaxCodeSizeAmsterdam // Maximum initcode to permit in a creation transaction and create instructions post Amsterdam // Precompiled contract gas prices From 1c9ddee16f925989cc6bda3df39a98cd46c8b1f1 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:20:12 +0100 Subject: [PATCH 30/52] trie/bintrie: use a sync.Pool when hashing binary tree nodes (#33989) Binary tree hashing is quite slow, owing to many factors. One of them is the GC pressure that is the consequence of allocating many hashers, as a binary tree has 4x the size of an MPT. This PR introduces an optimization that already exists for the MPT: keep a pool of hashers, in order to reduce the amount of allocations. --- trie/bintrie/hasher.go | 39 +++++++++++++++++++++++++++++++++++ trie/bintrie/internal_node.go | 4 ++-- trie/bintrie/key_encoding.go | 4 ++-- trie/bintrie/stem_node.go | 10 +++++---- 4 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 trie/bintrie/hasher.go diff --git a/trie/bintrie/hasher.go b/trie/bintrie/hasher.go new file mode 100644 index 0000000000..b81c145723 --- /dev/null +++ b/trie/bintrie/hasher.go @@ -0,0 +1,39 @@ +// Copyright 2026 go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bintrie + +import ( + "crypto/sha256" + "hash" + "sync" +) + +var sha256Pool = sync.Pool{ + New: func() any { + return sha256.New() + }, +} + +func newSha256() hash.Hash { + h := sha256Pool.Get().(hash.Hash) + h.Reset() + return h +} + +func returnSha256(h hash.Hash) { + sha256Pool.Put(h) +} diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index 2d02e240be..7ad76aa9db 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -17,7 +17,6 @@ package bintrie import ( - "crypto/sha256" "errors" "fmt" @@ -125,7 +124,8 @@ func (bt *InternalNode) Hash() common.Hash { return bt.hash } - h := sha256.New() + h := newSha256() + defer returnSha256(h) if bt.left != nil { h.Write(bt.left.Hash().Bytes()) } else { diff --git a/trie/bintrie/key_encoding.go b/trie/bintrie/key_encoding.go index 9b98bee491..c009f1529f 100644 --- a/trie/bintrie/key_encoding.go +++ b/trie/bintrie/key_encoding.go @@ -18,7 +18,6 @@ package bintrie import ( "bytes" - "crypto/sha256" "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" @@ -51,7 +50,8 @@ func GetBinaryTreeKey(addr common.Address, key []byte) []byte { } func getBinaryTreeKey(addr common.Address, offset []byte, overflow bool) []byte { - hasher := sha256.New() + hasher := newSha256() + defer returnSha256(hasher) hasher.Write(zeroHash[:12]) hasher.Write(addr[:]) var buf [32]byte diff --git a/trie/bintrie/stem_node.go b/trie/bintrie/stem_node.go index f1ae2361ff..3f69261d62 100644 --- a/trie/bintrie/stem_node.go +++ b/trie/bintrie/stem_node.go @@ -18,7 +18,6 @@ package bintrie import ( "bytes" - "crypto/sha256" "errors" "fmt" "slices" @@ -114,14 +113,17 @@ func (bt *StemNode) Hash() common.Hash { } var data [StemNodeWidth]common.Hash + h := newSha256() + defer returnSha256(h) for i, v := range bt.Values { if v != nil { - h := sha256.Sum256(v) - data[i] = common.BytesToHash(h[:]) + h.Reset() + h.Write(v) + h.Sum(data[i][:0]) } } + h.Reset() - h := sha256.New() for level := 1; level <= 8; level++ { for i := range StemNodeWidth / (1 << level) { h.Reset() From eaa9418ac184fba1be81e1473ba42db1668714f0 Mon Sep 17 00:00:00 2001 From: Lee Gyumin <82251551+legm0310@users.noreply.github.com> Date: Fri, 13 Mar 2026 17:45:14 +0900 Subject: [PATCH 31/52] core/rawdb: enforce exact key length for num->hash and td in db inspect (#34000) This PR improves `db inspect` classification accuracy in `core/rawdb/database.go` by tightening key-shape checks for: - `Block number->hash` - `Difficulties (deprecated)` Previously, both categories used prefix/suffix heuristics and could mis-bucket unrelated entries. --- core/rawdb/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 576e32b961..a13afb9c0e 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -478,9 +478,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bodies.add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): receipts.add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix) && len(key) == (len(headerPrefix)+8+common.HashLength+len(headerTDSuffix)): tds.add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix) && len(key) == (len(headerPrefix)+8+common.HashLength+len(headerHashSuffix)): numHashPairings.add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.add(size) From dba741fd3156a16bf26feb469f5e7e2de2207e5f Mon Sep 17 00:00:00 2001 From: Lessa <230214854+adblesss@users.noreply.github.com> Date: Fri, 13 Mar 2026 07:39:45 -0400 Subject: [PATCH 32/52] console: fix autocomplete digit range to include 0 (#34003) PR #26518 added digit support but used '1'-'9' instead of '0'-'9'. This breaks autocomplete for identifiers containing 0 like account0. --- console/console.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/console/console.go b/console/console.go index b5c77bd78f..573d1da570 100644 --- a/console/console.go +++ b/console/console.go @@ -282,7 +282,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str for ; start > 0; start-- { // Skip all methods and namespaces (i.e. including the dot) c := line[start] - if c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '1' && c <= '9') { + if c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') { continue } // We've hit an unexpected character, autocomplete form here From 189f9d0b17ca493ecc985a7896a0cc30dd157ca2 Mon Sep 17 00:00:00 2001 From: vickkkkkyy Date: Fri, 13 Mar 2026 20:26:20 +0800 Subject: [PATCH 33/52] eth/filters: check history pruning cutoff in GetFilterLogs (#33823) Return proper error for the log filters going beyond pruning point on a node with expired history. --- eth/filters/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/filters/api.go b/eth/filters/api.go index 2cb72dc114..e4ade96598 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -536,6 +536,9 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo if f.crit.ToBlock != nil { end = f.crit.ToBlock.Int64() } + if begin >= 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { + return nil, &history.PrunedHistoryError{} + } // Construct the range filter filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics, api.rangeLimit) } From ede376af8ea3b8f02c2649f8deab4e0764befbd2 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Fri, 13 Mar 2026 12:09:32 -0400 Subject: [PATCH 34/52] internal/ethapi: encode slotNumber as hex in RPCMarshalHeader (#34005) The slotNumber field was being passed as a raw *uint64 to the JSON marshaler, which serializes it as a plain decimal integer (e.g. 159). All Ethereum JSON-RPC quantity fields must be hex-encoded per spec. Wrap with hexutil.Uint64 to match the encoding of other numeric header fields like blobGasUsed and excessBlobGas. Co-authored-by: qu0b --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 41d165a423..bb0dd042ab 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -966,7 +966,7 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { result["requestsHash"] = head.RequestsHash } if head.SlotNumber != nil { - result["slotNumber"] = head.SlotNumber + result["slotNumber"] = hexutil.Uint64(*head.SlotNumber) } return result } From 24025c2bd0a0e2b01435fe29c2f18a3f9ec764c0 Mon Sep 17 00:00:00 2001 From: vickkkkkyy Date: Sat, 14 Mar 2026 17:22:50 +0800 Subject: [PATCH 35/52] build: fix signify flag name in doWindowsInstaller (#34006) The signify flag in `doWindowsInstaller` was defined as "signify key" (with a space), making it impossible to pass via CLI (`-signify `). This meant the Windows installer signify signing was silently never executed. Fix by renaming the flag to "signify", consistent with `doArchive` and `doKeeperArchive`. --- build/ci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci.go b/build/ci.go index 4d0a1d7e35..dce01f01a8 100644 --- a/build/ci.go +++ b/build/ci.go @@ -1196,7 +1196,7 @@ func doWindowsInstaller(cmdline []string) { var ( arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) - signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) + signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`) upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) ) From 77e7e5ad1a0c6e758312820af75e94293f025168 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:35:07 +0100 Subject: [PATCH 36/52] go.mod, go.sum: update karalabe/hid to fix broken FreeBSD ports build (#34008) cgo builds have been broken in FreeBSD ports because of the hid lib. @enriquefynn has made a temporary patch, but the fix has been merged in the master branch, so let's reflect that here. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bfe2df8c0c..37a2537dd0 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 - github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 + github.com/karalabe/hid v1.0.1-0.20260315100226-f5d04adeffeb github.com/klauspost/compress v1.17.8 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 diff --git a/go.sum b/go.sum index b8c0558c8c..c465603242 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 h1:msKODTL1m0wigztaqILOtla9HeW1ciscYG4xjLtvk5I= -github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= +github.com/karalabe/hid v1.0.1-0.20260315100226-f5d04adeffeb h1:Ag83At00qa4FLkcdMgrwHVSakqky/eZczOlxd4q336E= +github.com/karalabe/hid v1.0.1-0.20260315100226-f5d04adeffeb/go.mod h1:qk1sX/IBgppQNcGCRoj90u6EGC056EBoIc1oEjCWla8= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= From a7d09cc14ff5a3e88a4ed328354f918cc2ca0b69 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 16 Mar 2026 16:45:26 +0800 Subject: [PATCH 37/52] core: fix code database initialization in stateless mode (#34011) This PR fixes the statedb initialization, ensuring the data source is bound with the stateless input. --- core/stateless.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/stateless.go b/core/stateless.go index 88d8ed8138..86d4dc304b 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -53,7 +53,7 @@ func ExecuteStateless(ctx context.Context, config *params.ChainConfig, vmconfig } // Create and populate the state database to serve as the stateless backend memdb := witness.MakeHashDB() - db, err := state.New(witness.Root(), state.NewDatabase(triedb.NewDatabase(memdb, triedb.HashDefaults), nil)) + db, err := state.New(witness.Root(), state.NewDatabase(triedb.NewDatabase(memdb, triedb.HashDefaults), state.NewCodeDB(memdb))) if err != nil { return common.Hash{}, common.Hash{}, err } From 98b13f342f046f72d0f5eca1bd4ca5b6951a8d79 Mon Sep 17 00:00:00 2001 From: Jonny Rhea <5555162+jrhea@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:24:41 -0500 Subject: [PATCH 38/52] miner: add OpenTelemetry spans for block building path (#33773) Instruments the block building path with OpenTelemetry tracing spans. - added spans in forkchoiceUpdated -> buildPayload -> background payload loop -> generateWork iterations. Spans should look something like this: ``` jsonrpc.engine/forkchoiceUpdatedV3 |- rpc.runMethod | |- engine.forkchoiceUpdated | |- miner.buildPayload [payload.id, parent.hash, timestamp] | |- miner.generateWork [txs.count, gas.used, fees] (empty block) | | |- miner.prepareWork | | |- miner.FinalizeAndAssemble | | |- consensus.beacon.FinalizeAndAssemble [block.number, txs.count, withdrawals.count] | | |- consensus.beacon.Finalize | | |- consensus.beacon.IntermediateRoot | | |- consensus.beacon.NewBlock | |- miner.background [block.number, iterations.total, exit.reason, empty.delivered] | |- miner.buildIteration [iteration, update.accepted] | | |- miner.generateWork [txs.count, gas.used, fees] | | |- miner.prepareWork | | |- miner.fillTransactions [pending.plain.count, pending.blob.count] | | | |- miner.commitTransactions.priority (if prio txs exist) | | | | |- miner.commitTransactions | | | | |- miner.commitTransaction (per tx) | | | |- miner.commitTransactions.normal (if normal txs exist) | | | |- miner.commitTransactions | | | |- miner.commitTransaction (per tx) | | |- miner.FinalizeAndAssemble | | |- consensus.beacon.FinalizeAndAssemble [block.number, txs.count, withdrawals.count] | | |- consensus.beacon.Finalize | | |- consensus.beacon.IntermediateRoot | | |- consensus.beacon.NewBlock | |- miner.buildIteration [iteration, update.accepted] | | |- ... | |- ... ``` - added simulated server spans in SimulatedBeacon.sealBlock so dev mode (geth --dev) produces traces that mirror production Engine API calls from a real consensus client. --------- Co-authored-by: Felix Lange --- consensus/beacon/consensus.go | 23 ++++++-- consensus/clique/clique.go | 3 +- consensus/consensus.go | 3 +- consensus/ethash/consensus.go | 3 +- core/chain_makers.go | 3 +- core/txpool/blobpool/blobpool.go | 8 +-- core/txpool/blobpool/blobpool_test.go | 2 +- core/txpool/legacypool/legacypool.go | 8 +-- core/txpool/subpool.go | 2 +- core/txpool/txpool.go | 11 ++-- eth/api_backend.go | 2 +- eth/catalyst/api.go | 22 ++++---- eth/catalyst/api_test.go | 36 ++++++------- eth/catalyst/simulated_beacon.go | 38 +++++++++++--- eth/catalyst/witness.go | 12 ++--- eth/handler.go | 2 +- eth/handler_test.go | 6 ++- eth/sync.go | 3 +- internal/ethapi/simulate.go | 2 +- miner/miner.go | 26 ++++----- miner/payload_building.go | 76 ++++++++++++++++++++++----- miner/payload_building_test.go | 3 +- miner/worker.go | 57 +++++++++++++++----- 23 files changed, 245 insertions(+), 106 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 45a480c50e..25f4f9d2b2 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -17,6 +17,7 @@ package beacon import ( + "context" "errors" "fmt" "math/big" @@ -29,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" @@ -351,9 +353,17 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, setting the final state and // assembling the block. -func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (beacon *Beacon) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (result *types.Block, err error) { + ctx, _, spanEnd := telemetry.StartSpan(ctx, "consensus.beacon.FinalizeAndAssemble", + telemetry.Int64Attribute("block.number", int64(header.Number.Uint64())), + telemetry.Int64Attribute("txs.count", int64(len(body.Transactions))), + telemetry.Int64Attribute("withdrawals.count", int64(len(body.Withdrawals))), + ) + defer spanEnd(&err) + if !beacon.IsPoSHeader(header) { - return beacon.ethone.FinalizeAndAssemble(chain, header, state, body, receipts) + block, delegateErr := beacon.ethone.FinalizeAndAssemble(ctx, chain, header, state, body, receipts) + return block, delegateErr } shanghai := chain.Config().IsShanghai(header.Number, header.Time) if shanghai { @@ -367,13 +377,20 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea } } // Finalize and assemble the block. + _, _, finalizeSpanEnd := telemetry.StartSpan(ctx, "consensus.beacon.Finalize") beacon.Finalize(chain, header, state, body) + finalizeSpanEnd(nil) // Assign the final state root to header. + _, _, rootSpanEnd := telemetry.StartSpan(ctx, "consensus.beacon.IntermediateRoot") header.Root = state.IntermediateRoot(true) + rootSpanEnd(nil) // Assemble the final block. - return types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)), nil + _, _, blockSpanEnd := telemetry.StartSpan(ctx, "consensus.beacon.NewBlock") + block := types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)) + blockSpanEnd(nil) + return block, nil } // Seal generates a new sealing request for the given input block and pushes diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 28095011c1..3bf79d5a62 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -19,6 +19,7 @@ package clique import ( "bytes" + "context" "errors" "fmt" "io" @@ -581,7 +582,7 @@ func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Heade // FinalizeAndAssemble implements consensus.Engine, ensuring no uncles are set, // nor block rewards given, and returns the final block. -func (c *Clique) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (c *Clique) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if len(body.Withdrawals) > 0 { return nil, errors.New("clique does not support withdrawals") } diff --git a/consensus/consensus.go b/consensus/consensus.go index a68351f7ff..094026b614 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -18,6 +18,7 @@ package consensus import ( + "context" "math/big" "github.com/ethereum/go-ethereum/common" @@ -92,7 +93,7 @@ type Engine interface { // // Note: The block header and state database might be updated to reflect any // consensus rules that happen at finalization (e.g. block rewards). - FinalizeAndAssemble(chain ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) + FinalizeAndAssemble(ctx context.Context, chain ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) // Seal generates a new sealing request for the given input block and pushes // the result into the given channel. diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 61371e0636..56256d1215 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -17,6 +17,7 @@ package ethash import ( + "context" "errors" "fmt" "math/big" @@ -513,7 +514,7 @@ func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, accumulating the block and // uncle rewards, setting the final state and assembling the block. -func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (ethash *Ethash) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if len(body.Withdrawals) > 0 { return nil, errors.New("ethash does not support withdrawals") } diff --git a/core/chain_makers.go b/core/chain_makers.go index e4b5cf964f..8f6eed1697 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" @@ -411,7 +412,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } body := types.Body{Transactions: b.txs, Uncles: b.uncles, Withdrawals: b.withdrawals} - block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, &body, b.receipts) + block, err := b.engine.FinalizeAndAssemble(context.Background(), cm, b.header, statedb, &body, b.receipts) if err != nil { panic(err) } diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 27fdd00016..7155a67a9b 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1865,11 +1865,11 @@ func (p *BlobPool) drop() { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { +func (p *BlobPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) { // If only plain transactions are requested, this pool is unsuitable as it // contains none, don't even bother. if !filter.BlobTxs { - return nil + return nil, 0 } // Track the amount of time waiting to retrieve the list of pending blob txs // from the pool and the amount of time actually spent on assembling the data. @@ -1885,6 +1885,7 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*tx pendtimeHist.Update(time.Since(execStart).Nanoseconds()) }() + var count int pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index)) for addr, txs := range p.index { lazies := make([]*txpool.LazyTransaction, 0, len(txs)) @@ -1930,9 +1931,10 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*tx } if len(lazies) > 0 { pending[addr] = lazies + count += len(lazies) } } - return pending + return pending, count } // updateStorageMetrics retrieves a bunch of stats from the data store and pushes diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 6580a339e3..ba96bea8ed 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -2122,7 +2122,7 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { b.ReportAllocs() for i := 0; i < b.N; i++ { - p := pool.Pending(txpool.PendingFilter{ + p, _ := pool.Pending(txpool.PendingFilter{ MinTip: uint256.NewInt(1), BaseFee: chain.basefee, BlobFee: chain.blobfee, diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 36970c820e..25c4b13166 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -494,15 +494,16 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { +func (pool *LegacyPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) { // If only blob transactions are requested, this pool is unsuitable as it // contains none, don't even bother. if filter.BlobTxs { - return nil + return nil, 0 } pool.mu.Lock() defer pool.mu.Unlock() + var count int pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending)) for addr, list := range pool.pending { txs := list.Flatten() @@ -539,9 +540,10 @@ func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address] } } pending[addr] = lazies + count += len(lazies) } } - return pending + return pending, count } // ValidateTxBasics checks whether a transaction is valid according to the consensus diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index db099ddf98..4cc1b193d6 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -154,7 +154,7 @@ type SubPool interface { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. - Pending(filter PendingFilter) map[common.Address][]*LazyTransaction + Pending(filter PendingFilter) (map[common.Address][]*LazyTransaction, int) // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index a314a83f1b..25647e0cce 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -359,14 +359,17 @@ func (p *TxPool) Add(txs []*types.Transaction, sync bool) []error { // // The transactions can also be pre-filtered by the dynamic fee components to // reduce allocations and load on downstream subsystems. -func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction { +func (p *TxPool) Pending(filter PendingFilter) (map[common.Address][]*LazyTransaction, int) { + var count int txs := make(map[common.Address][]*LazyTransaction) for _, subpool := range p.subpools { - for addr, set := range subpool.Pending(filter) { - txs[addr] = set + set, n := subpool.Pending(filter) + for addr, list := range set { + txs[addr] = list } + count += n } - return txs + return txs, count } // SubscribeTransactions registers a subscription for new transaction events, diff --git a/eth/api_backend.go b/eth/api_backend.go index 3f826b7861..726d8316a0 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -347,7 +347,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(txpool.PendingFilter{}) + pending, _ := b.eth.txPool.Pending(txpool.PendingFilter{}) var txs types.Transactions for _, batch := range pending { for _, lazy := range batch { diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 96d4570561..8a4aced04b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -162,7 +162,7 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // // If there are payloadAttributes: we try to assemble a block with the payloadAttributes // and return its payloadID. -func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV1(ctx context.Context, update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if payloadAttributes != nil { switch { case payloadAttributes.Withdrawals != nil || payloadAttributes.BeaconRoot != nil: @@ -171,12 +171,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa return engine.STATUS_INVALID, paramsErr("fcuV1 called post-shanghai") } } - return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, false) + return api.forkchoiceUpdated(ctx, update, payloadAttributes, engine.PayloadV1, false) } // ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload // attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2. -func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV2(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.BeaconRoot != nil: @@ -189,12 +189,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa return engine.STATUS_INVALID, unsupportedForkErr("fcuV2 must only be called with paris or shanghai payloads") } } - return api.forkchoiceUpdated(update, params, engine.PayloadV2, false) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV2, false) } // ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root // in the payload attributes. It supports only PayloadAttributesV3. -func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV3(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.Withdrawals == nil: @@ -209,12 +209,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, pa // hash, even if params are wrong. To do this we need to split up // forkchoiceUpdate into a function that only updates the head and then a // function that kicks off block construction. - return api.forkchoiceUpdated(update, params, engine.PayloadV3, false) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV3, false) } // ForkchoiceUpdatedV4 is equivalent to V3 with the addition of slot number // in the payload attributes. It supports only PayloadAttributesV4. -func (api *ConsensusAPI) ForkchoiceUpdatedV4(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.Withdrawals == nil: @@ -231,10 +231,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV4(update engine.ForkchoiceStateV1, pa // hash, even if params are wrong. To do this we need to split up // forkchoiceUpdate into a function that only updates the head and then a // function that kicks off block construction. - return api.forkchoiceUpdated(update, params, engine.PayloadV4, false) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV4, false) } -func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, payloadWitness bool) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) forkchoiceUpdated(ctx context.Context, update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, payloadWitness bool) (result engine.ForkChoiceResponse, err error) { + ctx, _, spanEnd := telemetry.StartSpan(ctx, "engine.forkchoiceUpdated") + defer spanEnd(&err) api.forkchoiceLock.Lock() defer api.forkchoiceLock.Unlock() @@ -375,7 +377,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if api.localBlocks.has(id) { return valid(&id), nil } - payload, err := api.eth.Miner().BuildPayload(args, payloadWitness) + payload, err := api.eth.Miner().BuildPayload(ctx, args, payloadWitness) if err != nil { log.Error("Failed to build payload", "err", err) return valid(nil), engine.InvalidPayloadAttributes.With(err) diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index db0505101f..d126c362fe 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -190,7 +190,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - _, err := api.ForkchoiceUpdatedV1(fcState, &blockParams) + _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -270,7 +270,7 @@ func TestInvalidPayloadTimestamp(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - _, err := api.ForkchoiceUpdatedV1(fcState, ¶ms) + _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, ¶ms) if test.shouldErr && err == nil { t.Fatalf("expected error preparing payload with invalid timestamp, err=%v", err) } else if !test.shouldErr && err != nil { @@ -329,7 +329,7 @@ func TestEth2NewBlock(t *testing.T) { SafeBlockHash: block.Hash(), FinalizedBlockHash: block.Hash(), } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if have, want := ethservice.BlockChain().CurrentBlock().Number.Uint64(), block.NumberU64(); have != want { @@ -369,7 +369,7 @@ func TestEth2NewBlock(t *testing.T) { SafeBlockHash: block.Hash(), FinalizedBlockHash: block.Hash(), } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().Number.Uint64() != block.NumberU64() { @@ -515,7 +515,7 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.He SafeBlockHash: payload.ParentHash, FinalizedBlockHash: payload.ParentHash, } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().Number.Uint64() != payload.Number { @@ -629,7 +629,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { err error ) for i := 0; ; i++ { - if resp, err = api.ForkchoiceUpdatedV1(fcState, ¶ms); err != nil { + if resp, err = api.ForkchoiceUpdatedV1(context.Background(), fcState, ¶ms); err != nil { t.Fatalf("error preparing payload, err=%v", err) } if resp.PayloadStatus.Status != engine.VALID { @@ -660,7 +660,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { SafeBlockHash: payload.ExecutionPayload.ParentHash, FinalizedBlockHash: payload.ExecutionPayload.ParentHash, } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } if ethservice.BlockChain().CurrentBlock().Number.Uint64() != payload.ExecutionPayload.Number { @@ -679,7 +679,7 @@ func assembleEnvelope(api *ConsensusAPI, parentHash common.Hash, params *engine. Withdrawals: params.Withdrawals, BeaconRoot: params.BeaconRoot, } - payload, err := api.eth.Miner().BuildPayload(args, false) + payload, err := api.eth.Miner().BuildPayload(context.Background(), args, false) if err != nil { return nil, err } @@ -867,7 +867,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { t.Error("invalid status: VALID on an invalid chain") } // Now reorg to the head of the invalid chain - resp, err := apiB.ForkchoiceUpdatedV1(engine.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) + resp, err := apiB.ForkchoiceUpdatedV1(context.Background(), engine.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) if err != nil { t.Fatal(err) } @@ -970,7 +970,7 @@ func TestSimultaneousNewBlock(t *testing.T) { for ii := 0; ii < 10; ii++ { go func() { defer wg.Done() - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + if _, err := api.ForkchoiceUpdatedV1(context.Background(), fcState, nil); err != nil { errMu.Lock() testErr = fmt.Errorf("failed to insert block: %w", err) errMu.Unlock() @@ -1011,7 +1011,7 @@ func TestWithdrawals(t *testing.T) { fcState := engine.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), } - resp, err := api.ForkchoiceUpdatedV2(fcState, &blockParams) + resp, err := api.ForkchoiceUpdatedV2(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1063,7 +1063,7 @@ func TestWithdrawals(t *testing.T) { }, } fcState.HeadBlockHash = execData.ExecutionPayload.BlockHash - _, err = api.ForkchoiceUpdatedV2(fcState, &blockParams) + _, err = api.ForkchoiceUpdatedV2(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1090,7 +1090,7 @@ func TestWithdrawals(t *testing.T) { // 11: set block as head. fcState.HeadBlockHash = execData.ExecutionPayload.BlockHash - _, err = api.ForkchoiceUpdatedV2(fcState, nil) + _, err = api.ForkchoiceUpdatedV2(context.Background(), fcState, nil) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } @@ -1196,10 +1196,10 @@ func TestNilWithdrawals(t *testing.T) { ) if !shanghai { payloadVersion = engine.PayloadV1 - _, err = api.ForkchoiceUpdatedV1(fcState, &test.blockParams) + _, err = api.ForkchoiceUpdatedV1(context.Background(), fcState, &test.blockParams) } else { payloadVersion = engine.PayloadV2 - _, err = api.ForkchoiceUpdatedV2(fcState, &test.blockParams) + _, err = api.ForkchoiceUpdatedV2(context.Background(), fcState, &test.blockParams) } if test.wantErr { if err == nil { @@ -1579,7 +1579,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { fcState := engine.ForkchoiceStateV1{ HeadBlockHash: parent.Hash(), } - resp, err := api.ForkchoiceUpdatedV3(fcState, &blockParams) + resp, err := api.ForkchoiceUpdatedV3(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err.(*engine.EngineAPIError).ErrorData()) } @@ -1610,7 +1610,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { } fcState.HeadBlockHash = execData.ExecutionPayload.BlockHash - resp, err = api.ForkchoiceUpdatedV3(fcState, nil) + resp, err = api.ForkchoiceUpdatedV3(context.Background(), fcState, nil) if err != nil { t.Fatalf("error preparing payload, err=%v", err.(*engine.EngineAPIError).ErrorData()) } @@ -1666,7 +1666,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { SafeBlockHash: common.Hash{}, FinalizedBlockHash: common.Hash{}, } - _, err := api.ForkchoiceUpdatedWithWitnessV3(fcState, &blockParams) + _, err := api.ForkchoiceUpdatedWithWitnessV3(context.Background(), fcState, &blockParams) if err != nil { t.Fatalf("error preparing payload, err=%v", err) } diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 452902c78c..8a77cd8abe 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -126,7 +126,7 @@ func NewSimulatedBeacon(period uint64, feeRecipient common.Address, eth *eth.Eth // if genesis block, send forkchoiceUpdated to trigger transition to PoS if block.Number.Sign() == 0 { version := payloadVersion(eth.BlockChain().Config(), block.Time) - if _, err := engineAPI.forkchoiceUpdated(current, nil, version, false); err != nil { + if _, err := engineAPI.forkchoiceUpdated(context.Background(), current, nil, version, false); err != nil { return nil, err } } @@ -212,7 +212,16 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u slotNumber := uint64(0) attribute.SlotNumber = &slotNumber } - fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, attribute, version, false) + + // Create a server span for forkchoiceUpdated with payload attributes, + // simulating an incoming engine API request from a real consensus client. + fcCtx, fcSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "forkchoiceUpdatedV" + fmt.Sprintf("%d", version), + }) + fcResponse, err := c.engineAPI.forkchoiceUpdated(fcCtx, c.curForkchoiceState, attribute, version, false) + fcSpanEnd(&err) if err != nil { return err } @@ -226,7 +235,15 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u return nil } + // Create a server span for getPayload, simulating the consensus client + // coming back to retrieve the built payload. + _, gpSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "getPayloadV" + fmt.Sprintf("%d", version), + }) envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true, nil, nil) + gpSpanEnd(&err) if err != nil { return err } @@ -274,6 +291,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u Service: "engine", Method: "newPayloadV" + fmt.Sprintf("%d", version), }) + // Mark the payload as canon _, err = c.engineAPI.newPayload(npCtx, *payload, blobHashes, beaconRoot, requests, false) npSpanEnd(&err) @@ -282,8 +300,16 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u } c.setCurrentState(payload.BlockHash, finalizedHash) - // Mark the block containing the payload as canonical - if _, err = c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, nil, version, false); err != nil { + // Create a server span for the final forkchoiceUpdated (no payload attributes), + // which sets the new block as the canonical chain head. + fcuCtx, fcuSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "forkchoiceUpdatedV" + fmt.Sprintf("%d", version), + }) + _, err = c.engineAPI.forkchoiceUpdated(fcuCtx, c.curForkchoiceState, nil, version, false) + fcuSpanEnd(&err) + if err != nil { return err } c.lastBlockTime = payload.Timestamp @@ -349,7 +375,7 @@ func (c *SimulatedBeacon) Rollback() { func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // Ensure no pending transactions. c.eth.TxPool().Sync() - if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { + if pending, _ := c.eth.TxPool().Pending(txpool.PendingFilter{}); len(pending) != 0 { return errors.New("pending block dirty") } @@ -363,7 +389,7 @@ func (c *SimulatedBeacon) Fork(parentHash common.Hash) error { // AdjustTime creates a new block with an adjusted timestamp. func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error { - if len(c.eth.TxPool().Pending(txpool.PendingFilter{})) != 0 { + if pending, _ := c.eth.TxPool().Pending(txpool.PendingFilter{}); len(pending) != 0 { return errors.New("could not adjust time on non-empty block") } parent := c.eth.BlockChain().CurrentBlock() diff --git a/eth/catalyst/witness.go b/eth/catalyst/witness.go index 14ca29e079..fe75c66908 100644 --- a/eth/catalyst/witness.go +++ b/eth/catalyst/witness.go @@ -35,7 +35,7 @@ import ( // ForkchoiceUpdatedWithWitnessV1 is analogous to ForkchoiceUpdatedV1, only it // generates an execution witness too if block building was requested. -func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV1(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV1(ctx context.Context, update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if payloadAttributes != nil { switch { case payloadAttributes.Withdrawals != nil || payloadAttributes.BeaconRoot != nil: @@ -44,12 +44,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV1(update engine.Forkchoice return engine.STATUS_INVALID, paramsErr("fcuV1 called post-shanghai") } } - return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, true) + return api.forkchoiceUpdated(ctx, update, payloadAttributes, engine.PayloadV1, true) } // ForkchoiceUpdatedWithWitnessV2 is analogous to ForkchoiceUpdatedV2, only it // generates an execution witness too if block building was requested. -func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV2(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.BeaconRoot != nil: @@ -62,12 +62,12 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV2(update engine.Forkchoice return engine.STATUS_INVALID, unsupportedForkErr("fcuV2 must only be called with paris or shanghai payloads") } } - return api.forkchoiceUpdated(update, params, engine.PayloadV2, true) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV2, true) } // ForkchoiceUpdatedWithWitnessV3 is analogous to ForkchoiceUpdatedV3, only it // generates an execution witness too if block building was requested. -func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { +func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { if params != nil { switch { case params.Withdrawals == nil: @@ -82,7 +82,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(update engine.Forkchoice // hash, even if params are wrong. To do this we need to split up // forkchoiceUpdate into a function that only updates the head and then a // function that kicks off block construction. - return api.forkchoiceUpdated(update, params, engine.PayloadV3, true) + return api.forkchoiceUpdated(ctx, update, params, engine.PayloadV3, true) } // NewPayloadWithWitnessV1 is analogous to NewPayloadV1, only it also generates diff --git a/eth/handler.go b/eth/handler.go index bb2cd5f88b..27b5e60697 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -86,7 +86,7 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction + Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) // SubscribeTransactions subscribes to new transaction events. The subscriber // can decide whether to receive notifications only for newly seen transactions diff --git a/eth/handler_test.go b/eth/handler_test.go index 3470452980..fee6bae138 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -128,10 +128,11 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error { } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction { +func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*txpool.LazyTransaction, int) { p.lock.RLock() defer p.lock.RUnlock() + var count int batches := make(map[common.Address][]*types.Transaction) for _, tx := range p.pool { from, _ := types.Sender(types.HomesteadSigner{}, tx) @@ -152,9 +153,10 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) map[common.Address][]* Gas: tx.Gas(), BlobGas: tx.BlobGas(), }) + count++ } } - return pending + return pending, count } // SubscribeTransactions should return an event subscription of NewTxsEvent and diff --git a/eth/sync.go b/eth/sync.go index ddae8443a3..8b4bd90abf 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -25,7 +25,8 @@ import ( // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { var hashes []common.Hash - for _, batch := range h.txpool.Pending(txpool.PendingFilter{BlobTxs: false}) { + pending, _ := h.txpool.Pending(txpool.PendingFilter{BlobTxs: false}) + for _, batch := range pending { for _, tx := range batch { hashes = append(hashes, tx.Hash) } diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index f0c69e39a4..ebe221114f 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -406,7 +406,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, Withdrawals: *block.BlockOverrides.Withdrawals, } chainHeadReader := &simChainHeadReader{ctx, sim.b} - b, err := sim.b.Engine().FinalizeAndAssemble(chainHeadReader, header, sim.state, blockBody, receipts) + b, err := sim.b.Engine().FinalizeAndAssemble(ctx, chainHeadReader, header, sim.state, blockBody, receipts) if err != nil { return nil, nil, nil, err } diff --git a/miner/miner.go b/miner/miner.go index ee890b5e54..0ff0237a08 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -18,6 +18,7 @@ package miner import ( + "context" "fmt" "math/big" "sync" @@ -135,8 +136,8 @@ func (miner *Miner) SetGasTip(tip *big.Int) error { } // BuildPayload builds the payload according to the provided parameters. -func (miner *Miner) BuildPayload(args *BuildPayloadArgs, witness bool) (*Payload, error) { - return miner.buildPayload(args, witness) +func (miner *Miner) BuildPayload(ctx context.Context, args *BuildPayloadArgs, witness bool) (*Payload, error) { + return miner.buildPayload(ctx, args, witness) } // getPending retrieves the pending block based on the current head block. @@ -156,16 +157,17 @@ func (miner *Miner) getPending() *newPayloadResult { if miner.chainConfig.IsShanghai(new(big.Int).Add(header.Number, big.NewInt(1)), timestamp) { withdrawal = []*types.Withdrawal{} } - ret := miner.generateWork(&generateParams{ - timestamp: timestamp, - forceTime: false, - parentHash: header.Hash(), - coinbase: miner.config.PendingFeeRecipient, - random: common.Hash{}, - withdrawals: withdrawal, - beaconRoot: nil, - noTxs: false, - }, false) // we will never make a witness for a pending block + ret := miner.generateWork(context.Background(), + &generateParams{ + timestamp: timestamp, + forceTime: false, + parentHash: header.Hash(), + coinbase: miner.config.PendingFeeRecipient, + random: common.Hash{}, + withdrawals: withdrawal, + beaconRoot: nil, + noTxs: false, + }, false) // we will never make a witness for a pending block if ret.err != nil { return nil } diff --git a/miner/payload_building.go b/miner/payload_building.go index 8bd9552338..97b4d0c509 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -17,6 +17,7 @@ package miner import ( + "context" "crypto/sha256" "encoding/binary" "math/big" @@ -28,9 +29,11 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "go.opentelemetry.io/otel/trace" ) // BuildPayloadArgs contains the provided parameters for building payload. @@ -101,14 +104,15 @@ func newPayload(empty *types.Block, emptyRequests [][]byte, witness *stateless.W return payload } -// update updates the full-block with latest built version. -func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) { +// update updates the full-block with latest built version. It returns true if +// the update was accepted (i.e. the new block has higher fees than the previous). +func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) (result bool) { payload.lock.Lock() defer payload.lock.Unlock() select { case <-payload.stop: - return // reject stale update + return false // reject stale update default: } // Ensure the newly provided full block has a higher transaction fee. @@ -133,8 +137,10 @@ func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) { "root", r.block.Root(), "elapsed", common.PrettyDuration(elapsed), ) + result = true } payload.cond.Broadcast() // fire signal for notifying full block + return } // Resolve returns the latest built payload and also terminates the background @@ -209,8 +215,33 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { return envelope } +func (miner *Miner) runBuildIteration(ctx context.Context, start time.Time, iteration int, payload *Payload, params *generateParams, witness bool) { + ctx, span, spanEnd := telemetry.StartSpan(ctx, "miner.buildIteration", + telemetry.Int64Attribute("iteration", int64(iteration)), + ) + var err error + defer spanEnd(&err) + + r := miner.generateWork(ctx, params, witness) + err = r.err + if err == nil { + accepted := payload.update(r, time.Since(start)) + span.SetAttributes(telemetry.BoolAttribute("update.accepted", accepted)) + } else { + log.Info("Error while generating work", "id", payload.id, "err", err) + } +} + // buildPayload builds the payload according to the provided parameters. -func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload, error) { +func (miner *Miner) buildPayload(ctx context.Context, args *BuildPayloadArgs, witness bool) (result *Payload, err error) { + payloadID := args.Id() + ctx, _, spanEnd := telemetry.StartSpan(ctx, "miner.buildPayload", + telemetry.StringAttribute("payload.id", payloadID.String()), + telemetry.StringAttribute("parent.hash", args.Parent.String()), + telemetry.Int64Attribute("timestamp", int64(args.Timestamp)), + ) + defer spanEnd(&err) + // Build the initial version with no transaction included. It should be fast // enough to run. The empty payload can at least make sure there is something // to deliver for not missing slot. @@ -225,16 +256,25 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload slotNum: args.SlotNum, noTxs: true, } - empty := miner.generateWork(emptyParams, witness) + empty := miner.generateWork(ctx, emptyParams, witness) if empty.err != nil { return nil, empty.err } // Construct a payload object for return. - payload := newPayload(empty.block, empty.requests, empty.witness, args.Id()) + payload := newPayload(empty.block, empty.requests, empty.witness, payloadID) // Spin up a routine for updating the payload in background. This strategy // can maximum the revenue for including transactions with highest fee. go func() { + var iteration int + bCtx, bSpan, bSpanEnd := telemetry.StartSpan(ctx, "miner.background", + telemetry.Int64Attribute("block.number", int64(empty.block.NumberU64())), + ) + defer func() { + bSpan.SetAttributes(telemetry.Int64Attribute("iterations.total", int64(iteration))) + bSpanEnd(nil) + }() + // Setup the timer for re-building the payload. The initial clock is kept // for triggering process immediately. timer := time.NewTimer(0) @@ -256,7 +296,6 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload slotNum: args.SlotNum, noTxs: false, } - for { select { case <-timer.C: @@ -267,22 +306,21 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload // Check payload.stop first to avoid an unnecessary generateWork. select { case <-payload.stop: + payload.updateSpanForDelivery(bSpan) log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return default: } start := time.Now() - r := miner.generateWork(fullParams, witness) - if r.err == nil { - payload.update(r, time.Since(start)) - } else { - log.Info("Error while generating work", "id", payload.id, "err", r.err) - } + iteration++ + miner.runBuildIteration(bCtx, start, iteration, payload, fullParams, witness) timer.Reset(max(0, miner.config.Recommit-time.Since(start))) case <-payload.stop: + payload.updateSpanForDelivery(bSpan) log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return case <-endTimer.C: + bSpan.SetAttributes(telemetry.StringAttribute("exit.reason", "timeout")) log.Info("Stopping work on payload", "id", payload.id, "reason", "timeout") return } @@ -291,6 +329,16 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload return payload, nil } +func (payload *Payload) updateSpanForDelivery(bSpan trace.Span) { + payload.lock.Lock() + emptyDelivered := payload.full == nil + payload.lock.Unlock() + bSpan.SetAttributes( + telemetry.StringAttribute("exit.reason", "delivery"), + telemetry.BoolAttribute("empty.delivered", emptyDelivered), + ) +} + // BuildTestingPayload is for testing_buildBlockV*. It creates a block with the exact content given // by the parameters instead of using the locally available transactions. func (miner *Miner) BuildTestingPayload(args *BuildPayloadArgs, transactions []*types.Transaction, empty bool, extraData []byte) (*engine.ExecutionPayloadEnvelope, error) { @@ -307,7 +355,7 @@ func (miner *Miner) BuildTestingPayload(args *BuildPayloadArgs, transactions []* overrideExtraData: extraData, overrideTxs: transactions, } - res := miner.generateWork(fullParams, false) + res := miner.generateWork(context.Background(), fullParams, false) if res.err != nil { return nil, res.err } diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index 295962d7ef..f8e495cc99 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -17,6 +17,7 @@ package miner import ( + "context" "math/big" "reflect" "testing" @@ -156,7 +157,7 @@ func TestBuildPayload(t *testing.T) { Random: common.Hash{}, FeeRecipient: recipient, } - payload, err := w.buildPayload(args, false) + payload, err := w.buildPayload(context.Background(), args, false) if err != nil { t.Fatalf("Failed to build payload %v", err) } diff --git a/miner/worker.go b/miner/worker.go index bfeeaa248b..e82f5f6e55 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -17,6 +17,7 @@ package miner import ( + "context" "errors" "fmt" "math/big" @@ -32,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" @@ -125,8 +127,23 @@ type generateParams struct { } // generateWork generates a sealing block based on the given parameters. -func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPayloadResult { - work, err := miner.prepareWork(genParam, witness) +func (miner *Miner) generateWork(ctx context.Context, genParam *generateParams, witness bool) (result *newPayloadResult) { + ctx, span, spanEnd := telemetry.StartSpan(ctx, "miner.generateWork") + defer func() { + if result != nil && result.err == nil { + span.SetAttributes( + telemetry.Int64Attribute("txs.count", int64(len(result.block.Transactions()))), + telemetry.Int64Attribute("gas.used", int64(result.block.GasUsed())), + telemetry.StringAttribute("fees", result.fees.String()), + ) + } + if result != nil { + spanEnd(&result.err) + } else { + spanEnd(nil) + } + }() + work, err := miner.prepareWork(ctx, genParam, witness) if err != nil { return &newPayloadResult{err: err} } @@ -148,7 +165,7 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay if genParam.forceOverrides && len(genParam.overrideTxs) > 0 { for _, tx := range genParam.overrideTxs { work.state.SetTxContext(tx.Hash(), work.tcount) - if err := miner.commitTransaction(work, tx); err != nil { + if err := miner.commitTransaction(ctx, work, tx); err != nil { // all passed transactions HAVE to be valid at this point return &newPayloadResult{err: err} } @@ -160,7 +177,7 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay }) defer timer.Stop() - err := miner.fillTransactions(interrupt, work) + err := miner.fillTransactions(ctx, interrupt, work) if errors.Is(err, errBlockInterruptedByTimeout) { log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) } @@ -195,7 +212,7 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay work.header.RequestsHash = &reqHash } - block, err := miner.engine.FinalizeAndAssemble(miner.chain, work.header, work.state, &body, work.receipts) + block, err := miner.engine.FinalizeAndAssemble(ctx, miner.chain, work.header, work.state, &body, work.receipts) if err != nil { return &newPayloadResult{err: err} } @@ -213,7 +230,9 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay // prepareWork constructs the sealing task according to the given parameters, // either based on the last chain head or specified parent. In this function // the pending transactions are not filled yet, only the empty task returned. -func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*environment, error) { +func (miner *Miner) prepareWork(ctx context.Context, genParams *generateParams, witness bool) (result *environment, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "miner.prepareWork") + defer spanEnd(&err) miner.confMu.RLock() defer miner.confMu.RUnlock() @@ -330,7 +349,9 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase }, nil } -func (miner *Miner) commitTransaction(env *environment, tx *types.Transaction) error { +func (miner *Miner) commitTransaction(ctx context.Context, env *environment, tx *types.Transaction) (err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "miner.commitTransaction") + defer spanEnd(&err) if tx.Type() == types.BlobTxType { return miner.commitBlobTransaction(env, tx) } @@ -389,7 +410,9 @@ func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (* return receipt, nil } -func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { +func (miner *Miner) commitTransactions(ctx context.Context, env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { + ctx, _, spanEnd := telemetry.StartSpan(ctx, "miner.commitTransactions") + defer spanEnd(nil) isCancun := miner.chainConfig.IsCancun(env.header.Number, env.header.Time) for { // Check interruption signal and abort building if it's fired. @@ -479,7 +502,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran // Start executing the transaction env.state.SetTxContext(tx.Hash(), env.tcount) - err := miner.commitTransaction(env, tx) + err := miner.commitTransaction(ctx, env, tx) switch { case errors.Is(err, core.ErrNonceTooLow): // New head notification data race between the transaction pool and miner, shift @@ -503,7 +526,9 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran // fillTransactions retrieves the pending transactions from the txpool and fills them // into the given sealing block. The transaction selection and ordering strategy can // be customized with the plugin in the future. -func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) error { +func (miner *Miner) fillTransactions(ctx context.Context, interrupt *atomic.Int32, env *environment) (err error) { + ctx, span, spanEnd := telemetry.StartSpan(ctx, "miner.fillTransactions") + defer spanEnd(&err) miner.confMu.RLock() tip := miner.config.GasPrice prio := miner.prio @@ -523,7 +548,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) filter.GasLimitCap = params.MaxTxGas } filter.BlobTxs = false - pendingPlainTxs := miner.txpool.Pending(filter) + pendingPlainTxs, plainTxCount := miner.txpool.Pending(filter) filter.BlobTxs = true if miner.chainConfig.IsOsaka(env.header.Number, env.header.Time) { @@ -531,7 +556,11 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) } else { filter.BlobVersion = types.BlobSidecarVersion0 } - pendingBlobTxs := miner.txpool.Pending(filter) + pendingBlobTxs, blobTxCount := miner.txpool.Pending(filter) + span.SetAttributes( + telemetry.Int64Attribute("pending.plain.count", int64(plainTxCount)), + telemetry.Int64Attribute("pending.blob.count", int64(blobTxCount)), + ) // Split the pending transactions into locals and remotes. prioPlainTxs, normalPlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs @@ -552,7 +581,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) plainTxs := newTransactionsByPriceAndNonce(env.signer, prioPlainTxs, env.header.BaseFee) blobTxs := newTransactionsByPriceAndNonce(env.signer, prioBlobTxs, env.header.BaseFee) - if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + if err := miner.commitTransactions(ctx, env, plainTxs, blobTxs, interrupt); err != nil { return err } } @@ -560,7 +589,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) plainTxs := newTransactionsByPriceAndNonce(env.signer, normalPlainTxs, env.header.BaseFee) blobTxs := newTransactionsByPriceAndNonce(env.signer, normalBlobTxs, env.header.BaseFee) - if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { + if err := miner.commitTransactions(ctx, env, plainTxs, blobTxs, interrupt); err != nil { return err } } From 4b915af2c3a0097eac87b06915605f90cc8d14c6 Mon Sep 17 00:00:00 2001 From: CPerezz <37264926+CPerezz@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:42:42 +0100 Subject: [PATCH 39/52] core/state: avoid Bytes() allocation in flatReader hash computations (#34025) ## Summary Replace `addr.Bytes()` and `key.Bytes()` with `addr[:]` and `key[:]` in `flatReader`'s `Account` and `Storage` methods. The former allocates a copy while the latter creates a zero-allocation slice header over the existing backing array. ## Benchmark (AMD EPYC 48-core, 500K entries, screening `--benchtime=1x`) | Metric | Baseline | Slice syntax | Delta | |--------|----------|--------------|-------| | Approve (Mgas/s) | 4.13 | 4.22 | +2.2% | | BalanceOf (Mgas/s) | 168.3 | 190.0 | **+12.9%** | --- core/state/reader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/state/reader.go b/core/state/reader.go index 49375c467c..fe0ec71f2d 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -98,7 +98,7 @@ func newFlatReader(reader database.StateReader) *flatReader { // // The returned account might be nil if it's not existent. func (r *flatReader) Account(addr common.Address) (*types.StateAccount, error) { - account, err := r.reader.Account(crypto.Keccak256Hash(addr.Bytes())) + account, err := r.reader.Account(crypto.Keccak256Hash(addr[:])) if err != nil { return nil, err } @@ -128,8 +128,8 @@ func (r *flatReader) Account(addr common.Address) (*types.StateAccount, error) { // // The returned storage slot might be empty if it's not existent. func (r *flatReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { - addrHash := crypto.Keccak256Hash(addr.Bytes()) - slotHash := crypto.Keccak256Hash(key.Bytes()) + addrHash := crypto.Keccak256Hash(addr[:]) + slotHash := crypto.Keccak256Hash(key[:]) ret, err := r.reader.Storage(addrHash, slotHash) if err != nil { return common.Hash{}, err From 519a450c436970a8319f6c7cf383bf99cbc2c55d Mon Sep 17 00:00:00 2001 From: CPerezz <37264926+CPerezz@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:27:29 +0100 Subject: [PATCH 40/52] core/state: skip redundant trie Commit for Verkle in stateObject.commit (#34021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary **Bug fix.** In Verkle mode, all state objects share a single unified trie (`OpenStorageTrie` returns `self`). During `stateDB.commit()`, the main account trie is committed via `s.trie.Commit(true)`, which calls `CollectNodes` to traverse and serialize the entire tree. However, each dirty account's `obj.commit()` also calls `s.trie.Commit(false)` on the **same trie object**, redundantly traversing and serializing the full tree once per dirty account. With N dirty accounts per block, this causes **N+1 full-tree traversals** instead of 1. On a write-heavy workload (2250 SSTOREs), this produces ~131 GB of allocations per block from duplicate NodeSet creation and serialization. It also causes a latent data race from N+1 goroutines concurrently calling `CollectNodes` on shared `InternalNode` objects. This commit adds an `IsVerkle()` early return in `stateObject.commit()` to skip the redundant `trie.Commit()` call. ## Benchmark (AMD EPYC 48-core, 500K entries, `--benchtime=10s --count=3`) | Metric | Baseline | Fixed | Delta | |--------|----------|-------|-------| | Approve (Mgas/s) | 4.16 ± 0.37 | **220.2 ± 10.1** | **+5190%** | | BalanceOf (Mgas/s) | 966.2 ± 8.1 | 971.0 ± 3.0 | +0.5% | | Allocs/op (approve) | 136.4M | 792K | **-99.4%** | Resolves the TODO in statedb.go about the account trie commit being "very heavy" and "something's wonky". --------- Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- core/state/state_object.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/state/state_object.go b/core/state/state_object.go index dd30bb64a5..ec0c511737 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -474,6 +474,14 @@ func (s *stateObject) commit() (*accountUpdate, *trienode.NodeSet, error) { s.origin = s.data.Copy() return op, nil, nil } + // In Verkle/binary trie mode, all state objects share one unified trie. + // The main account trie commit in stateDB.commit() already calls + // CollectNodes on this trie, so calling Commit here again would + // redundantly traverse and serialize the entire tree per dirty account. + if s.db.GetTrie().IsVerkle() { + s.origin = s.data.Copy() + return op, nil, nil + } root, nodes := s.trie.Commit(false) s.data.Root = root s.origin = s.data.Copy() From fc1b0c0b83027b9e2ee44af6801728ac1e339f05 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:52:04 +0100 Subject: [PATCH 41/52] internal/ethapi: warn on reaching global gas cap for eth_simulateV1 (#34016) Warn user when gas limit of a tx is capped due to rpc server's gas cap being reached. --- internal/ethapi/simulate.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index ebe221114f..aa7609ff93 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -321,7 +321,8 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if err := ctx.Err(); err != nil { return nil, nil, nil, err } - if err := sim.sanitizeCall(&call, sim.state, header, gp); err != nil { + gasCapped, err := sim.sanitizeCall(&call, sim.state, header, gp) + if err != nil { return nil, nil, nil, err } var ( @@ -365,7 +366,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, revertErr := newRevertError(result.Revert()) callRes.Error = &callError{Message: revertErr.Error(), Code: revertErr.ErrorCode(), Data: revertErr.ErrorData().(string)} } else { - callRes.Error = &callError{Message: result.Err.Error(), Code: errCodeVMError} + msg := result.Err.Error() + if gasCapped { + msg += " (gas limit was capped by the RPC server's global gas cap)" + } + callRes.Error = &callError{Message: msg, Code: errCodeVMError} } } else { callRes.Status = hexutil.Uint64(types.ReceiptStatusSuccessful) @@ -425,7 +430,7 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, gp *core.GasPool) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, gp *core.GasPool) (bool, error) { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) @@ -436,13 +441,14 @@ func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, head call.Gas = (*hexutil.Uint64)(&remaining) } if remaining < uint64(*call.Gas) { - return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: remaining: %d, required: %d", remaining, *call.Gas)} + return false, &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: remaining: %d, required: %d", remaining, *call.Gas)} } // Clamp to the cross-block gas budget. gas := sim.budget.cap(uint64(*call.Gas)) + gasCapped := gas < uint64(*call.Gas) call.Gas = (*hexutil.Uint64)(&gas) - return call.CallDefaults(0, header.BaseFee, sim.chainConfig.ChainID) + return gasCapped, call.CallDefaults(0, header.BaseFee, sim.chainConfig.ChainID) } func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContracts { From 9b2ce121dc94fc964f9c8ba6cd6e70af5f11e5d2 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 17 Mar 2026 22:29:30 +0800 Subject: [PATCH 42/52] triedb/pathdb: enhance history index initer (#33640) This PR improves the pbss archive mode. Initial sync of an archive mode which has the --gcmode archive flag enabled will be significantly sped up. It achieves that with the following changes: The indexer now attempts to process histories in batch whenever possible. Batch indexing is enforced when the node is still syncing and the local chain head is behind the network chain head. In this scenario, instead of scheduling indexing frequently alongside block insertion, the indexer waits until a sufficient amount of history has accumulated and then processes it in a batch, which is significantly more efficient. --------- Co-authored-by: Sina M <1591639+s1na@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- triedb/pathdb/config.go | 7 +- triedb/pathdb/database.go | 4 +- triedb/pathdb/database_test.go | 1 + triedb/pathdb/history_indexer.go | 150 +++++++++++--------- triedb/pathdb/history_indexer_state.go | 183 +++++++++++++++++++++++++ triedb/pathdb/history_indexer_test.go | 4 +- 6 files changed, 279 insertions(+), 70 deletions(-) create mode 100644 triedb/pathdb/history_indexer_state.go diff --git a/triedb/pathdb/config.go b/triedb/pathdb/config.go index c236b34333..97ee1c2315 100644 --- a/triedb/pathdb/config.go +++ b/triedb/pathdb/config.go @@ -105,9 +105,10 @@ type Config struct { FullValueCheckpoint uint32 // The rate at which trie nodes are encoded in full-value format // Testing configurations - SnapshotNoBuild bool // Flag Whether the state generation is disabled - NoAsyncFlush bool // Flag whether the background buffer flushing is disabled - NoAsyncGeneration bool // Flag whether the background generation is disabled + SnapshotNoBuild bool // Flag Whether the state generation is disabled + NoAsyncFlush bool // Flag whether the background buffer flushing is disabled + NoAsyncGeneration bool // Flag whether the background generation is disabled + NoHistoryIndexDelay bool // Flag whether the history index delay is disabled } // sanitize checks the provided user configurations and changes anything that's diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 5255602a4e..86a42c69f4 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -215,14 +215,14 @@ func (db *Database) setHistoryIndexer() { if db.stateIndexer != nil { db.stateIndexer.close() } - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) + db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory, db.config.NoHistoryIndexDelay) log.Info("Enabled state history indexing") } if db.trienodeFreezer != nil { if db.trienodeIndexer != nil { db.trienodeIndexer.close() } - db.trienodeIndexer = newHistoryIndexer(db.diskdb, db.trienodeFreezer, db.tree.bottom().stateID(), typeTrienodeHistory) + db.trienodeIndexer = newHistoryIndexer(db.diskdb, db.trienodeFreezer, db.tree.bottom().stateID(), typeTrienodeHistory, db.config.NoHistoryIndexDelay) log.Info("Enabled trienode history indexing") } } diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 2d1819d08f..8ece83cad7 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -182,6 +182,7 @@ func newTester(t *testing.T, config *testerConfig) *tester { WriteBufferSize: config.writeBufferSize(), NoAsyncFlush: true, JournalDirectory: config.journalDir, + NoHistoryIndexDelay: true, }, config.isVerkle) obj = &tester{ diff --git a/triedb/pathdb/history_indexer.go b/triedb/pathdb/history_indexer.go index c987b380ed..c9bf3e87f1 100644 --- a/triedb/pathdb/history_indexer.go +++ b/triedb/pathdb/history_indexer.go @@ -41,6 +41,8 @@ const ( stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version trienodeHistoryIndexV0 = uint8(0) // initial version of trienode index structure trienodeHistoryIndexVersion = trienodeHistoryIndexV0 // the current trienode index version + + indexerProcessBatchInSync = 100000 // threshold for history batch indexing when node is in sync stage. ) // indexVersion returns the latest index version for the given history type. @@ -349,7 +351,8 @@ type interruptSignal struct { // If a state history is removed due to a rollback, the associated indexes should // be unmarked accordingly. type indexIniter struct { - disk ethdb.KeyValueStore + state *initerState + disk ethdb.Database freezer ethdb.AncientStore interrupt chan *interruptSignal done chan struct{} @@ -364,8 +367,9 @@ type indexIniter struct { wg sync.WaitGroup } -func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, typ historyType, lastID uint64) *indexIniter { +func newIndexIniter(disk ethdb.Database, freezer ethdb.AncientStore, typ historyType, lastID uint64, noWait bool) *indexIniter { initer := &indexIniter{ + state: newIniterState(disk, noWait), disk: disk, freezer: freezer, interrupt: make(chan *interruptSignal), @@ -385,12 +389,7 @@ func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, typ hi // Launch background indexer initer.wg.Add(1) - if recover { - log.Info("History indexer is recovering", "history", lastID, "indexed", metadata.Last) - go initer.recover(lastID) - } else { - go initer.run(lastID) - } + go initer.run(recover) return initer } @@ -400,6 +399,7 @@ func (i *indexIniter) close() { return default: close(i.closed) + i.state.close() i.wg.Wait() } } @@ -431,85 +431,109 @@ func (i *indexIniter) remain() uint64 { } } -func (i *indexIniter) run(lastID uint64) { +func (i *indexIniter) run(recover bool) { defer i.wg.Done() // Launch background indexing thread var ( - done = make(chan struct{}) - interrupt = new(atomic.Int32) + done chan struct{} + interrupt *atomic.Int32 - // checkDone indicates whether all requested state histories - // have been fully indexed. + // checkDone reports whether indexing has completed for all histories. checkDone = func() bool { metadata := loadIndexMetadata(i.disk, i.typ) - return metadata != nil && metadata.Last == lastID + return metadata != nil && metadata.Last == i.last.Load() } + // canExit reports whether the initial indexing phase has completed. + canExit = func() bool { + return !i.state.is(stateSyncing) && checkDone() + } + heartBeat = time.NewTimer(0) ) - go i.index(done, interrupt, lastID) + defer heartBeat.Stop() + if recover { + if aborted := i.recover(); aborted { + return + } + } for { select { case signal := <-i.interrupt: - // The indexing limit can only be extended or shortened continuously. newLastID := signal.newLastID - if newLastID != lastID+1 && newLastID != lastID-1 { - signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, newLastID) + oldLastID := i.last.Load() + + // The indexing limit can only be extended or shortened continuously. + if newLastID != oldLastID+1 && newLastID != oldLastID-1 { + signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", oldLastID, newLastID) continue } i.last.Store(newLastID) // update indexing range // The index limit is extended by one, update the limit without // interrupting the current background process. - if newLastID == lastID+1 { - lastID = newLastID + if newLastID == oldLastID+1 { signal.result <- nil - i.log.Debug("Extended history range", "last", lastID) + i.log.Debug("Extended history range", "last", newLastID) continue } - // The index limit is shortened by one, interrupt the current background - // process and relaunch with new target. - interrupt.Store(1) - <-done - + // The index limit is shortened, interrupt the current background + // process if it's active and update the target. + if done != nil { + interrupt.Store(1) + <-done + done, interrupt = nil, nil + } // If all state histories, including the one to be reverted, have // been fully indexed, unindex it here and shut down the initializer. if checkDone() { - i.log.Info("Truncate the extra history", "id", lastID) - if err := unindexSingle(lastID, i.disk, i.freezer, i.typ); err != nil { + i.log.Info("Truncate the extra history", "id", oldLastID) + if err := unindexSingle(oldLastID, i.disk, i.freezer, i.typ); err != nil { signal.result <- err return } close(i.done) signal.result <- nil - i.log.Info("Histories have been fully indexed", "last", lastID-1) + i.log.Info("Histories have been fully indexed", "last", i.last.Load()) return } - // Adjust the indexing target and relaunch the process - lastID = newLastID + // Adjust the indexing target signal.result <- nil - - done, interrupt = make(chan struct{}), new(atomic.Int32) - go i.index(done, interrupt, lastID) - i.log.Debug("Shortened history range", "last", lastID) + i.log.Debug("Shortened history range", "last", newLastID) case <-done: - if checkDone() { + done, interrupt = nil, nil + + if canExit() { close(i.done) - i.log.Info("Histories have been fully indexed", "last", lastID) return } - // Relaunch the background runner if some tasks are left + + case <-heartBeat.C: + heartBeat.Reset(time.Second * 15) + + // Short circuit if the indexer is still busy + if done != nil { + continue + } + if canExit() { + close(i.done) + return + } + // The local chain is still in the syncing phase. Only start the indexing + // when a sufficient amount of histories has accumulated. Batch indexing + // is more efficient than processing items individually. + if i.state.is(stateSyncing) && i.last.Load()-i.indexed.Load() < indexerProcessBatchInSync { + continue + } done, interrupt = make(chan struct{}), new(atomic.Int32) - go i.index(done, interrupt, lastID) + go i.index(done, interrupt, i.last.Load()) case <-i.closed: - interrupt.Store(1) - i.log.Info("Waiting background history index initer to exit") - <-done - - if checkDone() { - close(i.done) + if done != nil { + interrupt.Store(1) + i.log.Info("Waiting background history index initer to exit") + <-done } return } @@ -571,7 +595,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID } return } - i.log.Info("Start history indexing", "beginID", beginID, "lastID", lastID) + i.log.Debug("Start history indexing", "beginID", beginID, "lastID", lastID) var ( current = beginID @@ -618,7 +642,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID done = current - beginID ) eta := common.CalculateETA(done, left, time.Since(start)) - i.log.Info("Indexing history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) + i.log.Debug("Indexing history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) } } i.indexed.Store(current - 1) // update indexing progress @@ -629,7 +653,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID if err := batch.finish(true); err != nil { i.log.Error("Failed to flush index", "err", err) } - log.Info("State indexing interrupted") + log.Debug("State indexing interrupted") return } } @@ -637,7 +661,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID if err := batch.finish(true); err != nil { i.log.Error("Failed to flush index", "err", err) } - i.log.Info("Indexed history", "from", beginID, "to", lastID, "elapsed", common.PrettyDuration(time.Since(start))) + i.log.Debug("Indexed history", "from", beginID, "to", lastID, "elapsed", common.PrettyDuration(time.Since(start))) } // recover handles unclean shutdown recovery. After an unclean shutdown, any @@ -650,35 +674,35 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID // by chain recovery, under the assumption that the recovered histories will be // identical to the lost ones. Fork-awareness should be added in the future to // correctly handle histories affected by reorgs. -func (i *indexIniter) recover(lastID uint64) { - defer i.wg.Done() +func (i *indexIniter) recover() bool { + log.Info("History indexer is recovering", "last", i.last.Load(), "indexed", i.indexed.Load()) for { select { case signal := <-i.interrupt: newLastID := signal.newLastID - if newLastID != lastID+1 && newLastID != lastID-1 { - signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, newLastID) + oldLastID := i.last.Load() + + // The indexing limit can only be extended or shortened continuously. + if newLastID != oldLastID+1 && newLastID != oldLastID-1 { + signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", oldLastID, newLastID) continue } - // Update the last indexed flag - lastID = newLastID signal.result <- nil i.last.Store(newLastID) - i.log.Debug("Updated history index flag", "last", lastID) + i.log.Debug("Updated history index flag", "last", newLastID) // Terminate the recovery routine once the histories are fully aligned // with the index data, indicating that index initialization is complete. metadata := loadIndexMetadata(i.disk, i.typ) - if metadata != nil && metadata.Last == lastID { - close(i.done) - i.log.Info("History indexer is recovered", "last", lastID) - return + if metadata != nil && metadata.Last == newLastID { + i.log.Info("History indexer is recovered", "last", newLastID) + return false } case <-i.closed: - return + return true } } } @@ -746,10 +770,10 @@ func checkVersion(disk ethdb.KeyValueStore, typ historyType) { // newHistoryIndexer constructs the history indexer and launches the background // initer to complete the indexing of any remaining state histories. -func newHistoryIndexer(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastHistoryID uint64, typ historyType) *historyIndexer { +func newHistoryIndexer(disk ethdb.Database, freezer ethdb.AncientStore, lastHistoryID uint64, typ historyType, noWait bool) *historyIndexer { checkVersion(disk, typ) return &historyIndexer{ - initer: newIndexIniter(disk, freezer, typ, lastHistoryID), + initer: newIndexIniter(disk, freezer, typ, lastHistoryID, noWait), typ: typ, disk: disk, freezer: freezer, diff --git a/triedb/pathdb/history_indexer_state.go b/triedb/pathdb/history_indexer_state.go new file mode 100644 index 0000000000..2746083297 --- /dev/null +++ b/triedb/pathdb/history_indexer_state.go @@ -0,0 +1,183 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// state represents the syncing status of the node. +type state int + +const ( + // stateSynced indicates that the local chain head is sufficiently close to the + // network chain head, and the majority of the data has been fully synchronized. + stateSynced state = iota + + // stateSyncing indicates that the sync process is still in progress. Local node + // is actively catching up with the network chain head. + stateSyncing + + // stateStalled indicates that sync progress has stopped for a while + // with no progress. This may be caused by network instability (e.g., no peers), + // manual operation such as syncing the local chain to a specific block. + stateStalled +) + +const ( + // syncStateTimeWindow defines the maximum allowed lag behind the network + // chain head. + // + // If the local chain head falls within this threshold, the node is considered + // close to the tip and will be marked as stateSynced. + syncStateTimeWindow = 6 * time.Hour + + // syncStalledTimeout defines the maximum duration during which no sync + // progress is observed. If this timeout is exceeded, the node's status + // will be considered stalled. + syncStalledTimeout = 5 * time.Minute +) + +type initerState struct { + state state + stateLock sync.Mutex + disk ethdb.Database + term chan struct{} +} + +func newIniterState(disk ethdb.Database, noWait bool) *initerState { + s := &initerState{ + state: stateSyncing, + disk: disk, + term: make(chan struct{}), + } + go s.update(noWait) + return s +} + +func (s *initerState) get() state { + s.stateLock.Lock() + defer s.stateLock.Unlock() + + return s.state +} + +func (s *initerState) is(state state) bool { + return s.get() == state +} + +func (s *initerState) set(state state) { + s.stateLock.Lock() + defer s.stateLock.Unlock() + + s.state = state +} + +func (s *initerState) update(noWait bool) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + headBlock := s.readLastBlock() + if headBlock != nil && time.Since(time.Unix(int64(headBlock.Time), 0)) < syncStateTimeWindow { + s.set(stateSynced) + log.Info("Marked indexing initer as synced") + } else if noWait { + s.set(stateSynced) + log.Info("Marked indexing initer as synced forcibly") + } else { + s.set(stateSyncing) + } + + var ( + hhash = rawdb.ReadHeadHeaderHash(s.disk) + fhash = rawdb.ReadHeadFastBlockHash(s.disk) + bhash = rawdb.ReadHeadBlockHash(s.disk) + skeleton = rawdb.ReadSkeletonSyncStatus(s.disk) + lastProgress = time.Now() + ) + for { + select { + case <-ticker.C: + state := s.get() + if state == stateSynced || state == stateStalled { + continue + } + headBlock := s.readLastBlock() + if headBlock == nil { + continue + } + // State machine: stateSyncing => stateSynced + if time.Since(time.Unix(int64(headBlock.Time), 0)) < syncStateTimeWindow { + s.set(stateSynced) + log.Info("Marked indexing initer as synced") + continue + } + // State machine: stateSyncing => stateStalled + newhhash := rawdb.ReadHeadHeaderHash(s.disk) + newfhash := rawdb.ReadHeadFastBlockHash(s.disk) + newbhash := rawdb.ReadHeadBlockHash(s.disk) + newskeleton := rawdb.ReadSkeletonSyncStatus(s.disk) + hasProgress := newhhash.Cmp(hhash) != 0 || newfhash.Cmp(fhash) != 0 || newbhash.Cmp(bhash) != 0 || !bytes.Equal(newskeleton, skeleton) + + if !hasProgress && time.Since(lastProgress) > syncStalledTimeout { + s.set(stateStalled) + log.Info("Marked indexing initer as stalled") + continue + } + if hasProgress { + hhash = newhhash + fhash = newfhash + bhash = newbhash + skeleton = newskeleton + lastProgress = time.Now() + } + + case <-s.term: + return + } + } +} + +func (s *initerState) close() { + select { + case <-s.term: + default: + close(s.term) + } + return +} + +// readLastBlock returns the local chain head. +func (s *initerState) readLastBlock() *types.Header { + hash := rawdb.ReadHeadBlockHash(s.disk) + if hash == (common.Hash{}) { + return nil + } + number, exists := rawdb.ReadHeaderNumber(s.disk, hash) + if !exists { + return nil + } + return rawdb.ReadHeader(s.disk, hash, number) +} diff --git a/triedb/pathdb/history_indexer_test.go b/triedb/pathdb/history_indexer_test.go index f333d18d8b..8bb1db42da 100644 --- a/triedb/pathdb/history_indexer_test.go +++ b/triedb/pathdb/history_indexer_test.go @@ -27,7 +27,7 @@ import ( // deadlock when the indexer is active. This specifically targets the case where // signal.result must be sent to unblock the caller. func TestHistoryIndexerShortenDeadlock(t *testing.T) { - //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) db := rawdb.NewMemoryDatabase() freezer, _ := rawdb.NewStateFreezer(t.TempDir(), false, false) defer freezer.Close() @@ -38,7 +38,7 @@ func TestHistoryIndexerShortenDeadlock(t *testing.T) { rawdb.WriteStateHistory(freezer, uint64(i+1), h.meta.encode(), accountIndex, storageIndex, accountData, storageData) } // As a workaround, assign a future block to keep the initer running indefinitely - indexer := newHistoryIndexer(db, freezer, 200, typeStateHistory) + indexer := newHistoryIndexer(db, freezer, 200, typeStateHistory, true) defer indexer.close() done := make(chan error, 1) From ab357151da881f0e7dffd30b639c9dcdcdd4cc10 Mon Sep 17 00:00:00 2001 From: felipe Date: Tue, 17 Mar 2026 09:07:28 -0600 Subject: [PATCH 43/52] cmd/evm: don't strip prefixes on requests over t8n (#33997) Found this bug while implementing the Amsterdam changes t8n changes for benchmark test filling in EELS. Prefixes were incorrectly being stripped on requests over t8n and this was leading to `fill` correctly catching hash mismatches on the EELS side for some BAL tests. Though this was caught there, I think this change might as well be cherry-picked there instead and merged to `master`. This PR brings this behavior to parity with EELS for Osaka filling. There are still some quirks with regards to invalid block tests but I did not investigate this further. --- cmd/evm/internal/t8ntool/execution.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index b3fb79bc4a..efe22d36f5 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -365,10 +365,6 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Set requestsHash on block. h := types.CalcRequestsHash(requests) execRs.RequestsHash = &h - for i := range requests { - // remove prefix - requests[i] = requests[i][1:] - } execRs.Requests = requests } From b6115e9a304cb771a3ad8383da3c61a61c6ce6cf Mon Sep 17 00:00:00 2001 From: Mayveskii Date: Wed, 18 Mar 2026 10:43:24 +0300 Subject: [PATCH 44/52] core: fix txLookupLock mutex leak on error returns in reorg() (#34039) --- core/blockchain.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 8df2365072..42a8405ec9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2628,6 +2628,7 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // as the txlookups should be changed atomically, and all subsequent // reads should be blocked until the mutation is complete. bc.txLookupLock.Lock() + defer bc.txLookupLock.Unlock() // Reorg can be executed, start reducing the chain's old blocks and appending // the new blocks @@ -2730,9 +2731,6 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // Reset the tx lookup cache to clear stale txlookup cache. bc.txLookupCache.Purge() - // Release the tx-lookup lock after mutation. - bc.txLookupLock.Unlock() - return nil } From 6138a11c39aa162dd723518d6edba57cd538a867 Mon Sep 17 00:00:00 2001 From: CPerezz <37264926+CPerezz@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:54:23 +0100 Subject: [PATCH 45/52] trie/bintrie: parallelize InternalNode.Hash at shallow tree depths (#34032) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary At tree depths below `log2(NumCPU)` (clamped to [2, 8]), hash the left subtree in a goroutine while hashing the right subtree inline. This exploits available CPU cores for the top levels of the tree where subtree hashing is most expensive. On single-core machines, the parallel path is disabled entirely. Deeper nodes use sequential hashing with the existing `sync.Pool` hasher where goroutine overhead would exceed the hash computation cost. The parallel path uses `sha256.Sum256` with a stack-allocated buffer to avoid pool contention across goroutines. **Safety:** - Left/right subtrees are disjoint — no shared mutable state - `sync.WaitGroup` provides happens-before guarantee for the result - `defer wg.Done()` + `recover()` prevents goroutine panics from crashing the process - `!bt.mustRecompute` early return means clean nodes never enter the parallel path - Hash results are deterministic regardless of computation order — no consensus risk ## Benchmark (AMD EPYC 48-core, 500K entries, `--benchtime=10s --count=3`, post-H01 baseline) | Metric | Baseline | Parallel | Delta | |--------|----------|----------|-------| | Approve (Mgas/s) | 224.5 ± 7.1 | **259.6 ± 2.4** | **+15.6%** | | BalanceOf (Mgas/s) | 982.9 ± 5.1 | 954.3 ± 10.8 | -2.9% (noise, clean nodes skip parallel path) | | Allocs/op (approve) | ~810K | ~700K | -13.6% | --- trie/bintrie/internal_node.go | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index 7ad76aa9db..946203bcfb 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -17,12 +17,33 @@ package bintrie import ( + "crypto/sha256" "errors" "fmt" + "math/bits" + "runtime" + "sync" "github.com/ethereum/go-ethereum/common" ) +// parallelDepth returns the tree depth below which Hash() spawns goroutines. +func parallelDepth() int { + return min(bits.Len(uint(runtime.NumCPU())), 8) +} + +// isDirty reports whether a BinaryNode child needs rehashing. +func isDirty(n BinaryNode) bool { + switch v := n.(type) { + case *InternalNode: + return v.mustRecompute + case *StemNode: + return v.mustRecompute + default: + return false + } +} + func keyToPath(depth int, key []byte) ([]byte, error) { if depth > 31*8 { return nil, errors.New("node too deep") @@ -124,6 +145,29 @@ func (bt *InternalNode) Hash() common.Hash { return bt.hash } + // At shallow depths, parallelize when both children need rehashing: + // hash left subtree in a goroutine, right subtree inline, then combine. + // Skip goroutine overhead when only one child is dirty (common case + // for narrow state updates that touch a single path through the trie). + if bt.depth < parallelDepth() && isDirty(bt.left) && isDirty(bt.right) { + var input [64]byte + var lh common.Hash + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + lh = bt.left.Hash() + }() + rh := bt.right.Hash() + copy(input[32:], rh[:]) + wg.Wait() + copy(input[:32], lh[:]) + bt.hash = sha256.Sum256(input[:]) + bt.mustRecompute = false + return bt.hash + } + + // Deeper nodes: sequential using pooled hasher (goroutine overhead > hash cost) h := newSha256() defer returnSha256(h) if bt.left != nil { From 6ae3f9fa562f28e805e6a5c20f0e42c1efc7d729 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:54:29 +0100 Subject: [PATCH 46/52] core/history: refactor pruning configuration (#34036) This PR introduces a new type HistoryPolicy which captures user intent as opposed to pruning point stored in the blockchain which persists the actual tail of data in the database. It is in preparation for the rolling history expiry feature. It comes with a semantic change: if database was pruned and geth is running without a history mode flag (or explicit keep all flag) geth will emit a warning but continue running as opposed to stopping the world. --- cmd/geth/chaincmd.go | 11 +-- cmd/workload/testsuite.go | 8 ++- core/blockchain.go | 113 ++++++++++--------------------- core/blockchain_test.go | 16 +---- core/history/historymode.go | 81 +++++++++++----------- core/history/historymode_test.go | 58 ++++++++++++++++ eth/backend.go | 9 ++- 7 files changed, 159 insertions(+), 137 deletions(-) create mode 100644 core/history/historymode_test.go diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 7e14ec1c60..1084100f39 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -731,13 +731,16 @@ func pruneHistory(ctx *cli.Context) error { // Determine the prune point based on the history mode. genesisHash := chain.Genesis().Hash() - prunePoint := history.GetPrunePoint(genesisHash, mode) - if prunePoint == nil { + policy, err := history.NewPolicy(mode, genesisHash) + if err != nil { + return err + } + if policy.Target == nil { return fmt.Errorf("prune point for %q not found for this network", mode.String()) } var ( - targetBlock = prunePoint.BlockNumber - targetBlockHash = prunePoint.BlockHash + targetBlock = policy.Target.BlockNumber + targetBlockHash = policy.Target.BlockHash ) // Check the current freezer tail to see if pruning is needed/possible. diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index 80cbd15352..4e33522f1b 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -155,7 +155,9 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber + if p, err := history.NewPolicy(history.KeepPostMerge, params.MainnetGenesisHash); err == nil { + *cfg.historyPruneBlock = p.Target.BlockNumber + } case ctx.Bool(testSepoliaFlag.Name): cfg.fsys = builtinTestFiles if ctx.IsSet(filterQueryFileFlag.Name) { @@ -180,7 +182,9 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber + if p, err := history.NewPolicy(history.KeepPostMerge, params.SepoliaGenesisHash); err == nil { + *cfg.historyPruneBlock = p.Target.BlockNumber + } default: cfg.fsys = os.DirFS(".") cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) diff --git a/core/blockchain.go b/core/blockchain.go index 42a8405ec9..1b45a5ac39 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -194,9 +194,8 @@ type BlockChainConfig struct { SnapshotNoBuild bool // Whether the background generation is allowed SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it - // This defines the cutoff block for history expiry. - // Blocks before this number may be unavailable in the chain database. - ChainHistoryMode history.HistoryMode + // HistoryPolicy defines the chain history pruning intent. + HistoryPolicy history.HistoryPolicy // Misc options NoPrefetch bool // Whether to disable heuristic state prefetching when processing blocks @@ -227,13 +226,13 @@ type BlockChainConfig struct { // Note the returned object is safe to modify! func DefaultConfig() *BlockChainConfig { return &BlockChainConfig{ - TrieCleanLimit: 256, - TrieDirtyLimit: 256, - TrieTimeLimit: 5 * time.Minute, - StateScheme: rawdb.HashScheme, - SnapshotLimit: 256, - SnapshotWait: true, - ChainHistoryMode: history.KeepAll, + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + StateScheme: rawdb.HashScheme, + SnapshotLimit: 256, + SnapshotWait: true, + HistoryPolicy: history.HistoryPolicy{Mode: history.KeepAll}, // Transaction indexing is disabled by default. // This is appropriate for most unit tests. TxLookupLimit: -1, @@ -715,82 +714,44 @@ func (bc *BlockChain) loadLastState() error { // initializeHistoryPruning sets bc.historyPrunePoint. func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { - var ( - freezerTail, _ = bc.db.Tail() - genesisHash = bc.genesisBlock.Hash() - mergePoint = history.MergePrunePoints[genesisHash] - praguePoint = history.PraguePrunePoints[genesisHash] - ) - switch bc.cfg.ChainHistoryMode { - case history.KeepAll: - if freezerTail == 0 { - return nil - } - // The database was pruned somehow, so we need to figure out if it's a known - // configuration or an error. - if mergePoint != nil && freezerTail == mergePoint.BlockNumber { - bc.historyPrunePoint.Store(mergePoint) - return nil - } - if praguePoint != nil && freezerTail == praguePoint.BlockNumber { - bc.historyPrunePoint.Store(praguePoint) - return nil - } - log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) - return errors.New("unexpected database tail") + freezerTail, _ := bc.db.Tail() + policy := bc.cfg.HistoryPolicy - case history.KeepPostMerge: - if mergePoint == nil { - return errors.New("history pruning requested for unknown network") + switch policy.Mode { + case history.KeepAll: + if freezerTail > 0 { + // Database was pruned externally. Record the actual state. + log.Warn("Chain history database is pruned", "tail", freezerTail, "mode", policy.Mode) + bc.historyPrunePoint.Store(&history.PrunePoint{ + BlockNumber: freezerTail, + BlockHash: bc.GetCanonicalHash(freezerTail), + }) } - if freezerTail == 0 && latest != 0 { - log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) - log.Error("Run 'geth prune-history --history.chain postmerge' to prune pre-merge history.") - return errors.New("history pruning requested via configuration") - } - // Check if DB is pruned further than requested (to Prague). - if praguePoint != nil && freezerTail == praguePoint.BlockNumber { - log.Error("Chain history database is pruned to Prague block, but postmerge mode was requested.") - log.Error("History cannot be unpruned. To restore history, use 'geth import-history'.") - log.Error("If you intended to keep post-Prague history, use '--history.chain postprague' instead.") - return errors.New("database pruned beyond requested history mode") - } - if freezerTail > 0 && freezerTail != mergePoint.BlockNumber { - return errors.New("chain history database pruned to unknown block") - } - bc.historyPrunePoint.Store(mergePoint) return nil - case history.KeepPostPrague: - if praguePoint == nil { - return errors.New("history pruning requested for unknown network") - } - // Check if already at the prague prune point. - if freezerTail == praguePoint.BlockNumber { - bc.historyPrunePoint.Store(praguePoint) + case history.KeepPostMerge, history.KeepPostPrague: + target := policy.Target + // Already at the target. + if freezerTail == target.BlockNumber { + bc.historyPrunePoint.Store(target) return nil } - // Check if database needs pruning. - if latest != 0 { - if freezerTail == 0 { - log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) - log.Error("Run 'geth prune-history --history.chain postprague' to prune pre-Prague history.") - return errors.New("history pruning requested via configuration") - } - if mergePoint != nil && freezerTail == mergePoint.BlockNumber { - log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is only pruned to merge block.", bc.cfg.ChainHistoryMode.String())) - log.Error("Run 'geth prune-history --history.chain postprague' to prune pre-Prague history.") - return errors.New("history pruning requested via configuration") - } - log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) - return errors.New("unexpected database tail") + // Database is pruned beyond the target. + if freezerTail > target.BlockNumber { + return fmt.Errorf("database pruned beyond requested history (tail=%d, target=%d)", freezerTail, target.BlockNumber) } - // Fresh database (latest == 0), will sync from prague point. - bc.historyPrunePoint.Store(praguePoint) + // Database needs pruning (freezerTail < target). + if latest != 0 { + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned to the target block.", policy.Mode.String())) + log.Error(fmt.Sprintf("Run 'geth prune-history --history.chain %s' to prune history.", policy.Mode.String())) + return errors.New("history pruning required") + } + // Fresh database (latest == 0), will sync from target point. + bc.historyPrunePoint.Store(target) return nil default: - return fmt.Errorf("invalid history mode: %d", bc.cfg.ChainHistoryMode) + return fmt.Errorf("invalid history mode: %d", policy.Mode) } } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index ce592f0267..d3ca21b2b3 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -4337,26 +4336,13 @@ func TestInsertChainWithCutoff(t *testing.T) { func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) { // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) - // Add a known pruning point for the duration of the test. ghash := genesis.ToBlock().Hash() cutoffBlock := blocks[cutoff-1] - history.PrunePoints[ghash] = &history.PrunePoint{ - BlockNumber: cutoffBlock.NumberU64(), - BlockHash: cutoffBlock.Hash(), - } - defer func() { - delete(history.PrunePoints, ghash) - }() - - // Enable pruning in cache config. - config := DefaultConfig().WithStateScheme(rawdb.PathScheme) - config.ChainHistoryMode = history.KeepPostMerge db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) defer db.Close() - options := DefaultConfig().WithStateScheme(rawdb.PathScheme) - chain, _ := NewBlockChain(db, genesis, beacon.New(ethash.NewFaker()), options) + chain, _ := NewBlockChain(db, genesis, beacon.New(ethash.NewFaker()), DefaultConfig().WithStateScheme(rawdb.PathScheme)) defer chain.Stop() var ( diff --git a/core/history/historymode.go b/core/history/historymode.go index bdaf07826d..1adfe014b2 100644 --- a/core/history/historymode.go +++ b/core/history/historymode.go @@ -77,57 +77,62 @@ func (m *HistoryMode) UnmarshalText(text []byte) error { return nil } +// PrunePoint identifies a specific block for history pruning. type PrunePoint struct { BlockNumber uint64 BlockHash common.Hash } -// MergePrunePoints contains the pre-defined history pruning cutoff blocks for known networks. -// They point to the first post-merge block. Any pruning should truncate *up to* but excluding -// the given block. -var MergePrunePoints = map[common.Hash]*PrunePoint{ - // mainnet - params.MainnetGenesisHash: { - BlockNumber: 15537393, - BlockHash: common.HexToHash("0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286"), +// staticPrunePoints contains the pre-defined history pruning cutoff blocks for +// known networks, keyed by history mode and genesis hash. They point to the first +// block after the respective fork. Any pruning should truncate *up to* but +// excluding the given block. +var staticPrunePoints = map[HistoryMode]map[common.Hash]*PrunePoint{ + KeepPostMerge: { + params.MainnetGenesisHash: { + BlockNumber: 15537393, + BlockHash: common.HexToHash("0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286"), + }, + params.SepoliaGenesisHash: { + BlockNumber: 1450409, + BlockHash: common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"), + }, }, - // sepolia - params.SepoliaGenesisHash: { - BlockNumber: 1450409, - BlockHash: common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"), + KeepPostPrague: { + params.MainnetGenesisHash: { + BlockNumber: 22431084, + BlockHash: common.HexToHash("0x50c8cab760b2948349c590461b166773c45d8f4858cccf5a43025ab2960152e8"), + }, + params.SepoliaGenesisHash: { + BlockNumber: 7836331, + BlockHash: common.HexToHash("0xe6571beb68bf24dbd8a6ba354518996920c55a3f8d8fdca423e391b8ad071f22"), + }, }, } -// PraguePrunePoints contains the pre-defined history pruning cutoff blocks for the Prague -// (Pectra) upgrade. They point to the first post-Prague block. Any pruning should truncate -// *up to* but excluding the given block. -var PraguePrunePoints = map[common.Hash]*PrunePoint{ - // mainnet - first Prague block (May 7, 2025) - params.MainnetGenesisHash: { - BlockNumber: 22431084, - BlockHash: common.HexToHash("0x50c8cab760b2948349c590461b166773c45d8f4858cccf5a43025ab2960152e8"), - }, - // sepolia - first Prague block (March 5, 2025) - params.SepoliaGenesisHash: { - BlockNumber: 7836331, - BlockHash: common.HexToHash("0xe6571beb68bf24dbd8a6ba354518996920c55a3f8d8fdca423e391b8ad071f22"), - }, +// HistoryPolicy describes the configured history pruning strategy. It captures +// user intent as opposed to the actual DB state. +type HistoryPolicy struct { + Mode HistoryMode + // Static prune point for PostMerge/PostPrague, nil otherwise. + Target *PrunePoint } -// PrunePoints is an alias for MergePrunePoints for backward compatibility. -// Deprecated: Use GetPrunePoint or MergePrunePoints directly. -var PrunePoints = MergePrunePoints - -// GetPrunePoint returns the prune point for the given genesis hash and history mode. -// Returns nil if no prune point is defined for the given combination. -func GetPrunePoint(genesisHash common.Hash, mode HistoryMode) *PrunePoint { +// NewPolicy constructs a HistoryPolicy from the given mode and genesis hash. +func NewPolicy(mode HistoryMode, genesisHash common.Hash) (HistoryPolicy, error) { switch mode { - case KeepPostMerge: - return MergePrunePoints[genesisHash] - case KeepPostPrague: - return PraguePrunePoints[genesisHash] + case KeepAll: + return HistoryPolicy{Mode: KeepAll}, nil + + case KeepPostMerge, KeepPostPrague: + point := staticPrunePoints[mode][genesisHash] + if point == nil { + return HistoryPolicy{}, fmt.Errorf("%s history pruning not available for network %s", mode, genesisHash.Hex()) + } + return HistoryPolicy{Mode: mode, Target: point}, nil + default: - return nil + return HistoryPolicy{}, fmt.Errorf("invalid history mode: %d", mode) } } diff --git a/core/history/historymode_test.go b/core/history/historymode_test.go new file mode 100644 index 0000000000..87eae188dd --- /dev/null +++ b/core/history/historymode_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package history + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" +) + +func TestNewPolicy(t *testing.T) { + // KeepAll: no target. + p, err := NewPolicy(KeepAll, params.MainnetGenesisHash) + if err != nil { + t.Fatalf("KeepAll: %v", err) + } + if p.Mode != KeepAll || p.Target != nil { + t.Errorf("KeepAll: unexpected policy %+v", p) + } + + // PostMerge: resolves known mainnet prune point. + p, err = NewPolicy(KeepPostMerge, params.MainnetGenesisHash) + if err != nil { + t.Fatalf("PostMerge: %v", err) + } + if p.Target == nil || p.Target.BlockNumber != 15537393 { + t.Errorf("PostMerge: unexpected target %+v", p.Target) + } + + // PostPrague: resolves known mainnet prune point. + p, err = NewPolicy(KeepPostPrague, params.MainnetGenesisHash) + if err != nil { + t.Fatalf("PostPrague: %v", err) + } + if p.Target == nil || p.Target.BlockNumber != 22431084 { + t.Errorf("PostPrague: unexpected target %+v", p.Target) + } + + // PostMerge on unknown network: error. + if _, err = NewPolicy(KeepPostMerge, common.HexToHash("0xdeadbeef")); err == nil { + t.Fatal("PostMerge unknown network: expected error") + } +} diff --git a/eth/backend.go b/eth/backend.go index 72228614f0..e9bea59734 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/txpool" @@ -175,7 +176,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Here we determine genesis hash and active ChainConfig. // We need these to figure out the consensus parameters and to set up history pruning. - chainConfig, _, err := core.LoadChainConfig(chainDb, config.Genesis) + chainConfig, genesisHash, err := core.LoadChainConfig(chainDb, config.Genesis) if err != nil { return nil, err } @@ -220,6 +221,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } } + histPolicy, err := history.NewPolicy(config.HistoryMode, genesisHash) + if err != nil { + return nil, err + } var ( options = &core.BlockChainConfig{ TrieCleanLimit: config.TrieCleanCache, @@ -233,7 +238,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrienodeHistory: config.TrienodeHistory, NodeFullValueCheckpoint: config.NodeFullValueCheckpoint, StateScheme: scheme, - ChainHistoryMode: config.HistoryMode, + HistoryPolicy: histPolicy, TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, From b35645bdf7dfb2f0a22f14e8d278b9ec3cb1d48b Mon Sep 17 00:00:00 2001 From: haoyu-haoyu <85037553+haoyu-haoyu@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:56:26 +0000 Subject: [PATCH 47/52] build: fix missing '!' in shebang of generated oss-fuzz scripts (#34044) \`oss-fuzz.sh\` line 38 writes \`#/bin/sh\` instead of \`#!/bin/sh\` as the shebang of generated fuzz test runner scripts. \`\`\`diff -#/bin/sh +#!/bin/sh \`\`\` Without the \`!\`, the kernel does not recognize the interpreter directive. Co-authored-by: Claude Opus 4.6 --- oss-fuzz.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss-fuzz.sh b/oss-fuzz.sh index bd87665125..73209fd8c8 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -35,7 +35,7 @@ function coverbuild { sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go cat << DOG > $OUT/$fuzzer -#/bin/sh +#!/bin/sh cd $OUT/$path go test -run Test${function}Corpus -v $tags -coverprofile \$1 -coverpkg $coverpkg From 3341d8ace0dd85cb5aa90548269e307439ef8b35 Mon Sep 17 00:00:00 2001 From: vickkkkkyy Date: Thu, 19 Mar 2026 06:31:40 +0800 Subject: [PATCH 48/52] eth/filters: rangeLogs should error on invalid block range (#33763) Fixes log filter to reject out of order block ranges. --- eth/filters/filter.go | 2 +- eth/filters/filter_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 9915f28128..04e11f0475 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -390,7 +390,7 @@ func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([ } if firstBlock > lastBlock { - return nil, nil + return nil, errInvalidBlockRange } mb := f.sys.backend.NewMatcherBackend() defer mb.Close() diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 63727200f7..e7b1b08046 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -357,7 +357,8 @@ func testFilters(t *testing.T, history uint64, noHistory bool) { want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`, }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0), + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0), + err: errInvalidBlockRange.Error(), }, { f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0), From 4faadf17fbc29d7890089acc660d553be454067a Mon Sep 17 00:00:00 2001 From: Bosul Mun Date: Thu, 19 Mar 2026 17:51:03 +0900 Subject: [PATCH 49/52] rlp: add AppendList method to RawList (#34048) This the AppendList method to merge two RawList instances by appending the raw content. --- rlp/raw.go | 12 ++++++++++++ rlp/raw_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/rlp/raw.go b/rlp/raw.go index 08ec667158..5f41cad5c4 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -168,6 +168,18 @@ func (r *RawList[T]) AppendRaw(b []byte) error { return nil } +// AppendList appends all items from another RawList to this list. +func (r *RawList[T]) AppendList(other *RawList[T]) { + if other.enc == nil || other.length == 0 { + return + } + if r.enc == nil { + r.enc = make([]byte, 9) + } + r.enc = append(r.enc, other.Content()...) + r.length += other.length +} + // StringSize returns the encoded size of a string. func StringSize(s string) uint64 { switch n := len(s); n { diff --git a/rlp/raw_test.go b/rlp/raw_test.go index 112c5d7897..ed7d3524c2 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -246,6 +246,54 @@ func TestRawListAppendRaw(t *testing.T) { t.Fatalf("wrong Len %d after invalid appends, want 2", rl.Len()) } } +func TestRawListAppendList(t *testing.T) { + var rl1 RawList[uint64] + if err := rl1.Append(uint64(1)); err != nil { + t.Fatal("append 1 failed:", err) + } + if err := rl1.Append(uint64(2)); err != nil { + t.Fatal("append 2 failed:", err) + } + + var rl2 RawList[uint64] + if err := rl2.Append(uint64(3)); err != nil { + t.Fatal("append 3 failed:", err) + } + if err := rl2.Append(uint64(4)); err != nil { + t.Fatal("append 4 failed:", err) + } + + rl1.AppendList(&rl2) + + if rl1.Len() != 4 { + t.Fatalf("wrong Len %d, want 4", rl1.Len()) + } + if rl1.Size() != 5 { + t.Fatalf("wrong Size %d, want 5", rl1.Size()) + } + + items, err := rl1.Items() + if err != nil { + t.Fatal("Items failed:", err) + } + if !reflect.DeepEqual(items, []uint64{1, 2, 3, 4}) { + t.Fatalf("wrong items: %v", items) + } + + var empty RawList[uint64] + prevLen := rl1.Len() + rl1.AppendList(&empty) + + if rl1.Len() != prevLen { + t.Fatalf("appending empty list changed Len: got %d, want %d", rl1.Len(), prevLen) + } + + empty.AppendList(&rl1) + + if empty.Len() != 4 { + t.Fatalf("wrong Len %d, want 4", empty.Len()) + } +} func TestRawListDecodeInvalid(t *testing.T) { tests := []struct { From a3083ff5d0fdc8dec370a421ca4a7ad876e4fe08 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 19 Mar 2026 16:52:10 +0800 Subject: [PATCH 50/52] cmd: add support for enumerating a single storage trie (#34051) --- cmd/geth/snapshot.go | 239 +++++++++++++++++++++++++++++++------------ cmd/utils/flags.go | 4 + 2 files changed, 179 insertions(+), 64 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index fc0658a59c..c177fb5ea2 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" "github.com/urfave/cli/v2" ) @@ -105,7 +106,9 @@ information about the specified address. Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", Action: traverseState, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{ + utils.AccountFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), Description: ` geth snapshot traverse-state will traverse the whole state from the given state root and will abort if any @@ -113,6 +116,8 @@ referenced trie node or contract code is missing. This command can be used for state integrity verification. The default checking target is the HEAD state. It's also usable without snapshot enabled. + +If --account is specified, only the storage trie of that account is traversed. `, }, { @@ -120,7 +125,9 @@ It's also usable without snapshot enabled. Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", Action: traverseRawState, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{ + utils.AccountFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), Description: ` geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced @@ -129,6 +136,8 @@ verification. The default checking target is the HEAD state. It's basically iden to traverse-state, but the check granularity is smaller. It's also usable without snapshot enabled. + +If --account is specified, only the storage trie of that account is traversed. `, }, { @@ -272,6 +281,120 @@ func checkDanglingStorage(ctx *cli.Context) error { return snapshot.CheckDanglingStorage(db) } +// parseAccount parses the account flag value as either an address (20 bytes) +// or an account hash (32 bytes) and returns the hashed account key. +func parseAccount(input string) (common.Hash, error) { + switch len(input) { + case 40, 42: // address + return crypto.Keccak256Hash(common.HexToAddress(input).Bytes()), nil + case 64, 66: // hash + return common.HexToHash(input), nil + default: + return common.Hash{}, errors.New("malformed account address or hash") + } +} + +// lookupAccount resolves the account from the state trie using the given +// account hash. +func lookupAccount(accountHash common.Hash, tr *trie.Trie) (*types.StateAccount, error) { + accData, err := tr.Get(accountHash.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to get account %s: %w", accountHash, err) + } + if accData == nil { + return nil, fmt.Errorf("account not found: %s", accountHash) + } + var acc types.StateAccount + if err := rlp.DecodeBytes(accData, &acc); err != nil { + return nil, fmt.Errorf("invalid account data %s: %w", accountHash, err) + } + return &acc, nil +} + +func traverseStorage(id *trie.ID, db *triedb.Database, report bool, detail bool) error { + tr, err := trie.NewStateTrie(id, db) + if err != nil { + log.Error("Failed to open storage trie", "account", id.Owner, "root", id.Root, "err", err) + return err + } + var ( + slots int + nodes int + lastReport time.Time + start = time.Now() + ) + it, err := tr.NodeIterator(nil) + if err != nil { + log.Error("Failed to open storage iterator", "account", id.Owner, "root", id.Root, "err", err) + return err + } + logger := log.Debug + if report { + logger = log.Info + } + logger("Start traversing storage trie", "account", id.Owner, "storageRoot", id.Root) + + if !detail { + iter := trie.NewIterator(it) + for iter.Next() { + slots += 1 + if time.Since(lastReport) > time.Second*8 { + logger("Traversing storage", "account", id.Owner, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + if iter.Err != nil { + log.Error("Failed to traverse storage trie", "root", id.Root, "err", iter.Err) + return iter.Err + } + logger("Storage is complete", "account", id.Owner, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + } else { + reader, err := db.NodeReader(id.StateRoot) + if err != nil { + log.Error("Failed to open state reader", "err", err) + return err + } + var ( + buffer = make([]byte, 32) + hasher = crypto.NewKeccakState() + ) + for it.Next(true) { + nodes += 1 + node := it.Hash() + + // Check the presence for non-empty hash node(embedded node doesn't + // have their own hash). + if node != (common.Hash{}) { + blob, _ := reader.Node(id.Owner, it.Path(), node) + if len(blob) == 0 { + log.Error("Missing trie node(storage)", "hash", node) + return errors.New("missing storage") + } + hasher.Reset() + hasher.Write(blob) + hasher.Read(buffer) + if !bytes.Equal(buffer, node.Bytes()) { + log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) + return errors.New("invalid storage node") + } + } + if it.Leaf() { + slots += 1 + } + if time.Since(lastReport) > time.Second*8 { + logger("Traversing storage", "account", id.Owner, "nodes", nodes, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + } + if err := it.Error(); err != nil { + log.Error("Failed to traverse storage trie", "root", id.Root, "err", err) + return err + } + logger("Storage is complete", "account", id.Owner, "nodes", nodes, "slots", slots, "elapsed", common.PrettyDuration(time.Since(start))) + } + return nil +} + // traverseState is a helper function used for pruning verification. // Basically it just iterates the trie, ensure all nodes and associated // contract codes are present. @@ -309,6 +432,30 @@ func traverseState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } + // If --account is specified, only traverse the storage trie of that account. + if accountStr := ctx.String(utils.AccountFlag.Name); accountStr != "" { + accountHash, err := parseAccount(accountStr) + if err != nil { + log.Error("Failed to parse account", "err", err) + return err + } + // Use raw trie since the account key is already hashed. + t, err := trie.New(trie.StateTrieID(root), triedb) + if err != nil { + log.Error("Failed to open state trie", "root", root, "err", err) + return err + } + acc, err := lookupAccount(accountHash, t) + if err != nil { + log.Error("Failed to look up account", "hash", accountHash, "err", err) + return err + } + if acc.Root == types.EmptyRootHash { + log.Info("Account has no storage", "hash", accountHash) + return nil + } + return traverseStorage(trie.StorageTrieID(root, accountHash, acc.Root), triedb, true, false) + } t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -335,30 +482,10 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != types.EmptyRootHash { - id := trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root) - storageTrie, err := trie.NewStateTrie(id, triedb) + err := traverseStorage(trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root), triedb, false, false) if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err } - storageIt, err := storageTrie.NodeIterator(nil) - if err != nil { - log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) - return err - } - storageIter := trie.NewIterator(storageIt) - for storageIter.Next() { - slots += 1 - - if time.Since(lastReport) > time.Second*8 { - log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) - lastReport = time.Now() - } - } - if storageIter.Err != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err) - return storageIter.Err - } } if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) { if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { @@ -418,6 +545,30 @@ func traverseRawState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } + // If --account is specified, only traverse the storage trie of that account. + if accountStr := ctx.String(utils.AccountFlag.Name); accountStr != "" { + accountHash, err := parseAccount(accountStr) + if err != nil { + log.Error("Failed to parse account", "err", err) + return err + } + // Use raw trie since the account key is already hashed. + t, err := trie.New(trie.StateTrieID(root), triedb) + if err != nil { + log.Error("Failed to open state trie", "root", root, "err", err) + return err + } + acc, err := lookupAccount(accountHash, t) + if err != nil { + log.Error("Failed to look up account", "hash", accountHash, "err", err) + return err + } + if acc.Root == types.EmptyRootHash { + log.Info("Account has no storage", "hash", accountHash) + return nil + } + return traverseStorage(trie.StorageTrieID(root, accountHash, acc.Root), triedb, true, true) + } t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -473,50 +624,10 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != types.EmptyRootHash { - id := trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root) - storageTrie, err := trie.NewStateTrie(id, triedb) + err := traverseStorage(trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root), triedb, false, true) if err != nil { - log.Error("Failed to open storage trie", "root", acc.Root, "err", err) - return errors.New("missing storage trie") - } - storageIter, err := storageTrie.NodeIterator(nil) - if err != nil { - log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) return err } - for storageIter.Next(true) { - nodes += 1 - node := storageIter.Hash() - - // Check the presence for non-empty hash node(embedded node doesn't - // have their own hash). - if node != (common.Hash{}) { - blob, _ := reader.Node(common.BytesToHash(accIter.LeafKey()), storageIter.Path(), node) - if len(blob) == 0 { - log.Error("Missing trie node(storage)", "hash", node) - return errors.New("missing storage") - } - hasher.Reset() - hasher.Write(blob) - hasher.Read(got) - if !bytes.Equal(got, node.Bytes()) { - log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) - return errors.New("invalid storage node") - } - } - // Bump the counter if it's leaf node. - if storageIter.Leaf() { - slots += 1 - } - if time.Since(lastReport) > time.Second*8 { - log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) - lastReport = time.Now() - } - } - if storageIter.Error() != nil { - log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error()) - return storageIter.Error() - } } if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) { if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 792e0e55ab..3a0bcc6b05 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -218,6 +218,10 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } + AccountFlag = &cli.StringFlag{ + Name: "account", + Usage: "Specifies the account address or hash to traverse a single storage trie", + } OutputFileFlag = &cli.StringFlag{ Name: "output", Usage: "Writes the result in json to the output", From fd859638bd76d15b15468c9b2dea601035779769 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Thu, 19 Mar 2026 12:02:49 -0400 Subject: [PATCH 51/52] core/vm: rework gas measurement for call variants (#33648) EIP-7928 brings state reads into consensus by recording accounts and storage accessed during execution in the block access list. As part of the spec, we need to check that there is enough gas available to cover the cost component which doesn't depend on looking up state. If this component can't be covered by the available gas, we exit immediately. The portion of the call dynamic cost which doesn't depend on state look ups: - EIP2929 call costs - value transfer cost - memory expansion cost This PR: - breaks up the "inner" gas calculation for each call variant into a pair of stateless/stateful cost methods - modifies the gas calculation logic of calls to check stateless cost component first, and go out of gas immediately if it is not covered. --------- Co-authored-by: Gary Rong --- core/vm/gas.go | 1 - core/vm/gas_table.go | 106 ++++++++++++++++++-------------------- core/vm/operations_acl.go | 91 ++++++++++++++++++++------------ 3 files changed, 108 insertions(+), 90 deletions(-) diff --git a/core/vm/gas.go b/core/vm/gas.go index 5fe589bce6..dcb20893c5 100644 --- a/core/vm/gas.go +++ b/core/vm/gas.go @@ -49,6 +49,5 @@ func callGas(isEip150 bool, availableGas, base uint64, callCost *uint256.Int) (u if !callCost.IsUint64() { return 0, ErrGasUintOverflow } - return callCost.Uint64(), nil } diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index aa1ad918bb..f075a99468 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -373,7 +373,32 @@ func gasExpEIP158(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memor return gas, nil } -func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { +var ( + gasCall = makeCallVariantGasCost(gasCallIntrinsic) + gasCallCode = makeCallVariantGasCost(gasCallCodeIntrinsic) + gasDelegateCall = makeCallVariantGasCost(gasDelegateCallIntrinsic) + gasStaticCall = makeCallVariantGasCost(gasStaticCallIntrinsic) +) + +func makeCallVariantGasCost(intrinsicFunc gasFunc) gasFunc { + return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + intrinsic, err := intrinsicFunc(evm, contract, stack, mem, memorySize) + if err != nil { + return 0, err + } + evm.callGasTemp, err = callGas(evm.chainRules.IsEIP150, contract.Gas, intrinsic, stack.Back(0)) + if err != nil { + return 0, err + } + gas, overflow := math.SafeAdd(intrinsic, evm.callGasTemp) + if overflow { + return 0, ErrGasUintOverflow + } + return gas, nil + } +} + +func gasCallIntrinsic(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { var ( gas uint64 transfersValue = !stack.Back(2).IsZero() @@ -382,38 +407,40 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize if evm.readOnly && transfersValue { return 0, ErrWriteProtection } - - if evm.chainRules.IsEIP158 { - if transfersValue && evm.StateDB.Empty(address) { - gas += params.CallNewAccountGas - } - } else if !evm.StateDB.Exist(address) { - gas += params.CallNewAccountGas - } - if transfersValue && !evm.chainRules.IsEIP4762 { - gas += params.CallValueTransferGas - } + // Stateless check memoryGas, err := memoryGasCost(mem, memorySize) if err != nil { return 0, err } + var transferGas uint64 + if transfersValue && !evm.chainRules.IsEIP4762 { + transferGas = params.CallValueTransferGas + } var overflow bool - if gas, overflow = math.SafeAdd(gas, memoryGas); overflow { + if gas, overflow = math.SafeAdd(memoryGas, transferGas); overflow { return 0, ErrGasUintOverflow } - - evm.callGasTemp, err = callGas(evm.chainRules.IsEIP150, contract.Gas, gas, stack.Back(0)) - if err != nil { - return 0, err + // Terminate the gas measurement if the leftover gas is not sufficient, + // it can effectively prevent accessing the states in the following steps. + if contract.Gas < gas { + return 0, ErrOutOfGas } - if gas, overflow = math.SafeAdd(gas, evm.callGasTemp); overflow { + // Stateful check + var stateGas uint64 + if evm.chainRules.IsEIP158 { + if transfersValue && evm.StateDB.Empty(address) { + stateGas += params.CallNewAccountGas + } + } else if !evm.StateDB.Exist(address) { + stateGas += params.CallNewAccountGas + } + if gas, overflow = math.SafeAdd(gas, stateGas); overflow { return 0, ErrGasUintOverflow } - return gas, nil } -func gasCallCode(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { +func gasCallCodeIntrinsic(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { memoryGas, err := memoryGasCost(mem, memorySize) if err != nil { return 0, err @@ -428,46 +455,15 @@ func gasCallCode(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memory if gas, overflow = math.SafeAdd(gas, memoryGas); overflow { return 0, ErrGasUintOverflow } - evm.callGasTemp, err = callGas(evm.chainRules.IsEIP150, contract.Gas, gas, stack.Back(0)) - if err != nil { - return 0, err - } - if gas, overflow = math.SafeAdd(gas, evm.callGasTemp); overflow { - return 0, ErrGasUintOverflow - } return gas, nil } -func gasDelegateCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { - gas, err := memoryGasCost(mem, memorySize) - if err != nil { - return 0, err - } - evm.callGasTemp, err = callGas(evm.chainRules.IsEIP150, contract.Gas, gas, stack.Back(0)) - if err != nil { - return 0, err - } - var overflow bool - if gas, overflow = math.SafeAdd(gas, evm.callGasTemp); overflow { - return 0, ErrGasUintOverflow - } - return gas, nil +func gasDelegateCallIntrinsic(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return memoryGasCost(mem, memorySize) } -func gasStaticCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { - gas, err := memoryGasCost(mem, memorySize) - if err != nil { - return 0, err - } - evm.callGasTemp, err = callGas(evm.chainRules.IsEIP150, contract.Gas, gas, stack.Back(0)) - if err != nil { - return 0, err - } - var overflow bool - if gas, overflow = math.SafeAdd(gas, evm.callGasTemp); overflow { - return 0, ErrGasUintOverflow - } - return gas, nil +func gasStaticCallIntrinsic(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return memoryGasCost(mem, memorySize) } func gasSelfdestruct(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index ce394d9384..addd2b162f 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -256,10 +256,10 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { } var ( - innerGasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall) - gasDelegateCallEIP7702 = makeCallVariantGasCallEIP7702(gasDelegateCall) - gasStaticCallEIP7702 = makeCallVariantGasCallEIP7702(gasStaticCall) - gasCallCodeEIP7702 = makeCallVariantGasCallEIP7702(gasCallCode) + innerGasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCallIntrinsic) + gasDelegateCallEIP7702 = makeCallVariantGasCallEIP7702(gasDelegateCallIntrinsic) + gasStaticCallEIP7702 = makeCallVariantGasCallEIP7702(gasStaticCallIntrinsic) + gasCallCodeEIP7702 = makeCallVariantGasCallEIP7702(gasCallCodeIntrinsic) ) func gasCallEIP7702(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { @@ -274,62 +274,85 @@ func gasCallEIP7702(evm *EVM, contract *Contract, stack *Stack, mem *Memory, mem return innerGasCallEIP7702(evm, contract, stack, mem, memorySize) } -func makeCallVariantGasCallEIP7702(oldCalculator gasFunc) gasFunc { +func makeCallVariantGasCallEIP7702(intrinsicFunc gasFunc) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { var ( - total uint64 // total dynamic gas used - addr = common.Address(stack.Back(1).Bytes20()) + eip2929Cost uint64 + eip7702Cost uint64 + addr = common.Address(stack.Back(1).Bytes20()) ) - - // Check slot presence in the access list + // Perform EIP-2929 checks (stateless), checking address presence + // in the accessList and charge the cold access accordingly. if !evm.StateDB.AddressInAccessList(addr) { evm.StateDB.AddAddressToAccessList(addr) - // The WarmStorageReadCostEIP2929 (100) is already deducted in the form of a constant cost, so - // the cost to charge for cold access, if any, is Cold - Warm - coldCost := params.ColdAccountAccessCostEIP2929 - params.WarmStorageReadCostEIP2929 - // Charge the remaining difference here already, to correctly calculate available - // gas for call - if !contract.UseGas(coldCost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { + + // The WarmStorageReadCostEIP2929 (100) is already deducted in the form + // of a constant cost, so the cost to charge for cold access, if any, + // is Cold - Warm + eip2929Cost = params.ColdAccountAccessCostEIP2929 - params.WarmStorageReadCostEIP2929 + + // Charge the remaining difference here already, to correctly calculate + // available gas for call + if !contract.UseGas(eip2929Cost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { return 0, ErrOutOfGas } - total += coldCost + } + + // Perform the intrinsic cost calculation including: + // + // - transfer value + // - memory expansion + // - create new account + intrinsicCost, err := intrinsicFunc(evm, contract, stack, mem, memorySize) + if err != nil { + return 0, err + } + // Terminate the gas measurement if the leftover gas is not sufficient, + // it can effectively prevent accessing the states in the following steps. + // It's an essential safeguard before any stateful check. + if contract.Gas < intrinsicCost { + return 0, ErrOutOfGas } // Check if code is a delegation and if so, charge for resolution. if target, ok := types.ParseDelegation(evm.StateDB.GetCode(addr)); ok { - var cost uint64 if evm.StateDB.AddressInAccessList(target) { - cost = params.WarmStorageReadCostEIP2929 + eip7702Cost = params.WarmStorageReadCostEIP2929 } else { evm.StateDB.AddAddressToAccessList(target) - cost = params.ColdAccountAccessCostEIP2929 + eip7702Cost = params.ColdAccountAccessCostEIP2929 } - if !contract.UseGas(cost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { + if !contract.UseGas(eip7702Cost, evm.Config.Tracer, tracing.GasChangeCallStorageColdAccess) { return 0, ErrOutOfGas } - total += cost } - - // Now call the old calculator, which takes into account - // - create new account - // - transfer value - // - memory expansion - // - 63/64ths rule - old, err := oldCalculator(evm, contract, stack, mem, memorySize) + // Calculate the gas budget for the nested call. The costs defined by + // EIP-2929 and EIP-7702 have already been applied. + evm.callGasTemp, err = callGas(evm.chainRules.IsEIP150, contract.Gas, intrinsicCost, stack.Back(0)) if err != nil { - return old, err + return 0, err } - // Temporarily add the gas charge back to the contract and return value. By // adding it to the return, it will be charged outside of this function, as // part of the dynamic gas. This will ensure it is correctly reported to // tracers. - contract.Gas += total + contract.Gas += eip2929Cost + eip7702Cost - var overflow bool - if total, overflow = math.SafeAdd(old, total); overflow { + // Aggregate the gas costs from all components, including EIP-2929, EIP-7702, + // the CALL opcode itself, and the cost incurred by nested calls. + var ( + overflow bool + totalCost uint64 + ) + if totalCost, overflow = math.SafeAdd(eip2929Cost, eip7702Cost); overflow { return 0, ErrGasUintOverflow } - return total, nil + if totalCost, overflow = math.SafeAdd(totalCost, intrinsicCost); overflow { + return 0, ErrGasUintOverflow + } + if totalCost, overflow = math.SafeAdd(totalCost, evm.callGasTemp); overflow { + return 0, ErrGasUintOverflow + } + return totalCost, nil } } From 35b91092c5b75399b15354da23f9433c574ce3bc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 19 Mar 2026 18:26:00 +0100 Subject: [PATCH 52/52] rlp: add Size method to EncoderBuffer (#34052) The new method returns the size of the written data, excluding any unfinished list structure. --- rlp/encbuffer.go | 10 ++++++++++ rlp/encode_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go index 61d8bd059c..ca0aa290fe 100644 --- a/rlp/encbuffer.go +++ b/rlp/encbuffer.go @@ -366,6 +366,16 @@ func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { return out } +// Size returns the total size of the content that was encoded up to this point. +// Note this does not count the size of any lists which are still 'open' (i.e. for +// which ListEnd has not been called yet). +func (w EncoderBuffer) Size() int { + if w.buf == nil { + return 0 + } + return w.buf.size() +} + // Write appends b directly to the encoder output. func (w EncoderBuffer) Write(b []byte) (int, error) { return w.buf.Write(b) diff --git a/rlp/encode_test.go b/rlp/encode_test.go index e63ea319b4..8dc9fdaf1f 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -507,6 +507,39 @@ func TestEncodeToReaderReturnToPool(t *testing.T) { wg.Wait() } +func TestEncoderBufferSize(t *testing.T) { + var output bytes.Buffer + eb := NewEncoderBuffer(&output) + + assertSize := func(state string, expectedSize int) { + t.Helper() + if s := eb.Size(); s != expectedSize { + t.Fatalf("wrong size %s: %d", state, s) + } + } + + assertSize("empty buffer", 0) + outerList := eb.List() + assertSize("after outer List()", 0) + eb.WriteString("abc") + assertSize("after string write", 4) + innerList := eb.List() + assertSize("after inner List()", 4) + eb.WriteUint64(1) + eb.WriteUint64(2) + assertSize("after inner list writes", 6) + eb.ListEnd(innerList) + assertSize("after end of inner list", 7) + eb.ListEnd(outerList) + assertSize("after end of outer list", 8) + eb.Flush() + assertSize("after Flush()", 0) + + if output.Len() != 8 { + t.Fatalf("wrong final output size %d", output.Len()) + } +} + var sink interface{} func BenchmarkIntsize(b *testing.B) {