diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 7ea6e578cc..f595a9bf4d 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -152,7 +151,6 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, gasUsed = uint64(0) blobGasUsed = uint64(0) receipts = make(types.Receipts, 0) - txIndex = 0 ) gaspool.AddGas(pre.Env.GasLimit) vmContext := vm.BlockContext{ @@ -250,24 +248,17 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, continue } } - statedb.SetTxContext(tx.Hash(), txIndex) + statedb.SetTxContext(tx.Hash(), len(receipts)) var ( snapshot = statedb.Snapshot() prevGas = gaspool.Gas() ) - if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxStart != nil { - evm.Config.Tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - } - // (ret []byte, usedGas uint64, failed bool, err error) - msgResult, err := core.ApplyMessage(evm, msg, gaspool) + receipt, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, &gasUsed, evm) if err != nil { statedb.RevertToSnapshot(snapshot) log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) gaspool.SetGas(prevGas) - if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxEnd != nil { - evm.Config.Tracer.OnTxEnd(nil, err) - } continue } includedTxs = append(includedTxs, tx) @@ -275,50 +266,11 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return nil, nil, nil, NewError(ErrorMissingBlockhash, hashError) } blobGasUsed += txBlobGas - gasUsed += msgResult.UsedGas - - // Receipt: - { - var root []byte - if chainConfig.IsByzantium(vmContext.BlockNumber) { - statedb.Finalise(true) - } else { - root = statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber)).Bytes() - } - - // Create a new receipt for the transaction, storing the intermediate root and - // gas used by the tx. - receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: gasUsed} - if msgResult.Failed() { - receipt.Status = types.ReceiptStatusFailed - } else { - receipt.Status = types.ReceiptStatusSuccessful - } - receipt.TxHash = tx.Hash() - receipt.GasUsed = msgResult.UsedGas - - // If the transaction created a contract, store the creation address in the receipt. - if msg.To == nil { - receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce()) - } - - // Set the receipt logs and create the bloom filter. - receipt.Logs = statedb.GetLogs(tx.Hash(), vmContext.BlockNumber.Uint64(), blockHash, vmContext.Time) - receipt.Bloom = types.CreateBloom(receipt) - - // These three are non-consensus fields: - //receipt.BlockHash - //receipt.BlockNumber - receipt.TransactionIndex = uint(txIndex) - receipts = append(receipts, receipt) - if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxEnd != nil { - evm.Config.Tracer.OnTxEnd(receipt, nil) - } - } - - txIndex++ + receipts = append(receipts, receipt) } + statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber)) + // Add mining reward? (-1 means rewards are disabled) if miningReward >= 0 { // Add mining reward. The mining reward may be `0`, which only makes a difference in the cases diff --git a/cmd/evm/testdata/1/exp.json b/cmd/evm/testdata/1/exp.json index 50662f35ea..6537c9517d 100644 --- a/cmd/evm/testdata/1/exp.json +++ b/cmd/evm/testdata/1/exp.json @@ -29,7 +29,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" } ], diff --git a/cmd/evm/testdata/13/exp2.json b/cmd/evm/testdata/13/exp2.json index 6415a4f1f4..f716289cf7 100644 --- a/cmd/evm/testdata/13/exp2.json +++ b/cmd/evm/testdata/13/exp2.json @@ -17,7 +17,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x84d0", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" }, { @@ -31,7 +32,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x84d0", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x1" } ], diff --git a/cmd/evm/testdata/23/exp.json b/cmd/evm/testdata/23/exp.json index 7f36165e35..2d9cd492db 100644 --- a/cmd/evm/testdata/23/exp.json +++ b/cmd/evm/testdata/23/exp.json @@ -16,7 +16,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x520b", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x5", "transactionIndex": "0x0" } ], diff --git a/cmd/evm/testdata/24/exp.json b/cmd/evm/testdata/24/exp.json index 8f380c662b..0dd552e112 100644 --- a/cmd/evm/testdata/24/exp.json +++ b/cmd/evm/testdata/24/exp.json @@ -32,7 +32,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0xa861", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" }, { @@ -45,7 +46,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5aa5", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x1" } ], diff --git a/cmd/evm/testdata/25/exp.json b/cmd/evm/testdata/25/exp.json index a674633762..3dac46aa60 100644 --- a/cmd/evm/testdata/25/exp.json +++ b/cmd/evm/testdata/25/exp.json @@ -28,7 +28,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" } ], diff --git a/cmd/evm/testdata/28/exp.json b/cmd/evm/testdata/28/exp.json index b86c2d8def..15b29bc0ac 100644 --- a/cmd/evm/testdata/28/exp.json +++ b/cmd/evm/testdata/28/exp.json @@ -33,7 +33,10 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0xa865", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blobGasUsed": "0x20000", + "blobGasPrice": "0x1", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" } ], diff --git a/cmd/evm/testdata/29/exp.json b/cmd/evm/testdata/29/exp.json index 7fbdc18283..69c8661aa8 100644 --- a/cmd/evm/testdata/29/exp.json +++ b/cmd/evm/testdata/29/exp.json @@ -31,7 +31,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" } ], diff --git a/cmd/evm/testdata/3/exp.json b/cmd/evm/testdata/3/exp.json index 831c078591..807cdccfb4 100644 --- a/cmd/evm/testdata/3/exp.json +++ b/cmd/evm/testdata/3/exp.json @@ -29,7 +29,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x521f", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x5", "transactionIndex": "0x0" } ], diff --git a/cmd/evm/testdata/30/exp.json b/cmd/evm/testdata/30/exp.json index a206c3bbdf..9861f5a071 100644 --- a/cmd/evm/testdata/30/exp.json +++ b/cmd/evm/testdata/30/exp.json @@ -30,7 +30,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" }, { @@ -44,7 +45,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x1" } ], diff --git a/cmd/evm/testdata/33/exp.json b/cmd/evm/testdata/33/exp.json index ae82ef3efa..b40ca9fee2 100644 --- a/cmd/evm/testdata/33/exp.json +++ b/cmd/evm/testdata/33/exp.json @@ -48,7 +48,8 @@ "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x15fa9", "effectiveGasPrice": null, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockHash": "0x1337000000000000000000000000000000000000000000000000000000000000", + "blockNumber": "0x1", "transactionIndex": "0x0" } ], diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 750bf55927..b661228681 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -133,6 +133,8 @@ var ( utils.VMEnableDebugFlag, utils.VMTraceFlag, utils.VMTraceJsonConfigFlag, + utils.VMWitnessStatsFlag, + utils.VMStatelessSelfValidationFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, utils.GpoBlocksFlag, diff --git a/cmd/keeper/main.go b/cmd/keeper/main.go index cfb06f0da0..9b459f6f36 100644 --- a/cmd/keeper/main.go +++ b/cmd/keeper/main.go @@ -19,6 +19,7 @@ package main import ( "fmt" "os" + "runtime/debug" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/stateless" @@ -35,6 +36,10 @@ type Payload struct { Witness *stateless.Witness } +func init() { + debug.SetGCPercent(-1) // Disable garbage collection +} + func main() { input := getInput() var payload Payload diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a134ea4308..83d1c8bda5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -571,6 +571,16 @@ var ( Value: "{}", Category: flags.VMCategory, } + VMWitnessStatsFlag = &cli.BoolFlag{ + Name: "vmwitnessstats", + Usage: "Enable collection of witness trie access statistics (automatically enables witness generation)", + Category: flags.VMCategory, + } + VMStatelessSelfValidationFlag = &cli.BoolFlag{ + Name: "stateless-self-validation", + Usage: "Generate execution witnesses and self-check against them (testing purpose)", + Category: flags.VMCategory, + } // API options. RPCGlobalGasCapFlag = &cli.Uint64Flag{ Name: "rpc.gascap", @@ -1707,6 +1717,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(VMEnableDebugFlag.Name) { cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name) } + if ctx.IsSet(VMWitnessStatsFlag.Name) { + cfg.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name) + } + if ctx.IsSet(VMStatelessSelfValidationFlag.Name) { + cfg.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name) + } + // Auto-enable StatelessSelfValidation when witness stats are enabled + if ctx.Bool(VMWitnessStatsFlag.Name) { + cfg.StatelessSelfValidation = true + } if ctx.IsSet(RPCGlobalGasCapFlag.Name) { cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name) @@ -2243,6 +2263,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } vmcfg := vm.Config{ EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name), + EnableWitnessStats: ctx.Bool(VMWitnessStatsFlag.Name), + StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name), } if ctx.IsSet(VMTraceFlag.Name) { if name := ctx.String(VMTraceFlag.Name); name != "" { diff --git a/core/blockchain.go b/core/blockchain.go index c97897cd70..3466923648 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1907,7 +1907,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness } // The traced section of block import. start := time.Now() - res, err := bc.processBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1) + res, err := bc.ProcessBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1) if err != nil { return nil, it.index, err } @@ -1973,9 +1973,13 @@ type blockProcessingResult struct { witness *stateless.Witness } -// processBlock executes and validates the given block. If there was no error +func (bpr *blockProcessingResult) Witness() *stateless.Witness { + return bpr.witness +} + +// ProcessBlock executes and validates the given block. If there was no error // it writes the block and associated state to database. -func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (_ *blockProcessingResult, blockEndErr error) { +func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (_ *blockProcessingResult, blockEndErr error) { var ( err error startTime = time.Now() @@ -2153,7 +2157,7 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s } // Report the collected witness statistics if witnessStats != nil { - witnessStats.ReportMetrics() + witnessStats.ReportMetrics(block.NumberU64()) } // Update the metrics touched during block commit diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 196f3dac8f..5ed65b69b5 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -511,7 +511,7 @@ func TestWriteAncientHeaderChain(t *testing.T) { t.Fatalf("unexpected body returned") } if blob := ReadReceiptsRLP(db, header.Hash(), header.Number.Uint64()); len(blob) != 0 { - t.Fatalf("unexpected body returned") + t.Fatalf("unexpected receipts returned") } } } diff --git a/core/stateless/encoding.go b/core/stateless/encoding.go index 5f4cb0ea3c..5c43159e66 100644 --- a/core/stateless/encoding.go +++ b/core/stateless/encoding.go @@ -19,20 +19,21 @@ package stateless import ( "io" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" ) -// toExtWitness converts our internal witness representation to the consensus one. -func (w *Witness) toExtWitness() *extWitness { - ext := &extWitness{ +// ToExtWitness converts our internal witness representation to the consensus one. +func (w *Witness) ToExtWitness() *ExtWitness { + ext := &ExtWitness{ Headers: w.Headers, } - ext.Codes = make([][]byte, 0, len(w.Codes)) + ext.Codes = make([]hexutil.Bytes, 0, len(w.Codes)) for code := range w.Codes { ext.Codes = append(ext.Codes, []byte(code)) } - ext.State = make([][]byte, 0, len(w.State)) + ext.State = make([]hexutil.Bytes, 0, len(w.State)) for node := range w.State { ext.State = append(ext.State, []byte(node)) } @@ -40,7 +41,7 @@ func (w *Witness) toExtWitness() *extWitness { } // fromExtWitness converts the consensus witness format into our internal one. -func (w *Witness) fromExtWitness(ext *extWitness) error { +func (w *Witness) fromExtWitness(ext *ExtWitness) error { w.Headers = ext.Headers w.Codes = make(map[string]struct{}, len(ext.Codes)) @@ -56,21 +57,22 @@ func (w *Witness) fromExtWitness(ext *extWitness) error { // EncodeRLP serializes a witness as RLP. func (w *Witness) EncodeRLP(wr io.Writer) error { - return rlp.Encode(wr, w.toExtWitness()) + return rlp.Encode(wr, w.ToExtWitness()) } // DecodeRLP decodes a witness from RLP. func (w *Witness) DecodeRLP(s *rlp.Stream) error { - var ext extWitness + var ext ExtWitness if err := s.Decode(&ext); err != nil { return err } return w.fromExtWitness(&ext) } -// extWitness is a witness RLP encoding for transferring across clients. -type extWitness struct { - Headers []*types.Header - Codes [][]byte - State [][]byte +// ExtWitness is a witness RLP encoding for transferring across clients. +type ExtWitness struct { + Headers []*types.Header `json:"headers"` + Codes []hexutil.Bytes `json:"codes"` + State []hexutil.Bytes `json:"state"` + Keys []hexutil.Bytes `json:"keys"` } diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 1a6389284c..94f5587f99 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -17,6 +17,7 @@ package stateless import ( + "encoding/json" "maps" "slices" "sort" @@ -24,6 +25,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" ) @@ -70,7 +72,19 @@ func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { } // ReportMetrics reports the collected statistics to the global metrics registry. -func (s *WitnessStats) ReportMetrics() { +func (s *WitnessStats) ReportMetrics(blockNumber uint64) { + // Encode the metrics as JSON for easier consumption + accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves) + storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves) + + // Log account trie depth statistics + log.Info("Account trie depth stats", + "block", blockNumber, + "leavesAtDepth", string(accountLeavesJson)) + log.Info("Storage trie depth stats", + "block", blockNumber, + "leavesAtDepth", string(storageLeavesJson)) + for i := 0; i < 16; i++ { accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i]) storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i]) diff --git a/core/stateless/stats_test.go b/core/stateless/stats_test.go index 6219e622f5..e77084df5d 100644 --- a/core/stateless/stats_test.go +++ b/core/stateless/stats_test.go @@ -140,6 +140,64 @@ func TestWitnessStatsAdd(t *testing.T) { } } +func TestWitnessStatsMinMax(t *testing.T) { + stats := NewWitnessStats() + + // Add some account trie nodes with varying depths + stats.Add(map[string][]byte{ + "a": []byte("data1"), + "ab": []byte("data2"), + "abc": []byte("data3"), + "abcd": []byte("data4"), + "abcde": []byte("data5"), + }, common.Hash{}) + + // Only "abcde" is a leaf (depth 5) + for i, v := range stats.accountTrieLeaves { + if v != 0 && i != 5 { + t.Errorf("leaf found at invalid depth %d", i) + } + } + + // Add more leaves with different depths + stats.Add(map[string][]byte{ + "x": []byte("data6"), + "yz": []byte("data7"), + }, common.Hash{}) + + // Now we have leaves at depths 1, 2, and 5 + for i, v := range stats.accountTrieLeaves { + if v != 0 && (i != 5 && i != 2 && i != 1) { + t.Errorf("leaf found at invalid depth %d", i) + } + } +} + +func TestWitnessStatsAverage(t *testing.T) { + stats := NewWitnessStats() + + // Add nodes that will create leaves at depths 2, 3, and 4 + stats.Add(map[string][]byte{ + "aa": []byte("data1"), + "bb": []byte("data2"), + "ccc": []byte("data3"), + "dddd": []byte("data4"), + }, common.Hash{}) + + // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples + expectedAvg := int64(11) / int64(4) + var actualAvg, totalSamples int64 + for i, c := range stats.accountTrieLeaves { + actualAvg += c * int64(i) + totalSamples += c + } + actualAvg = actualAvg / totalSamples + + if actualAvg != expectedAvg { + t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg) + } +} + func BenchmarkWitnessStatsAdd(b *testing.B) { // Create a realistic trie node structure nodes := make(map[string][]byte) diff --git a/core/stateless/witness.go b/core/stateless/witness.go index 371a128f48..588c895a2f 100644 --- a/core/stateless/witness.go +++ b/core/stateless/witness.go @@ -100,6 +100,10 @@ func (w *Witness) AddState(nodes map[string][]byte) { } } +func (w *Witness) AddKey() { + panic("not yet implemented") +} + // Copy deep-copies the witness object. Witness.Block isn't deep-copied as it // is never mutated by Witness func (w *Witness) Copy() *Witness { diff --git a/core/tracing/journal_test.go b/core/tracing/journal_test.go index cf74d83483..e00447f5f3 100644 --- a/core/tracing/journal_test.go +++ b/core/tracing/journal_test.go @@ -280,10 +280,6 @@ func TestAllHooksCalled(t *testing.T) { if field.Type.Kind() != reflect.Func { continue } - // Skip non-hooks, i.e. Copy - if field.Name == "copy" { - continue - } // Skip if field is not set if wrappedValue.Field(i).IsNil() { continue diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 34cb766708..cae0be9f2d 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -30,6 +30,7 @@ import ( bls12381 "github.com/consensys/gnark-crypto/ecc/bls12-381" "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" + patched_big "github.com/ethereum/go-bigmodexpfix/src/math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core/tracing" @@ -631,9 +632,9 @@ func (c *bigModExp) Run(input []byte) ([]byte, error) { } // Retrieve the operands and execute the exponentiation var ( - base = new(big.Int).SetBytes(getData(input, 0, baseLen)) - exp = new(big.Int).SetBytes(getData(input, baseLen, expLen)) - mod = new(big.Int).SetBytes(getData(input, baseLen+expLen, modLen)) + base = new(patched_big.Int).SetBytes(getData(input, 0, baseLen)) + exp = new(patched_big.Int).SetBytes(getData(input, baseLen, expLen)) + mod = new(patched_big.Int).SetBytes(getData(input, baseLen+expLen, modLen)) v []byte ) switch { diff --git a/eth/api_debug.go b/eth/api_debug.go index 9cedbcbb2a..892e103213 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -491,3 +492,43 @@ func (api *DebugAPI) StateSize(blockHashOrNumber *rpc.BlockNumberOrHash) (interf "contractCodeBytes": hexutil.Uint64(stats.ContractCodeBytes), }, nil } + +func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumber) (*stateless.ExtWitness, error) { + bc := api.eth.blockchain + block, err := api.eth.APIBackend.BlockByNumber(context.Background(), bn) + if err != nil { + return &stateless.ExtWitness{}, fmt.Errorf("block number %v not found", bn) + } + + parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return &stateless.ExtWitness{}, fmt.Errorf("block number %v found, but parent missing", bn) + } + + result, err := bc.ProcessBlock(parent.Root, block, false, true) + if err != nil { + return nil, err + } + + return result.Witness().ToExtWitness(), nil +} + +func (api *DebugAPI) ExecutionWitnessByHash(hash common.Hash) (*stateless.ExtWitness, error) { + bc := api.eth.blockchain + block := bc.GetBlockByHash(hash) + if block == nil { + return &stateless.ExtWitness{}, fmt.Errorf("block hash %x not found", hash) + } + + parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return &stateless.ExtWitness{}, fmt.Errorf("block number %x found, but parent missing", hash) + } + + result, err := bc.ProcessBlock(parent.Root, block, false, true) + if err != nil { + return nil, err + } + + return result.Witness().ToExtWitness(), nil +} diff --git a/eth/backend.go b/eth/backend.go index 4356733189..3bfe0765f4 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -235,6 +235,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, + EnableWitnessStats: config.EnableWitnessStats, + StatelessSelfValidation: config.StatelessSelfValidation, }, // Enables file journaling for the trie database. The journal files will be stored // within the data directory. The corresponding paths will be either: diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index dc77141081..ba0a7762c7 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -144,6 +144,12 @@ type Config struct { // Enables tracking of SHA3 preimages in the VM EnablePreimageRecording bool + // Enables collection of witness trie access statistics + EnableWitnessStats bool + + // Generate execution witnesses and self-check against them (testing purpose) + StatelessSelfValidation bool + // Enables tracking of state size EnableStateSizeTracking bool diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 2fdd219dee..b54ba14d68 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -49,6 +49,8 @@ func (c Config) MarshalTOML() (interface{}, error) { BlobPool blobpool.Config GPO gasprice.Config EnablePreimageRecording bool + EnableWitnessStats bool + StatelessSelfValidation bool EnableStateSizeTracking bool VMTrace string VMTraceJsonConfig string @@ -91,6 +93,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.BlobPool = c.BlobPool enc.GPO = c.GPO enc.EnablePreimageRecording = c.EnablePreimageRecording + enc.EnableWitnessStats = c.EnableWitnessStats + enc.StatelessSelfValidation = c.StatelessSelfValidation enc.EnableStateSizeTracking = c.EnableStateSizeTracking enc.VMTrace = c.VMTrace enc.VMTraceJsonConfig = c.VMTraceJsonConfig @@ -137,6 +141,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { BlobPool *blobpool.Config GPO *gasprice.Config EnablePreimageRecording *bool + EnableWitnessStats *bool + StatelessSelfValidation *bool EnableStateSizeTracking *bool VMTrace *string VMTraceJsonConfig *string @@ -246,6 +252,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.EnablePreimageRecording != nil { c.EnablePreimageRecording = *dec.EnablePreimageRecording } + if dec.EnableWitnessStats != nil { + c.EnableWitnessStats = *dec.EnableWitnessStats + } + if dec.StatelessSelfValidation != nil { + c.StatelessSelfValidation = *dec.StatelessSelfValidation + } if dec.EnableStateSizeTracking != nil { c.EnableStateSizeTracking = *dec.EnableStateSizeTracking } diff --git a/go.mod b/go.mod index 73a1a13d93..71d823ba09 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,8 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844/v2 v2.1.2 + github.com/ethereum/c-kzg-4844/v2 v2.1.3 + github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab github.com/ethereum/go-verkle v0.2.2 github.com/fatih/color v1.16.0 github.com/ferranbt/fastssz v0.1.4 @@ -59,7 +60,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 github.com/stretchr/testify v1.10.0 - github.com/supranational/blst v0.3.15 + github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.27.5 go.uber.org/automaxprocs v1.5.2 @@ -67,7 +68,7 @@ require ( golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 + golang.org/x/sys v0.36.0 golang.org/x/text v0.23.0 golang.org/x/time v0.9.0 golang.org/x/tools v0.29.0 diff --git a/go.sum b/go.sum index e7a4d1e5b0..e3c26c5988 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,10 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= -github.com/ethereum/c-kzg-4844/v2 v2.1.2 h1:TsHMflcX0Wjjdwvhtg39HOozknAlQKY9PnG5Zf3gdD4= -github.com/ethereum/c-kzg-4844/v2 v2.1.2/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= +github.com/ethereum/c-kzg-4844/v2 v2.1.3 h1:DQ21UU0VSsuGy8+pcMJHDS0CV1bKmJmxsJYK8l3MiLU= +github.com/ethereum/c-kzg-4844/v2 v2.1.3/go.mod h1:fyNcYI/yAuLWJxf4uzVtS8VDKeoAaRM8G/+ADz/pRdA= +github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= +github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -349,8 +351,8 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/supranational/blst v0.3.15 h1:rd9viN6tfARE5wv3KZJ9H8e1cg0jXW8syFCcsbHa76o= -github.com/supranational/blst v0.3.15/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= +github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -449,8 +451,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 09808b71e0..9cca0118ac 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -27,6 +27,7 @@ import ( // lookup performs a network search for nodes close to the given target. It approaches the // target by querying nodes that are closer to it on each iteration. The given target does // not need to be an actual node identifier. +// lookup on an empty table will return immediately with no nodes. type lookup struct { tab *Table queryfunc queryFunc @@ -49,11 +50,15 @@ func newLookup(ctx context.Context, tab *Table, target enode.ID, q queryFunc) *l result: nodesByDistance{target: target}, replyCh: make(chan []*enode.Node, alpha), cancelCh: ctx.Done(), - queries: -1, } // Don't query further if we hit ourself. // Unlikely to happen often in practice. it.asked[tab.self().ID()] = true + it.seen[tab.self().ID()] = true + + // Initialize the lookup with nodes from table. + closest := it.tab.findnodeByID(it.result.target, bucketSize, false) + it.addNodes(closest.entries) return it } @@ -64,22 +69,19 @@ func (it *lookup) run() []*enode.Node { return it.result.entries } +func (it *lookup) empty() bool { + return len(it.replyBuffer) == 0 +} + // advance advances the lookup until any new nodes have been found. // It returns false when the lookup has ended. func (it *lookup) advance() bool { for it.startQueries() { select { case nodes := <-it.replyCh: - it.replyBuffer = it.replyBuffer[:0] - for _, n := range nodes { - if n != nil && !it.seen[n.ID()] { - it.seen[n.ID()] = true - it.result.push(n, bucketSize) - it.replyBuffer = append(it.replyBuffer, n) - } - } it.queries-- - if len(it.replyBuffer) > 0 { + it.addNodes(nodes) + if !it.empty() { return true } case <-it.cancelCh: @@ -89,6 +91,17 @@ func (it *lookup) advance() bool { return false } +func (it *lookup) addNodes(nodes []*enode.Node) { + it.replyBuffer = it.replyBuffer[:0] + for _, n := range nodes { + if n != nil && !it.seen[n.ID()] { + it.seen[n.ID()] = true + it.result.push(n, bucketSize) + it.replyBuffer = append(it.replyBuffer, n) + } + } +} + func (it *lookup) shutdown() { for it.queries > 0 { <-it.replyCh @@ -103,20 +116,6 @@ func (it *lookup) startQueries() bool { return false } - // The first query returns nodes from the local table. - if it.queries == -1 { - closest := it.tab.findnodeByID(it.result.target, bucketSize, false) - // Avoid finishing the lookup too quickly if table is empty. It'd be better to wait - // for the table to fill in this case, but there is no good mechanism for that - // yet. - if len(closest.entries) == 0 { - it.slowdown() - } - it.queries = 1 - it.replyCh <- closest.entries - return true - } - // Ask the closest nodes that we haven't asked yet. for i := 0; i < len(it.result.entries) && it.queries < alpha; i++ { n := it.result.entries[i] @@ -130,15 +129,6 @@ func (it *lookup) startQueries() bool { return it.queries > 0 } -func (it *lookup) slowdown() { - sleep := time.NewTimer(1 * time.Second) - defer sleep.Stop() - select { - case <-sleep.C: - case <-it.tab.closeReq: - } -} - func (it *lookup) query(n *enode.Node, reply chan<- []*enode.Node) { r, err := it.queryfunc(n) if !errors.Is(err, errClosed) { // avoid recording failures on shutdown. @@ -153,12 +143,16 @@ func (it *lookup) query(n *enode.Node, reply chan<- []*enode.Node) { // lookupIterator performs lookup operations and iterates over all seen nodes. // When a lookup finishes, a new one is created through nextLookup. +// LookupIterator waits for table initialization and triggers a table refresh +// when necessary. + type lookupIterator struct { - buffer []*enode.Node - nextLookup lookupFunc - ctx context.Context - cancel func() - lookup *lookup + buffer []*enode.Node + nextLookup lookupFunc + ctx context.Context + cancel func() + lookup *lookup + tabRefreshing <-chan struct{} } type lookupFunc func(ctx context.Context) *lookup @@ -182,6 +176,7 @@ func (it *lookupIterator) Next() bool { if len(it.buffer) > 0 { it.buffer = it.buffer[1:] } + // Advance the lookup to refill the buffer. for len(it.buffer) == 0 { if it.ctx.Err() != nil { @@ -191,17 +186,55 @@ func (it *lookupIterator) Next() bool { } if it.lookup == nil { it.lookup = it.nextLookup(it.ctx) + if it.lookup.empty() { + // If the lookup is empty right after creation, it means the local table + // is in a degraded state, and we need to wait for it to fill again. + it.lookupFailed(it.lookup.tab, 1*time.Minute) + it.lookup = nil + continue + } + // Yield the initial nodes from the iterator before advancing the lookup. + it.buffer = it.lookup.replyBuffer continue } - if !it.lookup.advance() { - it.lookup = nil - continue - } + + newNodes := it.lookup.advance() it.buffer = it.lookup.replyBuffer + if !newNodes { + it.lookup = nil + } } return true } +// lookupFailed handles failed lookup attempts. This can be called when the table has +// exited, or when it runs out of nodes. +func (it *lookupIterator) lookupFailed(tab *Table, timeout time.Duration) { + tout, cancel := context.WithTimeout(it.ctx, timeout) + defer cancel() + + // Wait for Table initialization to complete, in case it is still in progress. + select { + case <-tab.initDone: + case <-tout.Done(): + return + } + + // Wait for ongoing refresh operation, or trigger one. + if it.tabRefreshing == nil { + it.tabRefreshing = tab.refresh() + } + select { + case <-it.tabRefreshing: + it.tabRefreshing = nil + case <-tout.Done(): + return + } + + // Wait for the table to fill. + tab.waitForNodes(tout, 1) +} + // Close ends the iterator. func (it *lookupIterator) Close() { it.cancel() diff --git a/p2p/discover/table.go b/p2p/discover/table.go index b6c35aaaa9..6a1c7494ee 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/enode" @@ -84,6 +85,7 @@ type Table struct { closeReq chan struct{} closed chan struct{} + nodeFeed event.FeedOf[*enode.Node] nodeAddedHook func(*bucket, *tableNode) nodeRemovedHook func(*bucket, *tableNode) } @@ -567,6 +569,8 @@ func (tab *Table) nodeAdded(b *bucket, n *tableNode) { } n.addedToBucket = time.Now() tab.revalidation.nodeAdded(tab, n) + + tab.nodeFeed.Send(n.Node) if tab.nodeAddedHook != nil { tab.nodeAddedHook(b, n) } @@ -702,3 +706,38 @@ func (tab *Table) deleteNode(n *enode.Node) { b := tab.bucket(n.ID()) tab.deleteInBucket(b, n.ID()) } + +// waitForNodes blocks until the table contains at least n nodes. +func (tab *Table) waitForNodes(ctx context.Context, n int) error { + getlength := func() (count int) { + for _, b := range &tab.buckets { + count += len(b.entries) + } + return count + } + + var ch chan *enode.Node + for { + tab.mutex.Lock() + if getlength() >= n { + tab.mutex.Unlock() + return nil + } + if ch == nil { + // Init subscription. + ch = make(chan *enode.Node) + sub := tab.nodeFeed.Subscribe(ch) + defer sub.Unsubscribe() + } + tab.mutex.Unlock() + + // Wait for a node add event. + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + case <-tab.closeReq: + return errClosed + } + } +} diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 1af31f4f1b..44863183fa 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -24,10 +24,12 @@ import ( "errors" "fmt" "io" + "maps" "math/rand" "net" "net/netip" "reflect" + "slices" "sync" "testing" "time" @@ -509,18 +511,26 @@ func TestUDPv4_smallNetConvergence(t *testing.T) { // they have all found each other. status := make(chan error, len(nodes)) for i := range nodes { - node := nodes[i] + self := nodes[i] go func() { - found := make(map[enode.ID]bool, len(nodes)) - it := node.RandomNodes() + missing := make(map[enode.ID]bool, len(nodes)) + for _, n := range nodes { + if n.Self().ID() == self.Self().ID() { + continue // skip self + } + missing[n.Self().ID()] = true + } + + it := self.RandomNodes() for it.Next() { - found[it.Node().ID()] = true - if len(found) == len(nodes) { + delete(missing, it.Node().ID()) + if len(missing) == 0 { status <- nil return } } - status <- fmt.Errorf("node %s didn't find all nodes", node.Self().ID().TerminalString()) + missingIDs := slices.Collect(maps.Keys(missing)) + status <- fmt.Errorf("node %s didn't find all nodes, missing %v", self.Self().ID().TerminalString(), missingIDs) }() } @@ -537,7 +547,6 @@ func TestUDPv4_smallNetConvergence(t *testing.T) { received++ if err != nil { t.Error("ERROR:", err) - return } } } diff --git a/trie/iterator.go b/trie/iterator.go index 80298ce48f..3d3191ffba 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -836,3 +836,91 @@ func (it *unionIterator) Error() error { } return nil } + +// subTreeIterator wraps nodeIterator to traverse a trie within a predefined +// start and limit range. +type subtreeIterator struct { + NodeIterator + + stopPath []byte // Precomputed hex path for stopKey (without terminator), nil means no limit + exhausted bool // Flag whether the iterator has been exhausted +} + +// newSubtreeIterator creates an iterator that only traverses nodes within a subtree +// defined by the given startKey and stopKey. This supports general range iteration +// where startKey is inclusive and stopKey is exclusive. +// +// The iterator will only visit nodes whose keys k satisfy: startKey <= k < stopKey, +// where comparisons are performed in lexicographic order of byte keys (internally +// implemented via hex-nibble path comparisons for efficiency). +// +// If startKey is nil, iteration starts from the beginning. If stopKey is nil, +// iteration continues to the end of the trie. +func newSubtreeIterator(trie *Trie, startKey, stopKey []byte) (NodeIterator, error) { + it, err := trie.NodeIterator(startKey) + if err != nil { + return nil, err + } + if startKey == nil && stopKey == nil { + return it, nil + } + // Precompute nibble paths for efficient comparison + var stopPath []byte + if stopKey != nil { + stopPath = keybytesToHex(stopKey) + if hasTerm(stopPath) { + stopPath = stopPath[:len(stopPath)-1] + } + } + return &subtreeIterator{ + NodeIterator: it, + stopPath: stopPath, + }, nil +} + +// nextKey returns the next possible key after the given prefix. +// For example, "abc" -> "abd", "ab\xff" -> "ac", etc. +func nextKey(prefix []byte) []byte { + if len(prefix) == 0 { + return nil + } + // Make a copy to avoid modifying the original + next := make([]byte, len(prefix)) + copy(next, prefix) + + // Increment the last byte that isn't 0xff + for i := len(next) - 1; i >= 0; i-- { + if next[i] < 0xff { + next[i]++ + return next + } + // If it's 0xff, we need to carry over + // Trim trailing 0xff bytes + next = next[:i] + } + // If all bytes were 0xff, return nil (no upper bound) + return nil +} + +// newPrefixIterator creates an iterator that only traverses nodes with the given prefix. +// This ensures that only keys starting with the prefix are visited. +func newPrefixIterator(trie *Trie, prefix []byte) (NodeIterator, error) { + return newSubtreeIterator(trie, prefix, nextKey(prefix)) +} + +// Next moves the iterator to the next node. If the parameter is false, any child +// nodes will be skipped. +func (it *subtreeIterator) Next(descend bool) bool { + if it.exhausted { + return false + } + if !it.NodeIterator.Next(descend) { + it.exhausted = true + return false + } + if it.stopPath != nil && reachedPath(it.NodeIterator.Path(), it.stopPath) { + it.exhausted = true + return false + } + return true +} diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 74a1aa378c..f1451cef90 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -19,7 +19,9 @@ package trie import ( "bytes" "fmt" + "maps" "math/rand" + "slices" "testing" "github.com/ethereum/go-ethereum/common" @@ -624,6 +626,540 @@ func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) { return true, path, hash } +func TestSubtreeIterator(t *testing.T) { + var ( + db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + tr = NewEmpty(db) + ) + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"dog", "puppy"}, + {"doge", "coin"}, + {"dog\xff", "value6"}, + {"dog\xff\xff", "value7"}, + {"horse", "stallion"}, + {"house", "building"}, + {"houses", "multiple"}, + {"xyz", "value"}, + {"xyz\xff", "value"}, + {"xyz\xff\xff", "value"}, + } + all := make(map[string]string) + for _, val := range vals { + all[val.k] = val.v + tr.MustUpdate([]byte(val.k), []byte(val.v)) + } + root, nodes := tr.Commit(false) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + + allNodes := make(map[string][]byte) + tr, _ = New(TrieID(root), db) + it, err := tr.NodeIterator(nil) + if err != nil { + t.Fatal(err) + } + for it.Next(true) { + allNodes[string(it.Path())] = it.NodeBlob() + } + allKeys := slices.Collect(maps.Keys(all)) + + suites := []struct { + start []byte + end []byte + expected []string + }{ + // entire key range + { + start: nil, + end: nil, + expected: allKeys, + }, + { + start: nil, + end: bytes.Repeat([]byte{0xff}, 32), + expected: allKeys, + }, + { + start: bytes.Repeat([]byte{0x0}, 32), + end: bytes.Repeat([]byte{0xff}, 32), + expected: allKeys, + }, + // key range with start + { + start: []byte("do"), + end: nil, + expected: allKeys, + }, + { + start: []byte("doe"), + end: nil, + expected: allKeys[1:], + }, + { + start: []byte("dog"), + end: nil, + expected: allKeys[1:], + }, + { + start: []byte("doge"), + end: nil, + expected: allKeys[2:], + }, + { + start: []byte("dog\xff"), + end: nil, + expected: allKeys[3:], + }, + { + start: []byte("dog\xff\xff"), + end: nil, + expected: allKeys[4:], + }, + { + start: []byte("dog\xff\xff\xff"), + end: nil, + expected: allKeys[5:], + }, + // key range with limit + { + start: nil, + end: []byte("xyz"), + expected: allKeys[:len(allKeys)-3], + }, + { + start: nil, + end: []byte("xyz\xff"), + expected: allKeys[:len(allKeys)-2], + }, + { + start: nil, + end: []byte("xyz\xff\xff"), + expected: allKeys[:len(allKeys)-1], + }, + { + start: nil, + end: []byte("xyz\xff\xff\xff"), + expected: allKeys, + }, + } + for _, suite := range suites { + // We need to re-open the trie from the committed state + tr, _ = New(TrieID(root), db) + it, err := newSubtreeIterator(tr, suite.start, suite.end) + if err != nil { + t.Fatal(err) + } + + found := make(map[string]string) + for it.Next(true) { + if it.Leaf() { + found[string(it.LeafKey())] = string(it.LeafBlob()) + } + } + if len(found) != len(suite.expected) { + t.Errorf("wrong number of values: got %d, want %d", len(found), len(suite.expected)) + } + for k, v := range found { + if all[k] != v { + t.Errorf("wrong value for %s: got %s, want %s", k, found[k], all[k]) + } + } + + expectedNodes := make(map[string][]byte) + for path, blob := range allNodes { + if suite.start != nil { + hexStart := keybytesToHex(suite.start) + hexStart = hexStart[:len(hexStart)-1] + if !reachedPath([]byte(path), hexStart) { + continue + } + } + if suite.end != nil { + hexEnd := keybytesToHex(suite.end) + hexEnd = hexEnd[:len(hexEnd)-1] + if reachedPath([]byte(path), hexEnd) { + continue + } + } + expectedNodes[path] = bytes.Clone(blob) + } + + // Compare the result yield from the subtree iterator + var ( + subCount int + subIt, _ = newSubtreeIterator(tr, suite.start, suite.end) + ) + for subIt.Next(true) { + blob, ok := expectedNodes[string(subIt.Path())] + if !ok { + t.Errorf("Unexpected node iterated, path: %v", subIt.Path()) + } + subCount++ + + if !bytes.Equal(blob, subIt.NodeBlob()) { + t.Errorf("Unexpected node blob, path: %v, want: %v, got: %v", subIt.Path(), blob, subIt.NodeBlob()) + } + } + if subCount != len(expectedNodes) { + t.Errorf("Unexpected node being iterated, want: %d, got: %d", len(expectedNodes), subCount) + } + } +} + +func TestPrefixIterator(t *testing.T) { + // Create a new trie + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) + + // Insert test data + testData := map[string]string{ + "key1": "value1", + "key2": "value2", + "key10": "value10", + "key11": "value11", + "different": "value_different", + } + + for key, value := range testData { + trie.Update([]byte(key), []byte(value)) + } + + // Test prefix iteration for "key1" prefix + prefix := []byte("key1") + iter, err := trie.NodeIteratorWithPrefix(prefix) + if err != nil { + t.Fatalf("Failed to create prefix iterator: %v", err) + } + + var foundKeys [][]byte + for iter.Next(true) { + if iter.Leaf() { + foundKeys = append(foundKeys, iter.LeafKey()) + } + } + + if err := iter.Error(); err != nil { + t.Fatalf("Iterator error: %v", err) + } + + // Verify only keys starting with "key1" were found + expectedCount := 3 // "key1", "key10", "key11" + if len(foundKeys) != expectedCount { + t.Errorf("Expected %d keys, found %d", expectedCount, len(foundKeys)) + } + + for _, key := range foundKeys { + keyStr := string(key) + if !bytes.HasPrefix(key, prefix) { + t.Errorf("Found key %s doesn't have prefix %s", keyStr, string(prefix)) + } + } +} + +func TestPrefixIteratorVsFullIterator(t *testing.T) { + // Create a new trie with more structured data + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) + + // Insert structured test data + testData := map[string]string{ + "aaa": "value_aaa", + "aab": "value_aab", + "aba": "value_aba", + "bbb": "value_bbb", + } + + for key, value := range testData { + trie.Update([]byte(key), []byte(value)) + } + + // Test that prefix iterator stops at boundary + prefix := []byte("aa") + prefixIter, err := trie.NodeIteratorWithPrefix(prefix) + if err != nil { + t.Fatalf("Failed to create prefix iterator: %v", err) + } + + var prefixKeys [][]byte + for prefixIter.Next(true) { + if prefixIter.Leaf() { + prefixKeys = append(prefixKeys, prefixIter.LeafKey()) + } + } + + // Should only find "aaa" and "aab", not "aba" or "bbb" + if len(prefixKeys) != 2 { + t.Errorf("Expected 2 keys with prefix 'aa', found %d", len(prefixKeys)) + } + + // Verify no keys outside prefix were found + for _, key := range prefixKeys { + if !bytes.HasPrefix(key, prefix) { + t.Errorf("Prefix iterator returned key %s outside prefix %s", string(key), string(prefix)) + } + } +} + +func TestEmptyPrefixIterator(t *testing.T) { + // Test with empty trie + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) + + iter, err := trie.NodeIteratorWithPrefix([]byte("nonexistent")) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + + if iter.Next(true) { + t.Error("Expected no results from empty trie") + } +} + +// TestPrefixIteratorEdgeCases tests various edge cases for prefix iteration +func TestPrefixIteratorEdgeCases(t *testing.T) { + // Create a trie with test data + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) + testData := map[string]string{ + "abc": "value1", + "abcd": "value2", + "abce": "value3", + "abd": "value4", + "dog": "value5", + "dog\xff": "value6", // Test with 0xff byte + "dog\xff\xff": "value7", // Multiple 0xff bytes + } + for key, value := range testData { + trie.Update([]byte(key), []byte(value)) + } + + // Test 1: Prefix not present in trie + t.Run("NonexistentPrefix", func(t *testing.T) { + iter, err := trie.NodeIteratorWithPrefix([]byte("xyz")) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + count := 0 + for iter.Next(true) { + if iter.Leaf() { + count++ + } + } + if count != 0 { + t.Errorf("Expected 0 results for nonexistent prefix, got %d", count) + } + }) + + // Test 2: Prefix exactly equals an existing key + t.Run("ExactKeyPrefix", func(t *testing.T) { + iter, err := trie.NodeIteratorWithPrefix([]byte("abc")) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + found := make(map[string]bool) + for iter.Next(true) { + if iter.Leaf() { + found[string(iter.LeafKey())] = true + } + } + // Should find "abc", "abcd", "abce" but not "abd" + if !found["abc"] || !found["abcd"] || !found["abce"] { + t.Errorf("Missing expected keys: got %v", found) + } + if found["abd"] { + t.Errorf("Found unexpected key 'abd' with prefix 'abc'") + } + }) + + // Test 3: Prefix with trailing 0xff + t.Run("TrailingFFPrefix", func(t *testing.T) { + iter, err := trie.NodeIteratorWithPrefix([]byte("dog\xff")) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + found := make(map[string]bool) + for iter.Next(true) { + if iter.Leaf() { + found[string(iter.LeafKey())] = true + } + } + // Should find "dog\xff" and "dog\xff\xff" + if !found["dog\xff"] || !found["dog\xff\xff"] { + t.Errorf("Missing expected keys with 0xff: got %v", found) + } + if found["dog"] { + t.Errorf("Found unexpected key 'dog' with prefix 'dog\\xff'") + } + }) + + // Test 4: All 0xff case (edge case for nextKey) + t.Run("AllFFPrefix", func(t *testing.T) { + // Add a key with all 0xff bytes + allFF := []byte{0xff, 0xff} + trie.Update(allFF, []byte("all_ff_value")) + trie.Update(append(allFF, 0x00), []byte("all_ff_plus")) + + iter, err := trie.NodeIteratorWithPrefix(allFF) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + count := 0 + for iter.Next(true) { + if iter.Leaf() { + count++ + } + } + // Should find at least the allFF key itself + if count != 2 { + t.Errorf("Expected at least 1 result for all-0xff prefix, got %d", count) + } + }) + + // Test 5: Empty prefix (should iterate entire trie) + t.Run("EmptyPrefix", func(t *testing.T) { + iter, err := trie.NodeIteratorWithPrefix([]byte{}) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + count := 0 + for iter.Next(true) { + if iter.Leaf() { + count++ + } + } + // Should find all keys in the trie + expectedCount := len(testData) + 2 // +2 for the extra keys added in test 4 + if count != expectedCount { + t.Errorf("Expected %d results for empty prefix, got %d", expectedCount, count) + } + }) +} + +// TestGeneralRangeIteration tests NewSubtreeIterator with arbitrary start/stop ranges +func TestGeneralRangeIteration(t *testing.T) { + // Create a trie with test data + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) + testData := map[string]string{ + "apple": "fruit1", + "apricot": "fruit2", + "banana": "fruit3", + "cherry": "fruit4", + "date": "fruit5", + "fig": "fruit6", + "grape": "fruit7", + } + for key, value := range testData { + trie.Update([]byte(key), []byte(value)) + } + + // Test range iteration from "banana" to "fig" (exclusive) + t.Run("RangeIteration", func(t *testing.T) { + iter, _ := newSubtreeIterator(trie, []byte("banana"), []byte("fig")) + found := make(map[string]bool) + for iter.Next(true) { + if iter.Leaf() { + found[string(iter.LeafKey())] = true + } + } + // Should find "banana", "cherry", "date" but not "fig" + if !found["banana"] || !found["cherry"] || !found["date"] { + t.Errorf("Missing expected keys in range: got %v", found) + } + if found["apple"] || found["apricot"] || found["fig"] || found["grape"] { + t.Errorf("Found unexpected keys outside range: got %v", found) + } + }) + + // Test with nil stopKey (iterate to end) + t.Run("NilStopKey", func(t *testing.T) { + iter, _ := newSubtreeIterator(trie, []byte("date"), nil) + found := make(map[string]bool) + for iter.Next(true) { + if iter.Leaf() { + found[string(iter.LeafKey())] = true + } + } + // Should find "date", "fig", "grape" + if !found["date"] || !found["fig"] || !found["grape"] { + t.Errorf("Missing expected keys from 'date' to end: got %v", found) + } + if found["apple"] || found["banana"] || found["cherry"] { + t.Errorf("Found unexpected keys before 'date': got %v", found) + } + }) + + // Test with nil startKey (iterate from beginning) + t.Run("NilStartKey", func(t *testing.T) { + iter, _ := newSubtreeIterator(trie, nil, []byte("cherry")) + found := make(map[string]bool) + for iter.Next(true) { + if iter.Leaf() { + found[string(iter.LeafKey())] = true + } + } + // Should find "apple", "apricot", "banana" but not "cherry" or later + if !found["apple"] || !found["apricot"] || !found["banana"] { + t.Errorf("Missing expected keys before 'cherry': got %v", found) + } + if found["cherry"] || found["date"] || found["fig"] || found["grape"] { + t.Errorf("Found unexpected keys at or after 'cherry': got %v", found) + } + }) +} + +// TestPrefixIteratorWithDescend tests prefix iteration with descend=false +func TestPrefixIteratorWithDescend(t *testing.T) { + // Create a trie with nested structure + trie := NewEmpty(newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)) + testData := map[string]string{ + "a": "value_a", + "a/b": "value_ab", + "a/b/c": "value_abc", + "a/b/d": "value_abd", + "a/e": "value_ae", + "b": "value_b", + } + for key, value := range testData { + trie.Update([]byte(key), []byte(value)) + } + + // Test skipping subtrees with descend=false + t.Run("SkipSubtrees", func(t *testing.T) { + iter, err := trie.NodeIteratorWithPrefix([]byte("a")) + if err != nil { + t.Fatalf("Failed to create iterator: %v", err) + } + + // Count nodes at each level + nodesVisited := 0 + leafsFound := make(map[string]bool) + + // First call with descend=true to enter the "a" subtree + if !iter.Next(true) { + t.Fatal("Expected to find at least one node") + } + nodesVisited++ + + // Continue iteration, sometimes with descend=false + descendPattern := []bool{false, true, false, true, true} + for i := 0; iter.Next(descendPattern[i%len(descendPattern)]); i++ { + nodesVisited++ + if iter.Leaf() { + leafsFound[string(iter.LeafKey())] = true + } + } + + // We should still respect the prefix boundary even when skipping + prefix := []byte("a") + for key := range leafsFound { + if !bytes.HasPrefix([]byte(key), prefix) { + t.Errorf("Found key outside prefix when using descend=false: %s", key) + } + } + + // Should not have found "b" even if we skip some subtrees + if leafsFound["b"] { + t.Error("Iterator leaked outside prefix boundary with descend=false") + } + }) +} + func BenchmarkIterator(b *testing.B) { diskDb, srcDb, tr, _ := makeTestTrie(rawdb.HashScheme) root := tr.Hash() diff --git a/trie/trie.go b/trie/trie.go index 630462f8ca..36cc732ee8 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -134,6 +134,34 @@ func (t *Trie) NodeIterator(start []byte) (NodeIterator, error) { return newNodeIterator(t, start), nil } +// NodeIteratorWithPrefix returns an iterator that returns nodes of the trie +// whose leaf keys start with the given prefix. Iteration includes all keys +// where prefix <= k < nextKey(prefix), effectively returning only keys that +// have the prefix. The iteration stops once it would encounter a key that +// doesn't start with the prefix. +// +// For example, with prefix "dog", the iterator will return "dog", "dogcat", +// "dogfish" but not "dot" or "fog". An empty prefix iterates the entire trie. +func (t *Trie) NodeIteratorWithPrefix(prefix []byte) (NodeIterator, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, ErrCommitted + } + // Use the dedicated prefix iterator which handles prefix checking correctly + return newPrefixIterator(t, prefix) +} + +// NodeIteratorWithRange returns an iterator over trie nodes whose leaf keys +// fall within the specified range. It includes all keys where start <= k < end. +// Iteration stops once a key beyond the end boundary is encountered. +func (t *Trie) NodeIteratorWithRange(start, end []byte) (NodeIterator, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, ErrCommitted + } + return newSubtreeIterator(t, start, end) +} + // MustGet is a wrapper of Get and will omit any encountered error but just // print out an error message. func (t *Trie) MustGet(key []byte) []byte { diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index ae9574963e..546d2e0301 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -189,7 +189,7 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { } // TODO (rjl493456442) disable the background indexing in read-only mode if db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID()) + db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) log.Info("Enabled state history indexing") } fields := config.fields() @@ -245,7 +245,7 @@ func (db *Database) repairHistory() error { } // Truncate the extra state histories above in freezer in case it's not // aligned with the disk layer. It might happen after a unclean shutdown. - pruned, err := truncateFromHead(db.stateFreezer, id) + pruned, err := truncateFromHead(db.stateFreezer, typeStateHistory, id) if err != nil { log.Crit("Failed to truncate extra state histories", "err", err) } @@ -448,7 +448,7 @@ func (db *Database) Enable(root common.Hash) error { // 2. Re-initialize the indexer so it starts indexing from the new state root. if db.stateIndexer != nil && db.stateFreezer != nil && db.config.EnableStateIndexing { db.stateIndexer.close() - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID()) + db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) log.Info("Re-enabled state history indexing") } log.Info("Rebuilt trie database", "root", root) @@ -502,7 +502,7 @@ func (db *Database) Recover(root common.Hash) error { if err := db.diskdb.SyncKeyValue(); err != nil { return err } - _, err := truncateFromHead(db.stateFreezer, dl.stateID()) + _, err := truncateFromHead(db.stateFreezer, typeStateHistory, dl.stateID()) if err != nil { return err } diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 2042e91611..76f3f5a46e 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -378,7 +378,7 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) return true, nil } - pruned, err := truncateFromTail(dl.db.stateFreezer, newFirst-1) + pruned, err := truncateFromTail(dl.db.stateFreezer, typeStateHistory, newFirst-1) if err != nil { return false, err } diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index bbedd52f34..ae022236fe 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -19,11 +19,147 @@ package pathdb import ( "errors" "fmt" + "iter" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) +// historyType represents the category of historical data. +type historyType uint8 + +const ( + // typeStateHistory indicates history data related to account or storage changes. + typeStateHistory historyType = 0 +) + +// String returns the string format representation. +func (h historyType) String() string { + switch h { + case typeStateHistory: + return "state" + default: + return fmt.Sprintf("unknown type: %d", h) + } +} + +// elementType represents the category of state element. +type elementType uint8 + +const ( + typeAccount elementType = 0 // represents the account data + typeStorage elementType = 1 // represents the storage slot data +) + +// String returns the string format representation. +func (e elementType) String() string { + switch e { + case typeAccount: + return "account" + case typeStorage: + return "storage" + default: + return fmt.Sprintf("unknown element type: %d", e) + } +} + +// toHistoryType maps an element type to its corresponding history type. +func toHistoryType(typ elementType) historyType { + if typ == typeAccount || typ == typeStorage { + return typeStateHistory + } + panic(fmt.Sprintf("unknown element type %v", typ)) +} + +// stateIdent represents the identifier of a state element, which can be +// an account or a storage slot. +type stateIdent struct { + typ elementType + + // The hash of the account address. This is used instead of the raw account + // address is to align the traversal order with the Merkle-Patricia-Trie. + addressHash common.Hash + + // The hash of the storage slot key. This is used instead of the raw slot key + // because, in legacy state histories (prior to the Cancun fork), the slot + // identifier is the hash of the key, and the original key (preimage) cannot + // be recovered. To maintain backward compatibility, the key hash is used. + // + // Meanwhile, using the storage key hash also preserve the traversal order + // with Merkle-Patricia-Trie. + // + // This field is null if the identifier refers to an account or a trie node. + storageHash common.Hash +} + +// String returns the string format state identifier. +func (ident stateIdent) String() string { + if ident.typ == typeAccount { + return ident.addressHash.Hex() + } + return ident.addressHash.Hex() + ident.storageHash.Hex() +} + +// newAccountIdent constructs a state identifier for an account. +func newAccountIdent(addressHash common.Hash) stateIdent { + return stateIdent{ + typ: typeAccount, + addressHash: addressHash, + } +} + +// newStorageIdent constructs a state identifier for a storage slot. +// The address denotes the address hash of the associated account; +// the storageHash denotes the hash of the raw storage slot key; +func newStorageIdent(addressHash common.Hash, storageHash common.Hash) stateIdent { + return stateIdent{ + typ: typeStorage, + addressHash: addressHash, + storageHash: storageHash, + } +} + +// stateIdentQuery is the extension of stateIdent by adding the account address +// and raw storage key. +type stateIdentQuery struct { + stateIdent + + address common.Address + storageKey common.Hash +} + +// newAccountIdentQuery constructs a state identifier for an account. +func newAccountIdentQuery(address common.Address, addressHash common.Hash) stateIdentQuery { + return stateIdentQuery{ + stateIdent: newAccountIdent(addressHash), + address: address, + } +} + +// newStorageIdentQuery constructs a state identifier for a storage slot. +// the address denotes the address of the associated account; +// the addressHash denotes the address hash of the associated account; +// the storageKey denotes the raw storage slot key; +// the storageHash denotes the hash of the raw storage slot key; +func newStorageIdentQuery(address common.Address, addressHash common.Hash, storageKey common.Hash, storageHash common.Hash) stateIdentQuery { + return stateIdentQuery{ + stateIdent: newStorageIdent(addressHash, storageHash), + address: address, + storageKey: storageKey, + } +} + +// history defines the interface of historical data, implemented by stateHistory +// and trienodeHistory (in the near future). +type history interface { + // typ returns the historical data type held in the history. + typ() historyType + + // forEach returns an iterator to traverse the state entries in the history. + forEach() iter.Seq[stateIdent] +} + var ( errHeadTruncationOutOfRange = errors.New("history head truncation out of range") errTailTruncationOutOfRange = errors.New("history tail truncation out of range") @@ -31,7 +167,7 @@ var ( // truncateFromHead removes excess elements from the head of the freezer based // on the given parameters. It returns the number of items that were removed. -func truncateFromHead(store ethdb.AncientStore, nhead uint64) (int, error) { +func truncateFromHead(store ethdb.AncientStore, typ historyType, nhead uint64) (int, error) { ohead, err := store.Ancients() if err != nil { return 0, err @@ -40,11 +176,11 @@ func truncateFromHead(store ethdb.AncientStore, nhead uint64) (int, error) { if err != nil { return 0, err } - log.Info("Truncating from head", "ohead", ohead, "tail", otail, "nhead", nhead) + log.Info("Truncating from head", "type", typ.String(), "ohead", ohead, "tail", otail, "nhead", nhead) // Ensure that the truncation target falls within the valid range. if ohead < nhead || nhead < otail { - return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, otail, ohead, nhead) + return 0, fmt.Errorf("%w, %s, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, typ, otail, ohead, nhead) } // Short circuit if nothing to truncate. if ohead == nhead { @@ -61,7 +197,7 @@ func truncateFromHead(store ethdb.AncientStore, nhead uint64) (int, error) { // truncateFromTail removes excess elements from the end of the freezer based // on the given parameters. It returns the number of items that were removed. -func truncateFromTail(store ethdb.AncientStore, ntail uint64) (int, error) { +func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) (int, error) { ohead, err := store.Ancients() if err != nil { return 0, err @@ -72,7 +208,7 @@ func truncateFromTail(store ethdb.AncientStore, ntail uint64) (int, error) { } // Ensure that the truncation target falls within the valid range. if otail > ntail || ntail > ohead { - return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errTailTruncationOutOfRange, otail, ohead, ntail) + return 0, fmt.Errorf("%w, %s, tail: %d, head: %d, target: %d", errTailTruncationOutOfRange, typ, otail, ohead, ntail) } // Short circuit if nothing to truncate. if otail == ntail { diff --git a/triedb/pathdb/history_index.go b/triedb/pathdb/history_index.go index e781a898e1..47cee9820d 100644 --- a/triedb/pathdb/history_index.go +++ b/triedb/pathdb/history_index.go @@ -78,12 +78,7 @@ type indexReader struct { // loadIndexData loads the index data associated with the specified state. func loadIndexData(db ethdb.KeyValueReader, state stateIdent) ([]*indexBlockDesc, error) { - var blob []byte - if state.account { - blob = rawdb.ReadAccountHistoryIndex(db, state.addressHash) - } else { - blob = rawdb.ReadStorageHistoryIndex(db, state.addressHash, state.storageHash) - } + blob := readStateIndex(state, db) if len(blob) == 0 { return nil, nil } @@ -137,15 +132,8 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { br, ok := r.readers[desc.id] if !ok { - var ( - err error - blob []byte - ) - if r.state.account { - blob = rawdb.ReadAccountHistoryIndexBlock(r.db, r.state.addressHash, desc.id) - } else { - blob = rawdb.ReadStorageHistoryIndexBlock(r.db, r.state.addressHash, r.state.storageHash, desc.id) - } + var err error + blob := readStateIndexBlock(r.state, r.db, desc.id) br, err = newBlockReader(blob) if err != nil { return 0, err @@ -174,12 +162,7 @@ type indexWriter struct { // newIndexWriter constructs the index writer for the specified state. func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, error) { - var blob []byte - if state.account { - blob = rawdb.ReadAccountHistoryIndex(db, state.addressHash) - } else { - blob = rawdb.ReadStorageHistoryIndex(db, state.addressHash, state.storageHash) - } + blob := readStateIndex(state, db) if len(blob) == 0 { desc := newIndexBlockDesc(0) bw, _ := newBlockWriter(nil, desc) @@ -194,15 +177,8 @@ func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, er if err != nil { return nil, err } - var ( - indexBlock []byte - lastDesc = descList[len(descList)-1] - ) - if state.account { - indexBlock = rawdb.ReadAccountHistoryIndexBlock(db, state.addressHash, lastDesc.id) - } else { - indexBlock = rawdb.ReadStorageHistoryIndexBlock(db, state.addressHash, state.storageHash, lastDesc.id) - } + lastDesc := descList[len(descList)-1] + indexBlock := readStateIndexBlock(state, db, lastDesc.id) bw, err := newBlockWriter(indexBlock, lastDesc) if err != nil { return nil, err @@ -270,11 +246,7 @@ func (w *indexWriter) finish(batch ethdb.Batch) { return // nothing to commit } for _, bw := range writers { - if w.state.account { - rawdb.WriteAccountHistoryIndexBlock(batch, w.state.addressHash, bw.desc.id, bw.finish()) - } else { - rawdb.WriteStorageHistoryIndexBlock(batch, w.state.addressHash, w.state.storageHash, bw.desc.id, bw.finish()) - } + writeStateIndexBlock(w.state, batch, bw.desc.id, bw.finish()) } w.frozen = nil // release all the frozen writers @@ -282,11 +254,7 @@ func (w *indexWriter) finish(batch ethdb.Batch) { for _, desc := range descList { buf = append(buf, desc.encode()...) } - if w.state.account { - rawdb.WriteAccountHistoryIndex(batch, w.state.addressHash, buf) - } else { - rawdb.WriteStorageHistoryIndex(batch, w.state.addressHash, w.state.storageHash, buf) - } + writeStateIndex(w.state, batch, buf) } // indexDeleter is responsible for deleting index data for a specific state. @@ -301,12 +269,7 @@ type indexDeleter struct { // newIndexDeleter constructs the index deleter for the specified state. func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter, error) { - var blob []byte - if state.account { - blob = rawdb.ReadAccountHistoryIndex(db, state.addressHash) - } else { - blob = rawdb.ReadStorageHistoryIndex(db, state.addressHash, state.storageHash) - } + blob := readStateIndex(state, db) if len(blob) == 0 { // TODO(rjl493456442) we can probably return an error here, // deleter with no data is meaningless. @@ -323,15 +286,8 @@ func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter, if err != nil { return nil, err } - var ( - indexBlock []byte - lastDesc = descList[len(descList)-1] - ) - if state.account { - indexBlock = rawdb.ReadAccountHistoryIndexBlock(db, state.addressHash, lastDesc.id) - } else { - indexBlock = rawdb.ReadStorageHistoryIndexBlock(db, state.addressHash, state.storageHash, lastDesc.id) - } + lastDesc := descList[len(descList)-1] + indexBlock := readStateIndexBlock(state, db, lastDesc.id) bw, err := newBlockWriter(indexBlock, lastDesc) if err != nil { return nil, err @@ -376,15 +332,8 @@ func (d *indexDeleter) pop(id uint64) error { d.descList = d.descList[:len(d.descList)-1] // Open the previous block writer for deleting - var ( - indexBlock []byte - lastDesc = d.descList[len(d.descList)-1] - ) - if d.state.account { - indexBlock = rawdb.ReadAccountHistoryIndexBlock(d.db, d.state.addressHash, lastDesc.id) - } else { - indexBlock = rawdb.ReadStorageHistoryIndexBlock(d.db, d.state.addressHash, d.state.storageHash, lastDesc.id) - } + lastDesc := d.descList[len(d.descList)-1] + indexBlock := readStateIndexBlock(d.state, d.db, lastDesc.id) bw, err := newBlockWriter(indexBlock, lastDesc) if err != nil { return err @@ -399,38 +348,100 @@ func (d *indexDeleter) pop(id uint64) error { // This function is safe to be called multiple times. func (d *indexDeleter) finish(batch ethdb.Batch) { for _, id := range d.dropped { - if d.state.account { - rawdb.DeleteAccountHistoryIndexBlock(batch, d.state.addressHash, id) - } else { - rawdb.DeleteStorageHistoryIndexBlock(batch, d.state.addressHash, d.state.storageHash, id) - } + deleteStateIndexBlock(d.state, batch, id) } d.dropped = nil // Flush the content of last block writer, regardless it's dirty or not if !d.bw.empty() { - if d.state.account { - rawdb.WriteAccountHistoryIndexBlock(batch, d.state.addressHash, d.bw.desc.id, d.bw.finish()) - } else { - rawdb.WriteStorageHistoryIndexBlock(batch, d.state.addressHash, d.state.storageHash, d.bw.desc.id, d.bw.finish()) - } + writeStateIndexBlock(d.state, batch, d.bw.desc.id, d.bw.finish()) } // Flush the index metadata into the supplied batch if d.empty() { - if d.state.account { - rawdb.DeleteAccountHistoryIndex(batch, d.state.addressHash) - } else { - rawdb.DeleteStorageHistoryIndex(batch, d.state.addressHash, d.state.storageHash) - } + deleteStateIndex(d.state, batch) } else { buf := make([]byte, 0, indexBlockDescSize*len(d.descList)) for _, desc := range d.descList { buf = append(buf, desc.encode()...) } - if d.state.account { - rawdb.WriteAccountHistoryIndex(batch, d.state.addressHash, buf) - } else { - rawdb.WriteStorageHistoryIndex(batch, d.state.addressHash, d.state.storageHash, buf) - } + writeStateIndex(d.state, batch, buf) + } +} + +// readStateIndex retrieves the index metadata for the given state identifier. +// This function is shared by accounts and storage slots. +func readStateIndex(ident stateIdent, db ethdb.KeyValueReader) []byte { + switch ident.typ { + case typeAccount: + return rawdb.ReadAccountHistoryIndex(db, ident.addressHash) + case typeStorage: + return rawdb.ReadStorageHistoryIndex(db, ident.addressHash, ident.storageHash) + default: + panic(fmt.Errorf("unknown type: %v", ident.typ)) + } +} + +// writeStateIndex writes the provided index metadata into database with the +// given state identifier. This function is shared by accounts and storage slots. +func writeStateIndex(ident stateIdent, db ethdb.KeyValueWriter, data []byte) { + switch ident.typ { + case typeAccount: + rawdb.WriteAccountHistoryIndex(db, ident.addressHash, data) + case typeStorage: + rawdb.WriteStorageHistoryIndex(db, ident.addressHash, ident.storageHash, data) + default: + panic(fmt.Errorf("unknown type: %v", ident.typ)) + } +} + +// deleteStateIndex removes the index metadata for the given state identifier. +// This function is shared by accounts and storage slots. +func deleteStateIndex(ident stateIdent, db ethdb.KeyValueWriter) { + switch ident.typ { + case typeAccount: + rawdb.DeleteAccountHistoryIndex(db, ident.addressHash) + case typeStorage: + rawdb.DeleteStorageHistoryIndex(db, ident.addressHash, ident.storageHash) + default: + panic(fmt.Errorf("unknown type: %v", ident.typ)) + } +} + +// readStateIndexBlock retrieves the index block for the given state identifier +// and block ID. This function is shared by accounts and storage slots. +func readStateIndexBlock(ident stateIdent, db ethdb.KeyValueReader, id uint32) []byte { + switch ident.typ { + case typeAccount: + return rawdb.ReadAccountHistoryIndexBlock(db, ident.addressHash, id) + case typeStorage: + return rawdb.ReadStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id) + default: + panic(fmt.Errorf("unknown type: %v", ident.typ)) + } +} + +// writeStateIndexBlock writes the provided index block into database with the +// given state identifier. This function is shared by accounts and storage slots. +func writeStateIndexBlock(ident stateIdent, db ethdb.KeyValueWriter, id uint32, data []byte) { + switch ident.typ { + case typeAccount: + rawdb.WriteAccountHistoryIndexBlock(db, ident.addressHash, id, data) + case typeStorage: + rawdb.WriteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id, data) + default: + panic(fmt.Errorf("unknown type: %v", ident.typ)) + } +} + +// deleteStateIndexBlock removes the index block from database with the given +// state identifier. This function is shared by accounts and storage slots. +func deleteStateIndexBlock(ident stateIdent, db ethdb.KeyValueWriter, id uint32) { + switch ident.typ { + case typeAccount: + rawdb.DeleteAccountHistoryIndexBlock(db, ident.addressHash, id) + case typeStorage: + rawdb.DeleteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id) + default: + panic(fmt.Errorf("unknown type: %v", ident.typ)) } } diff --git a/triedb/pathdb/history_index_test.go b/triedb/pathdb/history_index_test.go index c83c33ffbd..be9b7c4049 100644 --- a/triedb/pathdb/history_index_test.go +++ b/triedb/pathdb/history_index_test.go @@ -179,7 +179,7 @@ func TestIndexWriterDelete(t *testing.T) { func TestBatchIndexerWrite(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - batch = newBatchIndexer(db, false) + batch = newBatchIndexer(db, false, typeStateHistory) histories = makeStateHistories(10) ) for i, h := range histories { @@ -190,7 +190,7 @@ func TestBatchIndexerWrite(t *testing.T) { if err := batch.finish(true); err != nil { t.Fatalf("Failed to finish batch indexer, %v", err) } - metadata := loadIndexMetadata(db) + metadata := loadIndexMetadata(db, typeStateHistory) if metadata == nil || metadata.Last != uint64(10) { t.Fatal("Unexpected index position") } @@ -256,7 +256,7 @@ func TestBatchIndexerWrite(t *testing.T) { func TestBatchIndexerDelete(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - bw = newBatchIndexer(db, false) + bw = newBatchIndexer(db, false, typeStateHistory) histories = makeStateHistories(10) ) // Index histories @@ -270,7 +270,7 @@ func TestBatchIndexerDelete(t *testing.T) { } // Unindex histories - bd := newBatchIndexer(db, true) + bd := newBatchIndexer(db, true, typeStateHistory) for i := len(histories) - 1; i >= 0; i-- { if err := bd.process(histories[i], uint64(i+1)); err != nil { t.Fatalf("Failed to process history, %v", err) @@ -280,7 +280,7 @@ func TestBatchIndexerDelete(t *testing.T) { t.Fatalf("Failed to finish batch indexer, %v", err) } - metadata := loadIndexMetadata(db) + metadata := loadIndexMetadata(db, typeStateHistory) if metadata != nil { t.Fatal("Unexpected index position") } diff --git a/triedb/pathdb/history_indexer.go b/triedb/pathdb/history_indexer.go index b4e89c3f17..d618585929 100644 --- a/triedb/pathdb/history_indexer.go +++ b/triedb/pathdb/history_indexer.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -41,13 +40,32 @@ const ( stateIndexVersion = stateIndexV0 // the current state index version ) +// indexVersion returns the latest index version for the given history type. +// It panics if the history type is unknown. +func indexVersion(typ historyType) uint8 { + switch typ { + case typeStateHistory: + return stateIndexVersion + default: + panic(fmt.Errorf("unknown history type: %d", typ)) + } +} + +// indexMetadata describes the metadata of the historical data index. type indexMetadata struct { Version uint8 Last uint64 } -func loadIndexMetadata(db ethdb.KeyValueReader) *indexMetadata { - blob := rawdb.ReadStateHistoryIndexMetadata(db) +// loadIndexMetadata reads the metadata of the specific history index. +func loadIndexMetadata(db ethdb.KeyValueReader, typ historyType) *indexMetadata { + var blob []byte + switch typ { + case typeStateHistory: + blob = rawdb.ReadStateHistoryIndexMetadata(db) + default: + panic(fmt.Errorf("unknown history type %d", typ)) + } if len(blob) == 0 { return nil } @@ -59,91 +77,94 @@ func loadIndexMetadata(db ethdb.KeyValueReader) *indexMetadata { return &m } -func storeIndexMetadata(db ethdb.KeyValueWriter, last uint64) { - var m indexMetadata - m.Version = stateIndexVersion - m.Last = last +// storeIndexMetadata stores the metadata of the specific history index. +func storeIndexMetadata(db ethdb.KeyValueWriter, typ historyType, last uint64) { + m := indexMetadata{ + Version: indexVersion(typ), + Last: last, + } blob, err := rlp.EncodeToBytes(m) if err != nil { - log.Crit("Failed to encode index metadata", "err", err) + panic(fmt.Errorf("fail to encode index metadata, %v", err)) } - rawdb.WriteStateHistoryIndexMetadata(db, blob) + switch typ { + case typeStateHistory: + rawdb.WriteStateHistoryIndexMetadata(db, blob) + default: + panic(fmt.Errorf("unknown history type %d", typ)) + } + log.Debug("Written index metadata", "type", typ, "last", last) } -// batchIndexer is a structure designed to perform batch indexing or unindexing -// of state histories atomically. +// deleteIndexMetadata deletes the metadata of the specific history index. +func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) { + switch typ { + case typeStateHistory: + rawdb.DeleteStateHistoryIndexMetadata(db) + default: + panic(fmt.Errorf("unknown history type %d", typ)) + } + log.Debug("Deleted index metadata", "type", typ) +} + +// batchIndexer is responsible for performing batch indexing or unindexing +// of historical data (e.g., state or trie node changes) atomically. type batchIndexer struct { - accounts map[common.Hash][]uint64 // History ID list, Keyed by the hash of account address - storages map[common.Hash]map[common.Hash][]uint64 // History ID list, Keyed by the hash of account address and the hash of raw storage key - counter int // The counter of processed states - delete bool // Index or unindex mode - lastID uint64 // The ID of latest processed history - db ethdb.KeyValueStore + index map[stateIdent][]uint64 // List of history IDs for tracked state entry + pending int // Number of entries processed in the current batch. + delete bool // Operation mode: true for unindex, false for index. + lastID uint64 // ID of the most recently processed history. + typ historyType // Type of history being processed (e.g., state or trienode). + db ethdb.KeyValueStore // Key-value database used to store or delete index data. } // newBatchIndexer constructs the batch indexer with the supplied mode. -func newBatchIndexer(db ethdb.KeyValueStore, delete bool) *batchIndexer { +func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batchIndexer { return &batchIndexer{ - accounts: make(map[common.Hash][]uint64), - storages: make(map[common.Hash]map[common.Hash][]uint64), - delete: delete, - db: db, + index: make(map[stateIdent][]uint64), + delete: delete, + typ: typ, + db: db, } } -// process iterates through the accounts and their associated storage slots in the -// state history, tracking the mapping between state and history IDs. -func (b *batchIndexer) process(h *stateHistory, historyID uint64) error { - for _, address := range h.accountList { - addrHash := crypto.Keccak256Hash(address.Bytes()) - b.counter += 1 - b.accounts[addrHash] = append(b.accounts[addrHash], historyID) - - for _, slotKey := range h.storageList[address] { - b.counter += 1 - if _, ok := b.storages[addrHash]; !ok { - b.storages[addrHash] = make(map[common.Hash][]uint64) - } - // The hash of the storage slot key is used as the identifier because the - // legacy history does not include the raw storage key, therefore, the - // conversion from storage key to hash is necessary for non-v0 histories. - slotHash := slotKey - if h.meta.version != stateHistoryV0 { - slotHash = crypto.Keccak256Hash(slotKey.Bytes()) - } - b.storages[addrHash][slotHash] = append(b.storages[addrHash][slotHash], historyID) - } +// process traverses the state entries within the provided history and tracks the mutation +// records for them. +func (b *batchIndexer) process(h history, id uint64) error { + for ident := range h.forEach() { + b.index[ident] = append(b.index[ident], id) + b.pending++ } - b.lastID = historyID + b.lastID = id + return b.finish(false) } // finish writes the accumulated state indexes into the disk if either the // memory limitation is reached or it's requested forcibly. func (b *batchIndexer) finish(force bool) error { - if b.counter == 0 { + if b.pending == 0 { return nil } - if !force && b.counter < historyIndexBatch { + if !force && b.pending < historyIndexBatch { return nil } var ( - batch = b.db.NewBatch() - batchMu sync.RWMutex - storages int - start = time.Now() - eg errgroup.Group + batch = b.db.NewBatch() + batchMu sync.RWMutex + start = time.Now() + eg errgroup.Group ) eg.SetLimit(runtime.NumCPU()) - for addrHash, idList := range b.accounts { + for ident, list := range b.index { eg.Go(func() error { if !b.delete { - iw, err := newIndexWriter(b.db, newAccountIdent(addrHash)) + iw, err := newIndexWriter(b.db, ident) if err != nil { return err } - for _, n := range idList { + for _, n := range list { if err := iw.append(n); err != nil { return err } @@ -152,11 +173,11 @@ func (b *batchIndexer) finish(force bool) error { iw.finish(batch) batchMu.Unlock() } else { - id, err := newIndexDeleter(b.db, newAccountIdent(addrHash)) + id, err := newIndexDeleter(b.db, ident) if err != nil { return err } - for _, n := range idList { + for _, n := range list { if err := id.pop(n); err != nil { return err } @@ -168,72 +189,36 @@ func (b *batchIndexer) finish(force bool) error { return nil }) } - for addrHash, slots := range b.storages { - storages += len(slots) - for storageHash, idList := range slots { - eg.Go(func() error { - if !b.delete { - iw, err := newIndexWriter(b.db, newStorageIdent(addrHash, storageHash)) - if err != nil { - return err - } - for _, n := range idList { - if err := iw.append(n); err != nil { - return err - } - } - batchMu.Lock() - iw.finish(batch) - batchMu.Unlock() - } else { - id, err := newIndexDeleter(b.db, newStorageIdent(addrHash, storageHash)) - if err != nil { - return err - } - for _, n := range idList { - if err := id.pop(n); err != nil { - return err - } - } - batchMu.Lock() - id.finish(batch) - batchMu.Unlock() - } - return nil - }) - } - } if err := eg.Wait(); err != nil { return err } // Update the position of last indexed state history if !b.delete { - storeIndexMetadata(batch, b.lastID) + storeIndexMetadata(batch, b.typ, b.lastID) } else { if b.lastID == 1 { - rawdb.DeleteStateHistoryIndexMetadata(batch) + deleteIndexMetadata(batch, b.typ) } else { - storeIndexMetadata(batch, b.lastID-1) + storeIndexMetadata(batch, b.typ, b.lastID-1) } } if err := batch.Write(); err != nil { return err } - log.Debug("Committed batch indexer", "accounts", len(b.accounts), "storages", storages, "records", b.counter, "elapsed", common.PrettyDuration(time.Since(start))) - b.counter = 0 - b.accounts = make(map[common.Hash][]uint64) - b.storages = make(map[common.Hash]map[common.Hash][]uint64) + log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "elapsed", common.PrettyDuration(time.Since(start))) + b.pending = 0 + b.index = make(map[stateIdent][]uint64) return nil } // indexSingle processes the state history with the specified ID for indexing. -func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader) error { +func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error { start := time.Now() defer func() { indexHistoryTimer.UpdateSince(start) }() - metadata := loadIndexMetadata(db) + metadata := loadIndexMetadata(db, typ) if metadata == nil || metadata.Last+1 != historyID { last := "null" if metadata != nil { @@ -241,29 +226,37 @@ func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancient } return fmt.Errorf("history indexing is out of order, last: %s, requested: %d", last, historyID) } - h, err := readStateHistory(freezer, historyID) + var ( + err error + h history + b = newBatchIndexer(db, false, typ) + ) + if typ == typeStateHistory { + h, err = readStateHistory(freezer, historyID) + } else { + // h, err = readTrienodeHistory(freezer, historyID) + } if err != nil { return err } - b := newBatchIndexer(db, false) if err := b.process(h, historyID); err != nil { return err } if err := b.finish(true); err != nil { return err } - log.Debug("Indexed state history", "id", historyID, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Indexed history", "type", typ, "id", historyID, "elapsed", common.PrettyDuration(time.Since(start))) return nil } // unindexSingle processes the state history with the specified ID for unindexing. -func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader) error { +func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error { start := time.Now() defer func() { unindexHistoryTimer.UpdateSince(start) }() - metadata := loadIndexMetadata(db) + metadata := loadIndexMetadata(db, typ) if metadata == nil || metadata.Last != historyID { last := "null" if metadata != nil { @@ -271,18 +264,26 @@ func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancie } return fmt.Errorf("history unindexing is out of order, last: %s, requested: %d", last, historyID) } - h, err := readStateHistory(freezer, historyID) + var ( + err error + h history + ) + b := newBatchIndexer(db, true, typ) + if typ == typeStateHistory { + h, err = readStateHistory(freezer, historyID) + } else { + // h, err = readTrienodeHistory(freezer, historyID) + } if err != nil { return err } - b := newBatchIndexer(db, true) if err := b.process(h, historyID); err != nil { return err } if err := b.finish(true); err != nil { return err } - log.Debug("Unindexed state history", "id", historyID, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Unindexed history", "type", typ, "id", historyID, "elapsed", common.PrettyDuration(time.Since(start))) return nil } @@ -305,6 +306,8 @@ type indexIniter struct { interrupt chan *interruptSignal done chan struct{} closed chan struct{} + typ historyType + log log.Logger // Contextual logger with the history type injected // indexing progress indexed atomic.Uint64 // the id of latest indexed state @@ -313,18 +316,20 @@ type indexIniter struct { wg sync.WaitGroup } -func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastID uint64) *indexIniter { +func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, typ historyType, lastID uint64) *indexIniter { initer := &indexIniter{ disk: disk, freezer: freezer, interrupt: make(chan *interruptSignal), done: make(chan struct{}), closed: make(chan struct{}), + typ: typ, + log: log.New("type", typ.String()), } // Load indexing progress var recover bool initer.last.Store(lastID) - metadata := loadIndexMetadata(disk) + metadata := loadIndexMetadata(disk, typ) if metadata != nil { initer.indexed.Store(metadata.Last) recover = metadata.Last > lastID @@ -371,7 +376,7 @@ func (i *indexIniter) remain() uint64 { default: last, indexed := i.last.Load(), i.indexed.Load() if last < indexed { - log.Warn("State indexer is in recovery", "indexed", indexed, "last", last) + i.log.Warn("State indexer is in recovery", "indexed", indexed, "last", last) return indexed - last } return last - indexed @@ -389,7 +394,7 @@ func (i *indexIniter) run(lastID uint64) { // checkDone indicates whether all requested state histories // have been fully indexed. checkDone = func() bool { - metadata := loadIndexMetadata(i.disk) + metadata := loadIndexMetadata(i.disk, i.typ) return metadata != nil && metadata.Last == lastID } ) @@ -411,7 +416,7 @@ func (i *indexIniter) run(lastID uint64) { if newLastID == lastID+1 { lastID = newLastID signal.result <- nil - log.Debug("Extended state history range", "last", lastID) + i.log.Debug("Extended history range", "last", lastID) continue } // The index limit is shortened by one, interrupt the current background @@ -422,14 +427,14 @@ func (i *indexIniter) run(lastID uint64) { // If all state histories, including the one to be reverted, have // been fully indexed, unindex it here and shut down the initializer. if checkDone() { - log.Info("Truncate the extra history", "id", lastID) - if err := unindexSingle(lastID, i.disk, i.freezer); err != nil { + i.log.Info("Truncate the extra history", "id", lastID) + if err := unindexSingle(lastID, i.disk, i.freezer, i.typ); err != nil { signal.result <- err return } close(i.done) signal.result <- nil - log.Info("State histories have been fully indexed", "last", lastID-1) + i.log.Info("Histories have been fully indexed", "last", lastID-1) return } // Adjust the indexing target and relaunch the process @@ -438,12 +443,12 @@ func (i *indexIniter) run(lastID uint64) { done, interrupt = make(chan struct{}), new(atomic.Int32) go i.index(done, interrupt, lastID) - log.Debug("Shortened state history range", "last", lastID) + i.log.Debug("Shortened history range", "last", lastID) case <-done: if checkDone() { close(i.done) - log.Info("State histories have been fully indexed", "last", lastID) + i.log.Info("Histories have been fully indexed", "last", lastID) return } // Relaunch the background runner if some tasks are left @@ -452,7 +457,7 @@ func (i *indexIniter) run(lastID uint64) { case <-i.closed: interrupt.Store(1) - log.Info("Waiting background history index initer to exit") + i.log.Info("Waiting background history index initer to exit") <-done if checkDone() { @@ -472,14 +477,14 @@ func (i *indexIniter) next() (uint64, error) { tailID := tail + 1 // compute the id of the oldest history // Start indexing from scratch if nothing has been indexed - metadata := loadIndexMetadata(i.disk) + metadata := loadIndexMetadata(i.disk, i.typ) if metadata == nil { - log.Debug("Initialize state history indexing from scratch", "id", tailID) + i.log.Debug("Initialize history indexing from scratch", "id", tailID) return tailID, nil } // Resume indexing from the last interrupted position if metadata.Last+1 >= tailID { - log.Debug("Resume state history indexing", "id", metadata.Last+1, "tail", tailID) + i.log.Debug("Resume history indexing", "id", metadata.Last+1, "tail", tailID) return metadata.Last + 1, nil } // History has been shortened without indexing. Discard the gapped segment @@ -487,7 +492,7 @@ func (i *indexIniter) next() (uint64, error) { // // The missing indexes corresponding to the gapped histories won't be visible. // It's fine to leave them unindexed. - log.Info("History gap detected, discard old segment", "oldHead", metadata.Last, "newHead", tailID) + i.log.Info("History gap detected, discard old segment", "oldHead", metadata.Last, "newHead", tailID) return tailID, nil } @@ -496,7 +501,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID beginID, err := i.next() if err != nil { - log.Error("Failed to find next state history for indexing", "err", err) + i.log.Error("Failed to find next history for indexing", "err", err) return } // All available state histories have been indexed, and the last indexed one @@ -511,36 +516,47 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID // // This step is essential to avoid spinning up indexing thread // endlessly until a history object is produced. - storeIndexMetadata(i.disk, 0) - log.Info("Initialized history indexing flag") + storeIndexMetadata(i.disk, i.typ, 0) + i.log.Info("Initialized history indexing flag") } else { - log.Debug("State history is fully indexed", "last", lastID) + i.log.Debug("History is fully indexed", "last", lastID) } return } - log.Info("Start history indexing", "beginID", beginID, "lastID", lastID) + i.log.Info("Start history indexing", "beginID", beginID, "lastID", lastID) var ( current = beginID start = time.Now() logged = time.Now() - batch = newBatchIndexer(i.disk, false) + batch = newBatchIndexer(i.disk, false, i.typ) ) for current <= lastID { count := lastID - current + 1 if count > historyReadBatch { count = historyReadBatch } - histories, err := readStateHistories(i.freezer, current, count) - if err != nil { - // The history read might fall if the history is truncated from - // head due to revert operation. - log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err) - return + var histories []history + if i.typ == typeStateHistory { + histories, err = readStateHistories(i.freezer, current, count) + if err != nil { + // The history read might fall if the history is truncated from + // head due to revert operation. + i.log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err) + return + } + } else { + // histories, err = readTrienodeHistories(i.freezer, current, count) + // if err != nil { + // // The history read might fall if the history is truncated from + // // head due to revert operation. + // i.log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err) + // return + // } } for _, h := range histories { if err := batch.process(h, current); err != nil { - log.Error("Failed to index history", "err", err) + i.log.Error("Failed to index history", "err", err) return } current += 1 @@ -554,7 +570,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID done = current - beginID ) eta := common.CalculateETA(done, left, time.Since(start)) - log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) + i.log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) } } i.indexed.Store(current - 1) // update indexing progress @@ -563,7 +579,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID if interrupt != nil { if signal := interrupt.Load(); signal != 0 { if err := batch.finish(true); err != nil { - log.Error("Failed to flush index", "err", err) + i.log.Error("Failed to flush index", "err", err) } log.Info("State indexing interrupted") return @@ -571,9 +587,9 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID } } if err := batch.finish(true); err != nil { - log.Error("Failed to flush index", "err", err) + i.log.Error("Failed to flush index", "err", err) } - log.Info("Indexed state history", "from", beginID, "to", lastID, "elapsed", common.PrettyDuration(time.Since(start))) + i.log.Info("Indexed history", "from", beginID, "to", lastID, "elapsed", common.PrettyDuration(time.Since(start))) } // recover handles unclean shutdown recovery. After an unclean shutdown, any @@ -602,14 +618,14 @@ func (i *indexIniter) recover(lastID uint64) { lastID = newLastID signal.result <- nil i.last.Store(newLastID) - log.Debug("Updated history index flag", "last", lastID) + i.log.Debug("Updated history index flag", "last", lastID) // Terminate the recovery routine once the histories are fully aligned // with the index data, indicating that index initialization is complete. - metadata := loadIndexMetadata(i.disk) + metadata := loadIndexMetadata(i.disk, i.typ) if metadata != nil && metadata.Last == lastID { close(i.done) - log.Info("History indexer is recovered", "last", lastID) + i.log.Info("History indexer is recovered", "last", lastID) return } @@ -631,21 +647,31 @@ func (i *indexIniter) recover(lastID uint64) { // state history. type historyIndexer struct { initer *indexIniter + typ historyType disk ethdb.KeyValueStore freezer ethdb.AncientStore } // checkVersion checks whether the index data in the database matches the version. -func checkVersion(disk ethdb.KeyValueStore) { - blob := rawdb.ReadStateHistoryIndexMetadata(disk) +func checkVersion(disk ethdb.KeyValueStore, typ historyType) { + var blob []byte + if typ == typeStateHistory { + blob = rawdb.ReadStateHistoryIndexMetadata(disk) + } else { + panic(fmt.Errorf("unknown history type: %v", typ)) + } + // Short circuit if metadata is not found, re-index is required + // from scratch. if len(blob) == 0 { return } + // Short circuit if the metadata is found and the version is matched var m indexMetadata err := rlp.DecodeBytes(blob, &m) if err == nil && m.Version == stateIndexVersion { return } + // Version is not matched, prune the existing data and re-index from scratch version := "unknown" if err == nil { version = fmt.Sprintf("%d", m.Version) @@ -662,10 +688,11 @@ func checkVersion(disk ethdb.KeyValueStore) { // newHistoryIndexer constructs the history indexer and launches the background // initer to complete the indexing of any remaining state histories. -func newHistoryIndexer(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastHistoryID uint64) *historyIndexer { - checkVersion(disk) +func newHistoryIndexer(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastHistoryID uint64, typ historyType) *historyIndexer { + checkVersion(disk, typ) return &historyIndexer{ - initer: newIndexIniter(disk, freezer, lastHistoryID), + initer: newIndexIniter(disk, freezer, typ, lastHistoryID), + typ: typ, disk: disk, freezer: freezer, } @@ -693,7 +720,7 @@ func (i *historyIndexer) extend(historyID uint64) error { case <-i.initer.closed: return errors.New("indexer is closed") case <-i.initer.done: - return indexSingle(historyID, i.disk, i.freezer) + return indexSingle(historyID, i.disk, i.freezer, i.typ) case i.initer.interrupt <- signal: return <-signal.result } @@ -710,7 +737,7 @@ func (i *historyIndexer) shorten(historyID uint64) error { case <-i.initer.closed: return errors.New("indexer is closed") case <-i.initer.done: - return unindexSingle(historyID, i.disk, i.freezer) + return unindexSingle(historyID, i.disk, i.freezer, i.typ) case i.initer.interrupt <- signal: return <-signal.result } diff --git a/triedb/pathdb/history_indexer_test.go b/triedb/pathdb/history_indexer_test.go index 96c87ccb1b..f333d18d8b 100644 --- a/triedb/pathdb/history_indexer_test.go +++ b/triedb/pathdb/history_indexer_test.go @@ -38,7 +38,7 @@ func TestHistoryIndexerShortenDeadlock(t *testing.T) { rawdb.WriteStateHistory(freezer, uint64(i+1), h.meta.encode(), accountIndex, storageIndex, accountData, storageData) } // As a workaround, assign a future block to keep the initer running indefinitely - indexer := newHistoryIndexer(db, freezer, 200) + indexer := newHistoryIndexer(db, freezer, 200, typeStateHistory) defer indexer.close() done := make(chan error, 1) diff --git a/triedb/pathdb/history_reader.go b/triedb/pathdb/history_reader.go index a11297b3f6..ce6aa693d1 100644 --- a/triedb/pathdb/history_reader.go +++ b/triedb/pathdb/history_reader.go @@ -29,88 +29,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" ) -// stateIdent represents the identifier of a state element, which can be -// either an account or a storage slot. -type stateIdent struct { - account bool - - // The hash of the account address. This is used instead of the raw account - // address is to align the traversal order with the Merkle-Patricia-Trie. - addressHash common.Hash - - // The hash of the storage slot key. This is used instead of the raw slot key - // because, in legacy state histories (prior to the Cancun fork), the slot - // identifier is the hash of the key, and the original key (preimage) cannot - // be recovered. To maintain backward compatibility, the key hash is used. - // - // Meanwhile, using the storage key hash also preserve the traversal order - // with Merkle-Patricia-Trie. - // - // This field is null if the identifier refers to account data. - storageHash common.Hash -} - -// String returns the string format state identifier. -func (ident stateIdent) String() string { - if ident.account { - return ident.addressHash.Hex() - } - return ident.addressHash.Hex() + ident.storageHash.Hex() -} - -// newAccountIdent constructs a state identifier for an account. -func newAccountIdent(addressHash common.Hash) stateIdent { - return stateIdent{ - account: true, - addressHash: addressHash, - } -} - -// newStorageIdent constructs a state identifier for a storage slot. -// The address denotes the address of the associated account; -// the storageHash denotes the hash of the raw storage slot key; -func newStorageIdent(addressHash common.Hash, storageHash common.Hash) stateIdent { - return stateIdent{ - addressHash: addressHash, - storageHash: storageHash, - } -} - -// stateIdentQuery is the extension of stateIdent by adding the raw storage key. -type stateIdentQuery struct { - stateIdent - - address common.Address - storageKey common.Hash -} - -// newAccountIdentQuery constructs a state identifier for an account. -func newAccountIdentQuery(address common.Address, addressHash common.Hash) stateIdentQuery { - return stateIdentQuery{ - stateIdent: stateIdent{ - account: true, - addressHash: addressHash, - }, - address: address, - } -} - -// newStorageIdentQuery constructs a state identifier for a storage slot. -// the address denotes the address of the associated account; -// the addressHash denotes the address hash of the associated account; -// the storageKey denotes the raw storage slot key; -// the storageHash denotes the hash of the raw storage slot key; -func newStorageIdentQuery(address common.Address, addressHash common.Hash, storageKey common.Hash, storageHash common.Hash) stateIdentQuery { - return stateIdentQuery{ - stateIdent: stateIdent{ - addressHash: addressHash, - storageHash: storageHash, - }, - address: address, - storageKey: storageKey, - } -} - // indexReaderWithLimitTag is a wrapper around indexReader that includes an // additional index position. This position represents the ID of the last // indexed state history at the time the reader was created, implying that @@ -169,7 +87,7 @@ func (r *indexReaderWithLimitTag) readGreaterThan(id uint64, lastID uint64) (uin // Given that it's very unlikely to occur and users try to perform historical // state queries while reverting the states at the same time. Simply returning // an error should be sufficient for now. - metadata := loadIndexMetadata(r.db) + metadata := loadIndexMetadata(r.db, toHistoryType(r.reader.state.typ)) if metadata == nil || metadata.Last < lastID { return 0, errors.New("state history hasn't been indexed yet") } @@ -331,7 +249,7 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 // To serve the request, all state histories from stateID+1 to lastID // must be indexed. It's not supposed to happen unless system is very // wrong. - metadata := loadIndexMetadata(r.disk) + metadata := loadIndexMetadata(r.disk, toHistoryType(state.typ)) if metadata == nil || metadata.Last < lastID { indexed := "null" if metadata != nil { @@ -364,7 +282,7 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 // that the associated state histories are no longer available due to a rollback. // Such truncation should be captured by the state resolver below, rather than returning // invalid data. - if state.account { + if state.typ == typeAccount { return r.readAccount(state.address, historyID) } return r.readStorage(state.address, state.storageKey, state.storageHash, historyID) diff --git a/triedb/pathdb/history_reader_test.go b/triedb/pathdb/history_reader_test.go index 75c5f701f9..3e1a545ff3 100644 --- a/triedb/pathdb/history_reader_test.go +++ b/triedb/pathdb/history_reader_test.go @@ -29,7 +29,7 @@ import ( func waitIndexing(db *Database) { for { - metadata := loadIndexMetadata(db.diskdb) + metadata := loadIndexMetadata(db.diskdb, typeStateHistory) if metadata != nil && metadata.Last >= db.tree.bottom().stateID() { return } diff --git a/triedb/pathdb/history_state.go b/triedb/pathdb/history_state.go index 3bb69a7f4d..9d1e4dfb09 100644 --- a/triedb/pathdb/history_state.go +++ b/triedb/pathdb/history_state.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "iter" "maps" "slices" "time" @@ -275,6 +276,36 @@ func newStateHistory(root common.Hash, parent common.Hash, block uint64, account } } +// typ implements the history interface, returning the historical data type held. +func (h *stateHistory) typ() historyType { + return typeStateHistory +} + +// forEach implements the history interface, returning an iterator to traverse the +// state entries in the history. +func (h *stateHistory) forEach() iter.Seq[stateIdent] { + return func(yield func(stateIdent) bool) { + for _, addr := range h.accountList { + addrHash := crypto.Keccak256Hash(addr.Bytes()) + if !yield(newAccountIdent(addrHash)) { + return + } + for _, slotKey := range h.storageList[addr] { + // The hash of the storage slot key is used as the identifier because the + // legacy history does not include the raw storage key, therefore, the + // conversion from storage key to hash is necessary for non-v0 histories. + slotHash := slotKey + if h.meta.version != stateHistoryV0 { + slotHash = crypto.Keccak256Hash(slotKey.Bytes()) + } + if !yield(newStorageIdent(addrHash, slotHash)) { + return + } + } + } + } +} + // stateSet returns the state set, keyed by the hash of the account address // and the hash of the storage slot key. func (h *stateHistory) stateSet() (map[common.Hash][]byte, map[common.Hash]map[common.Hash][]byte) { @@ -536,8 +567,8 @@ func readStateHistory(reader ethdb.AncientReader, id uint64) (*stateHistory, err } // readStateHistories reads a list of state history records within the specified range. -func readStateHistories(freezer ethdb.AncientReader, start uint64, count uint64) ([]*stateHistory, error) { - var histories []*stateHistory +func readStateHistories(freezer ethdb.AncientReader, start uint64, count uint64) ([]history, error) { + var histories []history metaList, aIndexList, sIndexList, aDataList, sDataList, err := rawdb.ReadStateHistoryList(freezer, start, count) if err != nil { return nil, err diff --git a/triedb/pathdb/history_state_test.go b/triedb/pathdb/history_state_test.go index 4a777111ea..5718081566 100644 --- a/triedb/pathdb/history_state_test.go +++ b/triedb/pathdb/history_state_test.go @@ -137,7 +137,7 @@ func TestTruncateHeadStateHistory(t *testing.T) { rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) } for size := len(hs); size > 0; size-- { - pruned, err := truncateFromHead(freezer, uint64(size-1)) + pruned, err := truncateFromHead(freezer, typeStateHistory, uint64(size-1)) if err != nil { t.Fatalf("Failed to truncate from head %v", err) } @@ -161,7 +161,7 @@ func TestTruncateTailStateHistory(t *testing.T) { rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) } for newTail := 1; newTail < len(hs); newTail++ { - pruned, _ := truncateFromTail(freezer, uint64(newTail)) + pruned, _ := truncateFromTail(freezer, typeStateHistory, uint64(newTail)) if pruned != 1 { t.Error("Unexpected pruned items", "want", 1, "got", pruned) } @@ -209,7 +209,7 @@ func TestTruncateTailStateHistories(t *testing.T) { accountData, storageData, accountIndex, storageIndex := hs[i].encode() rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) } - pruned, _ := truncateFromTail(freezer, uint64(10)-c.limit) + pruned, _ := truncateFromTail(freezer, typeStateHistory, uint64(10)-c.limit) if pruned != c.expPruned { t.Error("Unexpected pruned items", "want", c.expPruned, "got", pruned) } @@ -233,7 +233,7 @@ func TestTruncateOutOfRange(t *testing.T) { accountData, storageData, accountIndex, storageIndex := hs[i].encode() rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) } - truncateFromTail(freezer, uint64(len(hs)/2)) + truncateFromTail(freezer, typeStateHistory, uint64(len(hs)/2)) // Ensure of-out-range truncations are rejected correctly. head, _ := freezer.Ancients() @@ -254,9 +254,9 @@ func TestTruncateOutOfRange(t *testing.T) { for _, c := range cases { var gotErr error if c.mode == 0 { - _, gotErr = truncateFromHead(freezer, c.target) + _, gotErr = truncateFromHead(freezer, typeStateHistory, c.target) } else { - _, gotErr = truncateFromTail(freezer, c.target) + _, gotErr = truncateFromTail(freezer, typeStateHistory, c.target) } if !errors.Is(gotErr, c.expErr) { t.Errorf("Unexpected error, want: %v, got: %v", c.expErr, gotErr)