From a25be32fa92cfc27b82930a41a10f538d8479e2e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 22 Nov 2024 15:17:32 +0800 Subject: [PATCH 01/16] core, eth, internal, miner: remove unnecessary parameters (#30776) Follow-up to #30745 , this change removes some unnecessary parameters. --- cmd/evm/internal/t8ntool/execution.go | 8 ++-- core/chain_makers.go | 15 +++--- core/state_processor.go | 67 +++++++++++++-------------- core/verkle_witness_test.go | 2 +- eth/state_accessor.go | 4 +- eth/tracers/api.go | 18 +++---- internal/ethapi/simulate.go | 2 +- miner/worker.go | 10 ++-- 8 files changed, 61 insertions(+), 65 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index a868fbf11c..c0623a69bf 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -203,14 +203,14 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig) if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if pre.Env.BlockHashes != nil && chainConfig.IsPrague(new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) { var ( prevNumber = pre.Env.Number - 1 prevHash = pre.Env.BlockHashes[math.HexOrDecimal64(prevNumber)] ) - core.ProcessParentBlockHash(prevHash, evm, statedb) + core.ProcessParentBlockHash(prevHash, evm) } for i := 0; txIt.Next(); i++ { tx, err := txIt.Tx() @@ -378,9 +378,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, requests = append(requests, depositRequests) // EIP-7002 withdrawals - requests = append(requests, core.ProcessWithdrawalQueue(evm, statedb)) + requests = append(requests, core.ProcessWithdrawalQueue(evm)) // EIP-7251 consolidations - requests = append(requests, core.ProcessConsolidationQueue(evm, statedb)) + requests = append(requests, core.ProcessConsolidationQueue(evm)) } // Commit block diff --git a/core/chain_makers.go b/core/chain_makers.go index 07448abc3d..e679a9e557 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -98,11 +98,8 @@ func (b *BlockGen) Difficulty() *big.Int { // block. func (b *BlockGen) SetParentBeaconRoot(root common.Hash) { b.header.ParentBeaconRoot = &root - var ( - blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) - evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{}) - ) - ProcessBeaconBlockRoot(root, evm, b.statedb) + blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) + ProcessBeaconBlockRoot(root, vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{})) } // addTx adds a transaction to the generated block. If no coinbase has @@ -121,7 +118,7 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(b.cm.config, evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) + receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) if err != nil { panic(err) } @@ -366,10 +363,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(evm, statedb) + withdrawalRequests := ProcessWithdrawalQueue(evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(evm, statedb) + consolidationRequests := ProcessConsolidationQueue(evm) requests = append(requests, consolidationRequests) } if requests != nil { @@ -471,7 +468,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine // EIP-2935 blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, evm, statedb) + ProcessParentBlockHash(b.header.ParentHash, evm) } // Execute any user modifications to the block. diff --git a/core/state_processor.go b/core/state_processor.go index 74985b69a1..1703377111 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -82,10 +82,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg evm := vm.NewEVM(context, tracingStateDB, p.config, cfg) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - ProcessBeaconBlockRoot(*beaconRoot, evm, tracingStateDB) + ProcessBeaconBlockRoot(*beaconRoot, evm) } if p.config.IsPrague(block.Number(), block.Time()) { - ProcessParentBlockHash(block.ParentHash(), evm, tracingStateDB) + ProcessParentBlockHash(block.ParentHash(), evm) } // Iterate over and process the individual transactions @@ -96,7 +96,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } statedb.SetTxContext(tx.Hash(), i) - receipt, err := ApplyTransactionWithEVM(msg, p.config, gp, statedb, blockNumber, blockHash, tx, usedGas, evm) + receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, tx, usedGas, evm) if err != nil { return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -113,10 +113,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } requests = append(requests, depositRequests) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(evm, tracingStateDB) + withdrawalRequests := ProcessWithdrawalQueue(evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(evm, tracingStateDB) + consolidationRequests := ProcessConsolidationQueue(evm) requests = append(requests, consolidationRequests) } @@ -134,7 +134,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // ApplyTransactionWithEVM attempts to apply a transaction to the given state database // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. -func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { +func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { if hooks := evm.Config.Tracer; hooks != nil { if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) @@ -156,10 +156,10 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo // Update the state with pending changes. var root []byte - if config.IsByzantium(blockNumber) { + if evm.ChainConfig().IsByzantium(blockNumber) { evm.StateDB.Finalise(true) } else { - root = statedb.IntermediateRoot(config.IsEIP158(blockNumber)).Bytes() + root = statedb.IntermediateRoot(evm.ChainConfig().IsEIP158(blockNumber)).Bytes() } *usedGas += result.UsedGas @@ -208,19 +208,19 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { - msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) +func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { + msg, err := TransactionToMessage(tx, types.MakeSigner(evm.ChainConfig(), header.Number, header.Time), header.BaseFee) if err != nil { return nil, err } // Create a new context to be used in the EVM environment - return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, evm) + return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), tx, usedGas, evm) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root // contract. This method is exported to be used in tests. -func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.StateDB) { - if tracer := vmenv.Config.Tracer; tracer != nil { +func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -237,16 +237,16 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.St To: ¶ms.BeaconRootsAddress, Data: beaconRoot[:], } - vmenv.SetTxContext(NewEVMTxContext(msg)) - statedb.AddAddressToAccessList(params.BeaconRootsAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(params.BeaconRootsAddress) + _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) } // ProcessParentBlockHash stores the parent block hash in the history storage contract // as per EIP-2935. -func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.StateDB) { - if tracer := vmenv.Config.Tracer; tracer != nil { +func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -263,26 +263,26 @@ func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.Stat To: ¶ms.HistoryStorageAddress, Data: prevHash.Bytes(), } - vmenv.SetTxContext(NewEVMTxContext(msg)) - statedb.AddAddressToAccessList(params.HistoryStorageAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(params.HistoryStorageAddress) + _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) } // ProcessWithdrawalQueue calls the EIP-7002 withdrawal queue contract. // It returns the opaque request data returned by the contract. -func ProcessWithdrawalQueue(vmenv *vm.EVM, statedb vm.StateDB) []byte { - return processRequestsSystemCall(vmenv, statedb, 0x01, params.WithdrawalQueueAddress) +func ProcessWithdrawalQueue(evm *vm.EVM) []byte { + return processRequestsSystemCall(evm, 0x01, params.WithdrawalQueueAddress) } // ProcessConsolidationQueue calls the EIP-7251 consolidation queue contract. // It returns the opaque request data returned by the contract. -func ProcessConsolidationQueue(vmenv *vm.EVM, statedb vm.StateDB) []byte { - return processRequestsSystemCall(vmenv, statedb, 0x02, params.ConsolidationQueueAddress) +func ProcessConsolidationQueue(evm *vm.EVM) []byte { + return processRequestsSystemCall(evm, 0x02, params.ConsolidationQueueAddress) } -func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType byte, addr common.Address) []byte { - if tracer := vmenv.Config.Tracer; tracer != nil { +func processRequestsSystemCall(evm *vm.EVM, requestType byte, addr common.Address) []byte { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -290,7 +290,6 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by defer tracer.OnSystemCallEnd() } } - msg := &Message{ From: params.SystemAddress, GasLimit: 30_000_000, @@ -299,10 +298,10 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by GasTipCap: common.Big0, To: &addr, } - vmenv.SetTxContext(NewEVMTxContext(msg)) - statedb.AddAddressToAccessList(addr) - ret, _, _ := vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(addr) + ret, _, _ := evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) // Create withdrawals requestsData with prefix 0x01 requestsData := make([]byte, len(ret)+1) diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 80996eee33..72df5473bb 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -226,7 +226,7 @@ func TestProcessParentBlockHash(t *testing.T) { header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} vmContext := NewEVMBlockContext(header, nil, new(common.Address)) evm := vm.NewEVM(vmContext, statedb, params.MergedTestChainConfig, vm.Config{}) - ProcessParentBlockHash(header.ParentHash, evm, statedb) + ProcessParentBlockHash(header.ParentHash, evm) } // Read block hashes for block 0 .. num-1 for i := 0; i < num; i++ { diff --git a/eth/state_accessor.go b/eth/state_accessor.go index a919030249..43432cff31 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -238,11 +238,11 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) evm := vm.NewEVM(context, statedb, eth.blockchain.Config(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } // If prague hardfork, insert parent block hash in the state as per EIP-2935. if eth.blockchain.Config().IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } if txIndex == 0 && len(block.Transactions()) == 0 { return nil, vm.BlockContext{}, statedb, release, nil diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 671a86e78a..a2c11e0fe2 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -381,11 +381,11 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) evm := vm.NewEVM(context, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := next.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } // Insert parent hash in history contract. if api.backend.ChainConfig().IsPrague(next.Number(), next.Time()) { - core.ProcessParentBlockHash(next.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(next.ParentHash(), evm) } // Clean out any pending release functions of trace state. Note this // step must be done after constructing tracing state, because the @@ -537,10 +537,10 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config ) evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if chainConfig.IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } for i, tx := range block.Transactions() { if err := ctx.Err(); err != nil { @@ -605,10 +605,10 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) evm := vm.NewEVM(blockCtx, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if api.backend.ChainConfig().IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } // JS tracers have high overhead. In this case run a parallel @@ -784,10 +784,10 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if chainConfig.IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } for i, tx := range block.Transactions() { // Prepare the transaction for un-traced execution @@ -1037,7 +1037,7 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, err = core.ApplyTransactionWithEVM(message, api.backend.ChainConfig(), new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, evm) + _, err = core.ApplyTransactionWithEVM(message, new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, evm) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 979de352fd..0b58a27096 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -265,7 +265,7 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state *state.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) diff --git a/miner/worker.go b/miner/worker.go index 51ac5e2e36..aeb6cfcdc2 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -128,10 +128,10 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo } requests = append(requests, depositRequests) // EIP-7002 withdrawals - withdrawalRequests := core.ProcessWithdrawalQueue(work.evm, work.state) + withdrawalRequests := core.ProcessWithdrawalQueue(work.evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := core.ProcessConsolidationQueue(work.evm, work.state) + consolidationRequests := core.ProcessConsolidationQueue(work.evm) requests = append(requests, consolidationRequests) } if requests != nil { @@ -231,10 +231,10 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir return nil, err } if header.ParentBeaconRoot != nil { - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, env.evm, env.state) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, env.evm) } if miner.chainConfig.IsPrague(header.Number, header.Time) { - core.ProcessParentBlockHash(header.ParentHash, env.evm, env.state) + core.ProcessParentBlockHash(header.ParentHash, env.evm) } return env, nil } @@ -309,7 +309,7 @@ func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (* snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(miner.chainConfig, env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) From 2cd25fdd23b695cd8caf3ad0b0701d68bf9d99a3 Mon Sep 17 00:00:00 2001 From: "Hyunsoo Shin (Lake)" Date: Fri, 22 Nov 2024 16:21:20 +0900 Subject: [PATCH 02/16] internal/ethapi: remove double map-clone (#30788) `ActivePrecompiledContracts()` clones the precompiled contract map, thus its callsite does not need to clone it --- internal/ethapi/api.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9c08006f1e..ded4db50aa 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "errors" "fmt" - "maps" gomath "math" "math/big" "strings" @@ -254,7 +253,7 @@ func (api *TxPoolAPI) Inspect() map[string]map[string]map[string]string { pending, queue := api.b.TxPoolContent() // Define a formatter to flatten a transaction into a string - var format = func(tx *types.Transaction) string { + format := func(tx *types.Transaction) string { if to := tx.To(); to != nil { return fmt.Sprintf("%s: %v wei + %v gas × %v wei", tx.To().Hex(), tx.Value(), tx.Gas(), tx.GasPrice()) } @@ -825,7 +824,7 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S blockOverrides.Apply(&blockCtx) } rules := b.ChainConfig().Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time) - precompiles := maps.Clone(vm.ActivePrecompiledContracts(rules)) + precompiles := vm.ActivePrecompiledContracts(rules) if err := overrides.Apply(state, precompiles); err != nil { return nil, err } @@ -1762,11 +1761,11 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, matchTx := sendArgs.ToTransaction(types.LegacyTxType) // Before replacing the old transaction, ensure the _new_ transaction fee is reasonable. - var price = matchTx.GasPrice() + price := matchTx.GasPrice() if gasPrice != nil { price = gasPrice.ToInt() } - var gas = matchTx.Gas() + gas := matchTx.Gas() if gasLimit != nil { gas = uint64(*gasLimit) } From 16f2f7155fc92b94857d6d43e3758d3f71552618 Mon Sep 17 00:00:00 2001 From: wangjingcun Date: Fri, 22 Nov 2024 16:02:45 +0800 Subject: [PATCH 03/16] all: typos in comments (#30779) fixes some typos --- build/ci.go | 2 +- core/verkle_witness_test.go | 2 +- core/vm/eof.go | 2 +- trie/utils/verkle.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/ci.go b/build/ci.go index 754d88a86a..e3115af46d 100644 --- a/build/ci.go +++ b/build/ci.go @@ -256,7 +256,7 @@ func buildFlags(env build.Environment, staticLinking bool, buildTags []string) ( // See https://sourceware.org/binutils/docs-2.23.1/ld/Options.html#Options // regarding the options --build-id=none and --strip-all. It is needed for // reproducible builds; removing references to temporary files in C-land, and - // making build-id reproducably absent. + // making build-id reproducibly absent. extld := []string{"-Wl,-z,stack-size=0x800000,--build-id=none,--strip-all"} if staticLinking { extld = append(extld, "-static") diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 72df5473bb..45b317d3c0 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -1033,7 +1033,7 @@ func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount ) // Prefund the account, at an address that the contract will be deployed at, // before it selfdestrucs. We can therefore check that the account itseld is - // NOT destroyed, which is what the currrent version of the spec requires. + // NOT destroyed, which is what the current version of the spec requires. // TODO(gballet) revisit after the spec has been modified. gspec.Alloc[contract] = types.Account{ Balance: big.NewInt(100), diff --git a/core/vm/eof.go b/core/vm/eof.go index fcced6ae75..a5406283d5 100644 --- a/core/vm/eof.go +++ b/core/vm/eof.go @@ -96,7 +96,7 @@ func (meta *functionMetadata) checkInputs(stackMin int) error { } // checkStackMax checks the if current maximum stack combined with the -// functin max stack will result in a stack overflow, and if so returns an error. +// function max stack will result in a stack overflow, and if so returns an error. func (meta *functionMetadata) checkStackMax(stackMax int) error { newMaxStack := stackMax + int(meta.maxStackHeight) - int(meta.inputs) if newMaxStack > int(params.StackLimit) { diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go index 4294f5e4be..b785e512b1 100644 --- a/trie/utils/verkle.go +++ b/trie/utils/verkle.go @@ -217,7 +217,7 @@ func StorageIndex(storageKey []byte) (*uint256.Int, byte) { // The first MAIN_STORAGE_OFFSET group will see its // first 64 slots unreachable. This is either a typo in the // spec or intended to conserve the 256-u256 - // aligment. If we decide to ever access these 64 + // alignment. If we decide to ever access these 64 // slots, uncomment this. // // Get the new offset since we now know that we are above 64. // pos.Sub(&pos, codeStorageDelta) From 6eeff3ee7dd87125a6791a991977b9664d8924ba Mon Sep 17 00:00:00 2001 From: j2gg0s Date: Fri, 22 Nov 2024 16:16:42 +0800 Subject: [PATCH 04/16] trie: replace custom logic with bytes.HasPrefix (#30771) in `trie` - Replace custom logic with `bytes.HasPrefix` - Remove unnecessary code in `GetNode` --- trie/proof.go | 8 ++++---- trie/trie.go | 7 ++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/trie/proof.go b/trie/proof.go index a39d6b4ea3..40836fba33 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -48,7 +48,7 @@ func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if !bytes.HasPrefix(key, n.Key) { // The trie doesn't contain the key. tn = nil } else { @@ -371,7 +371,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error } return unset(cld, cld.Children[key[pos]], key, pos+1, removeLeft) case *shortNode: - if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) { + if !bytes.HasPrefix(key[pos:], cld.Key) { // Find the fork point, it's a non-existent branch. if removeLeft { if bytes.Compare(cld.Key, key[pos:]) < 0 { @@ -434,7 +434,7 @@ func hasRightElement(node node, key []byte) bool { } node, pos = rn.Children[key[pos]], pos+1 case *shortNode: - if len(key)-pos < len(rn.Key) || !bytes.Equal(rn.Key, key[pos:pos+len(rn.Key)]) { + if !bytes.HasPrefix(key[pos:], rn.Key) { return bytes.Compare(rn.Key, key[pos:]) > 0 } node, pos = rn.Val, pos+len(rn.Key) @@ -589,7 +589,7 @@ func get(tn node, key []byte, skipResolved bool) ([]byte, node) { for { switch n := tn.(type) { case *shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if !bytes.HasPrefix(key, n.Key) { return nil, nil } tn = n.Val diff --git a/trie/trie.go b/trie/trie.go index e3f3f39248..ae2a7b21a2 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -163,7 +163,7 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no case valueNode: return n, n, false, nil case *shortNode: - if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) { + if !bytes.HasPrefix(key[pos:], n.Key) { // key not found in trie return nil, n, false, nil } @@ -219,9 +219,6 @@ func (t *Trie) GetNode(path []byte) ([]byte, int, error) { if resolved > 0 { t.root = newroot } - if item == nil { - return nil, resolved, nil - } return item, resolved, nil } @@ -254,7 +251,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod return nil, nil, 0, nil case *shortNode: - if len(path)-pos < len(n.Key) || !bytes.Equal(n.Key, path[pos:pos+len(n.Key)]) { + if !bytes.HasPrefix(path[pos:], n.Key) { // Path branches off from short node return nil, n, 0, nil } From 6485d5e3ffc08ce9ad071eee28a08110ae7f93fd Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 22 Nov 2024 16:55:43 +0800 Subject: [PATCH 05/16] core, triedb: remove destruct flag in state snapshot (#30752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request removes the destruct flag from the state snapshot to simplify the code. Previously, this flag indicated that an account was removed during a state transition, making all associated storage slots inaccessible. Because storage deletion can involve a large number of slots, the actual deletion is deferred until the end of the process, where it is handled in batches. With the deprecation of self-destruct in the Cancun fork, storage deletions are no longer expected. Historically, the largest storage deletion event in Ethereum was around 15 megabytes—manageable in memory. In this pull request, the single destruct flag is replaced by a set of deletion markers for individual storage slots. Each deleted storage slot will now appear in the Storage set with a nil value. This change will simplify a lot logics, such as storage accessing, storage flushing, storage iteration and so on. --- core/state/snapshot/conversion.go | 10 -- core/state/snapshot/difflayer.go | 117 +++++------------------ core/state/snapshot/difflayer_test.go | 123 +++++++++--------------- core/state/snapshot/disklayer.go | 4 +- core/state/snapshot/disklayer_test.go | 86 ++++++++++------- core/state/snapshot/generate_test.go | 2 +- core/state/snapshot/iterator.go | 17 ++-- core/state/snapshot/iterator_binary.go | 33 +------ core/state/snapshot/iterator_fast.go | 10 +- core/state/snapshot/iterator_test.go | 126 ++++++++++++------------- core/state/snapshot/journal.go | 37 +++----- core/state/snapshot/snapshot.go | 61 +++++------- core/state/snapshot/snapshot_test.go | 30 +++--- core/state/snapshot/utils.go | 12 +-- core/state/statedb.go | 69 ++++++++------ core/state/statedb_fuzz_test.go | 105 ++++++++++++++------- core/state/statedb_test.go | 4 +- core/state/stateupdate.go | 50 ++++++---- triedb/states.go | 2 - 19 files changed, 384 insertions(+), 514 deletions(-) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 8a0fd1989a..4b0774f2ae 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -50,16 +50,6 @@ type ( leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) ) -// GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. -func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(nil, "", it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) -} - -// GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. -func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(nil, "", it, account, stackTrieGenerate, nil, newGenerateStats(), true) -} - // GenerateTrie takes the whole snapshot tree as the input, traverses all the // accounts as well as the corresponding storages and regenerate the whole state // (account trie + all storage tries). diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 779c1ea98c..dce4f79a11 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" bloomfilter "github.com/holiman/bloomfilter/v2" + "golang.org/x/exp/maps" ) var ( @@ -73,23 +74,14 @@ var ( // bloom key for an account/slot. This is randomized at init(), so that the // global population of nodes do not all display the exact same behaviour with // regards to bloom content - bloomDestructHasherOffset = 0 - bloomAccountHasherOffset = 0 - bloomStorageHasherOffset = 0 + bloomAccountHasherOffset = 0 + bloomStorageHasherOffset = 0 ) func init() { // Init the bloom offsets in the range [0:24] (requires 8 bytes) - bloomDestructHasherOffset = rand.Intn(25) bloomAccountHasherOffset = rand.Intn(25) bloomStorageHasherOffset = rand.Intn(25) - - // The destruct and account blooms must be different, as the storage slots - // will check for destruction too for every bloom miss. It should not collide - // with modified accounts. - for bloomAccountHasherOffset == bloomDestructHasherOffset { - bloomAccountHasherOffset = rand.Intn(25) - } } // diffLayer represents a collection of modifications made to a state snapshot @@ -106,29 +98,16 @@ type diffLayer struct { root common.Hash // Root hash to which this snapshot diff belongs to stale atomic.Bool // Signals that the layer became stale (state progressed) - // destructSet is a very special helper marker. If an account is marked as - // deleted, then it's recorded in this set. However it's allowed that an account - // is included here but still available in other sets(e.g. storageData). The - // reason is the diff layer includes all the changes in a *block*. It can - // happen that in the tx_1, account A is self-destructed while in the tx_2 - // it's recreated. But we still need this marker to indicate the "old" A is - // deleted, all data in other set belongs to the "new" A. - destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts - accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil accountData map[common.Hash][]byte // Keyed accounts for direct retrieval (nil means deleted) - storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted) + accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil + storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer lock sync.RWMutex } -// destructBloomHash is used to convert a destruct event into a 64 bit mini hash. -func destructBloomHash(h common.Hash) uint64 { - return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8]) -} - // accountBloomHash is used to convert an account hash into a 64 bit mini hash. func accountBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8]) @@ -142,12 +121,11 @@ func storageBloomHash(h0, h1 common.Hash) uint64 { // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low // level persistent database or a hierarchical diff already. -func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { +func newDiffLayer(parent snapshot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { // Create the new layer with some pre-allocated data segments dl := &diffLayer{ parent: parent, root: root, - destructSet: destructs, accountData: accounts, storageData: storage, storageList: make(map[common.Hash][]common.Hash), @@ -161,10 +139,7 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s panic("unknown parent type") } // Sanity check that accounts or storage slots are never nil - for accountHash, blob := range accounts { - if blob == nil { - panic(fmt.Sprintf("account %#x nil", accountHash)) - } + for _, blob := range accounts { // Determine memory size and track the dirty writes dl.memory += uint64(common.HashLength + len(blob)) snapshotDirtyAccountWriteMeter.Mark(int64(len(blob))) @@ -179,7 +154,6 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s snapshotDirtyStorageWriteMeter.Mark(int64(len(data))) } } - dl.memory += uint64(len(destructs) * common.HashLength) return dl } @@ -204,10 +178,6 @@ func (dl *diffLayer) rebloom(origin *diskLayer) { } else { dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs)) } - // Iterate over all the accounts and storage slots and index them - for hash := range dl.destructSet { - dl.diffed.AddHash(destructBloomHash(hash)) - } for hash := range dl.accountData { dl.diffed.AddHash(accountBloomHash(hash)) } @@ -274,11 +244,8 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { } // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below - hit := dl.diffed.ContainsHash(accountBloomHash(hash)) - if !hit { - hit = dl.diffed.ContainsHash(destructBloomHash(hash)) - } var origin *diskLayer + hit := dl.diffed.ContainsHash(accountBloomHash(hash)) if !hit { origin = dl.origin // extract origin while holding the lock } @@ -310,18 +277,14 @@ func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) { if data, ok := dl.accountData[hash]; ok { snapshotDirtyAccountHitMeter.Mark(1) snapshotDirtyAccountHitDepthHist.Update(int64(depth)) - snapshotDirtyAccountReadMeter.Mark(int64(len(data))) + if n := len(data); n > 0 { + snapshotDirtyAccountReadMeter.Mark(int64(n)) + } else { + snapshotDirtyAccountInexMeter.Mark(1) + } snapshotBloomAccountTrueHitMeter.Mark(1) return data, nil } - // If the account is known locally, but deleted, return it - if _, ok := dl.destructSet[hash]; ok { - snapshotDirtyAccountHitMeter.Mark(1) - snapshotDirtyAccountHitDepthHist.Update(int64(depth)) - snapshotDirtyAccountInexMeter.Mark(1) - snapshotBloomAccountTrueHitMeter.Mark(1) - return nil, nil - } // Account unknown to this diff, resolve from parent if diff, ok := dl.parent.(*diffLayer); ok { return diff.accountRLP(hash, depth+1) @@ -345,11 +308,8 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro dl.lock.RUnlock() return nil, ErrSnapshotStale } - hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) - if !hit { - hit = dl.diffed.ContainsHash(destructBloomHash(accountHash)) - } var origin *diskLayer + hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) if !hit { origin = dl.origin // extract origin while holding the lock } @@ -391,14 +351,6 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([ return data, nil } } - // If the account is known locally, but deleted, return an empty slot - if _, ok := dl.destructSet[accountHash]; ok { - snapshotDirtyStorageHitMeter.Mark(1) - snapshotDirtyStorageHitDepthHist.Update(int64(depth)) - snapshotDirtyStorageInexMeter.Mark(1) - snapshotBloomStorageTrueHitMeter.Mark(1) - return nil, nil - } // Storage slot unknown to this diff, resolve from parent if diff, ok := dl.parent.(*diffLayer); ok { return diff.storage(accountHash, storageHash, depth+1) @@ -410,8 +362,8 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([ // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. -func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { - return newDiffLayer(dl, blockRoot, destructs, accounts, storage) +func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockRoot, accounts, storage) } // flatten pushes all data from this point downwards, flattening everything into @@ -436,12 +388,6 @@ func (dl *diffLayer) flatten() snapshot { if parent.stale.Swap(true) { panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo } - // Overwrite all the updated accounts blindly, merge the sorted list - for hash := range dl.destructSet { - parent.destructSet[hash] = struct{}{} - delete(parent.accountData, hash) - delete(parent.storageData, hash) - } for hash, data := range dl.accountData { parent.accountData[hash] = data } @@ -453,17 +399,13 @@ func (dl *diffLayer) flatten() snapshot { continue } // Storage exists in both parent and child, merge the slots - comboData := parent.storageData[accountHash] - for storageHash, data := range storage { - comboData[storageHash] = data - } + maps.Copy(parent.storageData[accountHash], storage) } // Return the combo parent return &diffLayer{ parent: parent.parent, origin: parent.origin, root: dl.root, - destructSet: parent.destructSet, accountData: parent.accountData, storageData: parent.storageData, storageList: make(map[common.Hash][]common.Hash), @@ -489,15 +431,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.lock.Lock() defer dl.lock.Unlock() - dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData)) - for hash := range dl.accountData { - dl.accountList = append(dl.accountList, hash) - } - for hash := range dl.destructSet { - if _, ok := dl.accountData[hash]; !ok { - dl.accountList = append(dl.accountList, hash) - } - } + dl.accountList = maps.Keys(dl.accountData) slices.SortFunc(dl.accountList, common.Hash.Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList @@ -512,18 +446,17 @@ func (dl *diffLayer) AccountList() []common.Hash { // not empty but the flag is true. // // Note, the returned slice is not a copy, so do not modify it. -func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) { +func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { dl.lock.RLock() - _, destructed := dl.destructSet[accountHash] if _, ok := dl.storageData[accountHash]; !ok { // Account not tracked by this layer dl.lock.RUnlock() - return nil, destructed + return nil } // If an old list already exists, return it if list, exist := dl.storageList[accountHash]; exist { dl.lock.RUnlock() - return list, destructed // the cached list can't be nil + return list // the cached list can't be nil } dl.lock.RUnlock() @@ -531,13 +464,9 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) dl.lock.Lock() defer dl.lock.Unlock() - storageMap := dl.storageData[accountHash] - storageList := make([]common.Hash, 0, len(storageMap)) - for k := range storageMap { - storageList = append(storageList, k) - } + storageList := maps.Keys(dl.storageData[accountHash]) slices.SortFunc(storageList, common.Hash.Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) - return storageList, destructed + return storageList } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 674a031b16..b212098e75 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -28,14 +28,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} { - copy := make(map[common.Hash]struct{}) - for hash := range destructs { - copy[hash] = struct{}{} - } - return copy -} - func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte { copy := make(map[common.Hash][]byte) for hash, blob := range accounts { @@ -58,9 +50,8 @@ func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash // TestMergeBasics tests some simple merges func TestMergeBasics(t *testing.T) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) // Fill up a parent for i := 0; i < 100; i++ { @@ -69,7 +60,7 @@ func TestMergeBasics(t *testing.T) { accounts[h] = data if rand.Intn(4) == 0 { - destructs[h] = struct{}{} + accounts[h] = nil } if rand.Intn(2) == 0 { accStorage := make(map[common.Hash][]byte) @@ -80,11 +71,12 @@ func TestMergeBasics(t *testing.T) { } } // Add some (identical) layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + parent := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child := newDiffLayer(parent, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + // And flatten merged := (child.flatten()).(*diffLayer) @@ -99,18 +91,13 @@ func TestMergeBasics(t *testing.T) { t.Errorf("accountList [2] wrong: have %v, want %v", have, want) } } - { // Check account drops - if have, want := len(merged.destructSet), len(destructs); have != want { - t.Errorf("accountDrop wrong: have %v, want %v", have, want) - } - } { // Check storage lists i := 0 for aHash, sMap := range storage { if have, want := len(merged.storageList), i; have != want { t.Errorf("[1] storageList wrong: have %v, want %v", have, want) } - list, _ := merged.StorageList(aHash) + list := merged.StorageList(aHash) if have, want := len(list), len(sMap); have != want { t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want) } @@ -124,41 +111,32 @@ func TestMergeBasics(t *testing.T) { // TestMergeDelete tests some deletion func TestMergeDelete(t *testing.T) { - var ( - storage = make(map[common.Hash]map[common.Hash][]byte) - ) + storage := make(map[common.Hash]map[common.Hash][]byte) + // Fill up a parent h1 := common.HexToHash("0x01") h2 := common.HexToHash("0x02") - flipDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h2: {}, - } - } - flipAccs := func() map[common.Hash][]byte { + flip := func() map[common.Hash][]byte { return map[common.Hash][]byte{ h1: randomAccount(), + h2: nil, } } - flopDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h1: {}, - } - } - flopAccs := func() map[common.Hash][]byte { + flop := func() map[common.Hash][]byte { return map[common.Hash][]byte{ + h1: nil, h2: randomAccount(), } } // Add some flipAccs-flopping layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage) - child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) + parent := newDiffLayer(emptyLayer(), common.Hash{}, flip(), storage) + child := parent.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) + child = child.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) + child = child.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) if data, _ := child.Account(h1); data == nil { t.Errorf("last diff layer: expected %x account to be non-nil", h1) @@ -166,12 +144,7 @@ func TestMergeDelete(t *testing.T) { if data, _ := child.Account(h2); data != nil { t.Errorf("last diff layer: expected %x account to be nil", h2) } - if _, ok := child.destructSet[h1]; ok { - t.Errorf("last diff layer: expected %x drop to be missing", h1) - } - if _, ok := child.destructSet[h2]; !ok { - t.Errorf("last diff layer: expected %x drop to be present", h1) - } + // And flatten merged := (child.flatten()).(*diffLayer) @@ -181,12 +154,6 @@ func TestMergeDelete(t *testing.T) { if data, _ := merged.Account(h2); data != nil { t.Errorf("merged layer: expected %x account to be nil", h2) } - if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } - if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } // If we add more granular metering of memory, we can enable this again, // but it's not implemented for now //if have, want := merged.memory, child.memory; have != want { @@ -206,22 +173,20 @@ func TestInsertAndMerge(t *testing.T) { ) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) - parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage) + parent = newDiffLayer(emptyLayer(), common.Hash{}, accounts, storage) } { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) accounts[acc] = randomAccount() storage[acc] = make(map[common.Hash][]byte) storage[acc][slot] = []byte{0x01} - child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + child = newDiffLayer(parent, common.Hash{}, accounts, storage) } // And flatten merged := (child.flatten()).(*diffLayer) @@ -250,14 +215,13 @@ func BenchmarkSearch(b *testing.B) { // First, we set up 128 diff layers, with 1K items each fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 10000; i++ { accounts[randomHash()] = randomAccount() } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } var layer snapshot layer = emptyLayer() @@ -286,9 +250,8 @@ func BenchmarkSearchSlot(b *testing.B) { accountRLP := randomAccount() fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) accounts[accountKey] = accountRLP @@ -299,7 +262,7 @@ func BenchmarkSearchSlot(b *testing.B) { accStorage[randomHash()] = value storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } var layer snapshot layer = emptyLayer() @@ -320,9 +283,8 @@ func BenchmarkSearchSlot(b *testing.B) { func BenchmarkFlatten(b *testing.B) { fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 100; i++ { accountKey := randomHash() @@ -336,7 +298,7 @@ func BenchmarkFlatten(b *testing.B) { } storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -369,9 +331,8 @@ func BenchmarkFlatten(b *testing.B) { func BenchmarkJournal(b *testing.B) { fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 200; i++ { accountKey := randomHash() @@ -385,7 +346,7 @@ func BenchmarkJournal(b *testing.B) { } storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } layer := snapshot(emptyLayer()) for i := 1; i < 128; i++ { diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index 76928edf07..202e6c70ed 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -180,8 +180,8 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. Note, the maps are retained by the method to avoid // copying everything. -func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { - return newDiffLayer(dl, blockHash, destructs, accounts, storage) +func (dl *diskLayer) Update(blockHash common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockHash, accounts, storage) } // stopGeneration aborts the state snapshot generation if it is currently running. diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 168458c405..6d99a90d61 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -117,20 +117,22 @@ func TestDiskMerge(t *testing.T) { base.Storage(conNukeCache, conNukeCacheSlot) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accDelNoCache: nil, + accDelCache: nil, + conNukeNoCache: nil, + conNukeCache: nil, + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, map[common.Hash]map[common.Hash][]byte{ + conNukeNoCache: {conNukeNoCacheSlot: nil}, + conNukeCache: {conNukeCacheSlot: nil}, + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -340,20 +342,27 @@ func TestDiskPartialMerge(t *testing.T) { assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accDelNoCache: nil, + accDelCache: nil, + conNukeNoCache: nil, + conNukeCache: nil, + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, + map[common.Hash]map[common.Hash][]byte{ + conNukeNoCache: { + conNukeNoCacheSlot: nil, + }, + conNukeCache: { + conNukeCacheSlot: nil, + }, + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { t.Fatalf("test %d: failed to update snapshot tree: %v", i, err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -462,9 +471,11 @@ func TestDiskGeneratorPersistence(t *testing.T) { }, } // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, nil, map[common.Hash][]byte{ - accTwo: accTwo[:], - }, nil); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accTwo: accTwo[:], + }, nil, + ); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -480,11 +491,14 @@ func TestDiskGeneratorPersistence(t *testing.T) { } // Test scenario 2, the disk layer is fully generated // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ - accThree: accThree.Bytes(), - }, map[common.Hash]map[common.Hash][]byte{ - accThree: {accThreeSlot: accThreeSlot.Bytes()}, - }); err != nil { + if err := snaps.Update(diffTwoRoot, diffRoot, + map[common.Hash][]byte{ + accThree: accThree.Bytes(), + }, + map[common.Hash]map[common.Hash][]byte{ + accThree: {accThreeSlot: accThreeSlot.Bytes()}, + }, + ); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } diskLayer := snaps.layers[snaps.diskRoot()].(*diskLayer) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 56abff348d..661610840a 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -134,7 +134,7 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { snapRoot, err := generateTrieRoot(nil, "", accIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { - storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) + storageIt := snap.StorageIterator(accountHash, common.Hash{}) defer storageIt.Release() hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index c1a196c7ff..cd9c225844 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -115,6 +115,7 @@ func (it *diffAccountIterator) Next() bool { } // Iterator seems to be still alive, retrieve and cache the live hash it.curHash = it.keys[0] + // key cached, shift the iterator and notify the user of success it.keys = it.keys[1:] return true @@ -135,7 +136,7 @@ func (it *diffAccountIterator) Hash() common.Hash { // This method may _fail_, if the underlying layer has been flattened between // the call to Next and Account. That type of error will set it.Err. // This method assumes that flattening does not delete elements from -// the accountdata mapping (writing nil into it is fine though), and will panic +// the accountData mapping (writing nil into it is fine though), and will panic // if elements have been deleted. // // Note the returned account is not a copy, please don't modify it. @@ -143,10 +144,6 @@ func (it *diffAccountIterator) Account() []byte { it.layer.lock.RLock() blob, ok := it.layer.accountData[it.curHash] if !ok { - if _, ok := it.layer.destructSet[it.curHash]; ok { - it.layer.lock.RUnlock() - return nil - } panic(fmt.Sprintf("iterator referenced non-existent account: %x", it.curHash)) } it.layer.lock.RUnlock() @@ -247,11 +244,11 @@ type diffStorageIterator struct { // "destructed" returned. If it's true then it means the whole storage is // destructed in this layer(maybe recreated too), don't bother deeper layer // for storage retrieval. -func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { +func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) StorageIterator { // Create the storage for this account even it's marked // as destructed. The iterator is for the new one which // just has the same address as the deleted one. - hashes, destructed := dl.StorageList(account) + hashes := dl.StorageList(account) index := sort.Search(len(hashes), func(i int) bool { return bytes.Compare(seek[:], hashes[i][:]) <= 0 }) @@ -260,7 +257,7 @@ func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (Sto layer: dl, account: account, keys: hashes[index:], - }, destructed + } } // Next steps the iterator forward one element, returning false if exhausted. @@ -339,13 +336,13 @@ type diskStorageIterator struct { // If the whole storage is destructed, then all entries in the disk // layer are deleted already. So the "destructed" flag returned here // is always false. -func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { +func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) StorageIterator { pos := common.TrimRightZeroes(seek[:]) return &diskStorageIterator{ layer: dl, account: account, it: dl.diskdb.NewIterator(append(rawdb.SnapshotStoragePrefix, account.Bytes()...), pos), - }, false + } } // Next steps the iterator forward one element, returning false if exhausted. diff --git a/core/state/snapshot/iterator_binary.go b/core/state/snapshot/iterator_binary.go index 523c7b9946..254c4b860f 100644 --- a/core/state/snapshot/iterator_binary.go +++ b/core/state/snapshot/iterator_binary.go @@ -67,44 +67,17 @@ func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) Iterator { func (dl *diffLayer) initBinaryStorageIterator(account, seek common.Hash) Iterator { parent, ok := dl.parent.(*diffLayer) if !ok { - // If the storage in this layer is already destructed, discard all - // deeper layers but still return a valid single-branch iterator. - a, destructed := dl.StorageIterator(account, seek) - if destructed { - l := &binaryIterator{ - a: a, - account: account, - } - l.aDone = !l.a.Next() - l.bDone = true - return l - } - // The parent is disk layer, don't need to take care "destructed" - // anymore. - b, _ := dl.Parent().StorageIterator(account, seek) l := &binaryIterator{ - a: a, - b: b, + a: dl.StorageIterator(account, seek), + b: dl.Parent().StorageIterator(account, seek), account: account, } l.aDone = !l.a.Next() l.bDone = !l.b.Next() return l } - // If the storage in this layer is already destructed, discard all - // deeper layers but still return a valid single-branch iterator. - a, destructed := dl.StorageIterator(account, seek) - if destructed { - l := &binaryIterator{ - a: a, - account: account, - } - l.aDone = !l.a.Next() - l.bDone = true - return l - } l := &binaryIterator{ - a: a, + a: dl.StorageIterator(account, seek), b: parent.initBinaryStorageIterator(account, seek), account: account, } diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index fa0daea7ba..7f7ba876ff 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -90,18 +90,10 @@ func newFastIterator(tree *Tree, root common.Hash, account common.Hash, seek com priority: depth, }) } else { - // If the whole storage is destructed in this layer, don't - // bother deeper layer anymore. But we should still keep - // the iterator for this layer, since the iterator can contain - // some valid slots which belongs to the re-created account. - it, destructed := current.StorageIterator(account, seek) fi.iterators = append(fi.iterators, &weightedIterator{ - it: it, + it: current.StorageIterator(account, seek), priority: depth, }) - if destructed { - break - } } current = current.Parent() } diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go index f2f4d19f83..b9fe370b86 100644 --- a/core/state/snapshot/iterator_test.go +++ b/core/state/snapshot/iterator_test.go @@ -32,9 +32,8 @@ import ( // TestAccountIteratorBasics tests some simple single-layer(diff and disk) iteration func TestAccountIteratorBasics(t *testing.T) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) // Fill up a parent for i := 0; i < 100; i++ { @@ -42,9 +41,6 @@ func TestAccountIteratorBasics(t *testing.T) { data := randomAccount() accounts[h] = data - if rand.Intn(4) == 0 { - destructs[h] = struct{}{} - } if rand.Intn(2) == 0 { accStorage := make(map[common.Hash][]byte) value := make([]byte, 32) @@ -54,7 +50,7 @@ func TestAccountIteratorBasics(t *testing.T) { } } // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) it := diffLayer.AccountIterator(common.Hash{}) verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator @@ -95,15 +91,15 @@ func TestStorageIteratorBasics(t *testing.T) { nilStorage[h] = nilstorage } // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage)) + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) for account := range accounts { - it, _ := diffLayer.StorageIterator(account, common.Hash{}) + it := diffLayer.StorageIterator(account, common.Hash{}) verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator } diskLayer := diffToDisk(diffLayer) for account := range accounts { - it, _ := diskLayer.StorageIterator(account, common.Hash{}) + it := diskLayer.StorageIterator(account, common.Hash{}) verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator } } @@ -225,13 +221,13 @@ func TestAccountIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Verify the single and multi-layer iterators @@ -272,19 +268,19 @@ func TestStorageIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) // Verify the single and multi-layer iterators head := snaps.Snapshot(common.HexToHash("0x04")) - diffIter, _ := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) + diffIter := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 3, diffIter, verifyNothing) verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage) @@ -357,14 +353,14 @@ func TestAccountIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), a, nil) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), b, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), c, nil) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), d, nil) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), e, nil) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), f, nil) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), g, nil) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), h, nil) it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -456,14 +452,14 @@ func TestStorageIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), wrapStorage(a)) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), wrapStorage(b)) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), wrapStorage(c)) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), randomAccountSet("0xaa"), wrapStorage(d)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), randomAccountSet("0xaa"), wrapStorage(g)) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), randomAccountSet("0xaa"), wrapStorage(h)) it, _ := snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -526,7 +522,7 @@ func TestAccountIteratorLargeTraversal(t *testing.T) { }, } for i := 1; i < 128; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(200), nil) } // Iterate the entire stack and ensure everything is hit only once head := snaps.Snapshot(common.HexToHash("0x80")) @@ -584,13 +580,13 @@ func testAccountIteratorFlattening(t *testing.T, newIterator func(snaps *Tree, r }, } // Create a stack of diffs on top - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Create an iterator and flatten the data from underneath it @@ -630,13 +626,13 @@ func testAccountIteratorSeek(t *testing.T, newIterator func(snaps *Tree, root, s base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Account set is now @@ -708,13 +704,13 @@ func testStorageIteratorSeek(t *testing.T, newIterator func(snaps *Tree, root, a }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil)) // Account set is now @@ -785,21 +781,16 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), - nil, randomAccountSet("0x11", "0x22", "0x33"), nil) - - deleted := common.HexToHash("0x22") - destructed := map[common.Hash]struct{}{ - deleted: {}, - } - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), - destructed, randomAccountSet("0x11", "0x33"), nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0x11", "0x22", "0x33"), nil) + set := randomAccountSet("0x11", "0x33") + set[common.HexToHash("0x22")] = nil + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), set, nil) snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), - nil, randomAccountSet("0x33", "0x44", "0x55"), nil) + randomAccountSet("0x33", "0x44", "0x55"), nil) // The output should be 11,33,44,55 - it := newIterator(snaps, common.HexToHash("0x04"), (common.Hash{})) + it := newIterator(snaps, common.HexToHash("0x04"), common.Hash{}) // Do a quick check verifyIterator(t, 4, it, verifyAccount) @@ -813,8 +804,8 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro if it.Account() == nil { t.Errorf("iterator returned nil-value for hash %x", hash) } - if hash == deleted { - t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) + if hash == common.HexToHash("0x22") { + t.Errorf("expected deleted elem %x to not be returned by iterator", common.HexToHash("0x22")) } } } @@ -846,10 +837,10 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}})) // The output should be 02,04,05,06 @@ -863,17 +854,16 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro it.Release() // Destruct the whole storage - destructed := map[common.Hash]struct{}{ - common.HexToHash("0xaa"): {}, - } - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), + map[common.Hash][]byte{common.HexToHash("0xaa"): nil}, + randomStorageSet([]string{"0xaa"}, nil, [][]string{{"0x02", "0x04", "0x05", "0x06"}})) it = newIterator(snaps, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 0, it, verifyStorage) it.Release() // Re-insert the slots of the same account - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil)) // The output should be 07,08,09 @@ -883,7 +873,9 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro it.Release() // Destruct the whole storage but re-create the account in the same layer - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), + randomAccountSet("0xaa"), + randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}})) it = newIterator(snaps, common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 it.Release() @@ -928,7 +920,7 @@ func BenchmarkAccountIteratorTraversal(b *testing.B) { }, } for i := 1; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(200), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. @@ -1023,9 +1015,9 @@ func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(2000), nil) for i := 2; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(20), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 8513e73dd0..61552c73fe 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -33,7 +33,9 @@ import ( "github.com/ethereum/go-ethereum/triedb" ) -const journalVersion uint64 = 0 +// 0: initial version +// 1: destruct flag in diff layer is removed +const journalVersion uint64 = 1 // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { @@ -48,11 +50,6 @@ type journalGenerator struct { Storage uint64 } -// journalDestruct is an account deletion entry in a diffLayer's disk journal. -type journalDestruct struct { - Hash common.Hash -} - // journalAccount is an account entry in a diffLayer's disk journal. type journalAccount struct { Hash common.Hash @@ -109,8 +106,8 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. var current snapshot = base - err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { - current = newDiffLayer(current, root, destructSet, accountData, storageData) + err := iterateJournal(db, func(parent common.Hash, root common.Hash, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, accountData, storageData) return nil }) if err != nil { @@ -238,16 +235,12 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { if err := rlp.Encode(buffer, dl.root); err != nil { return common.Hash{}, err } - destructs := make([]journalDestruct, 0, len(dl.destructSet)) - for hash := range dl.destructSet { - destructs = append(destructs, journalDestruct{Hash: hash}) - } - if err := rlp.Encode(buffer, destructs); err != nil { - return common.Hash{}, err - } accounts := make([]journalAccount, 0, len(dl.accountData)) for hash, blob := range dl.accountData { - accounts = append(accounts, journalAccount{Hash: hash, Blob: blob}) + accounts = append(accounts, journalAccount{ + Hash: hash, + Blob: blob, + }) } if err := rlp.Encode(buffer, accounts); err != nil { return common.Hash{}, err @@ -271,7 +264,7 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { // journalCallback is a function which is invoked by iterateJournal, every // time a difflayer is loaded from disk. -type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error +type journalCallback = func(parent common.Hash, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error // iterateJournal iterates through the journalled difflayers, loading them from // the database, and invoking the callback for each loaded layer. @@ -310,10 +303,8 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { for { var ( root common.Hash - destructs []journalDestruct accounts []journalAccount storage []journalStorage - destructSet = make(map[common.Hash]struct{}) accountData = make(map[common.Hash][]byte) storageData = make(map[common.Hash]map[common.Hash][]byte) ) @@ -325,18 +316,12 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } return fmt.Errorf("load diff root: %v", err) } - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) - } if err := r.Decode(&accounts); err != nil { return fmt.Errorf("load diff accounts: %v", err) } if err := r.Decode(&storage); err != nil { return fmt.Errorf("load diff storage: %v", err) } - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } for _, entry := range accounts { if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that accountData[entry.Hash] = entry.Blob @@ -355,7 +340,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } storageData[entry.Hash] = slots } - if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + if err := callback(parent, root, accountData, storageData); err != nil { return err } parent = root diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index d1ffb5cc2d..7466f12351 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -130,7 +130,7 @@ type snapshot interface { // the specified data items. // // Note, the maps are retained by the method to avoid copying everything. - Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer + Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer // Journal commits an entire diff hierarchy to disk into a single journal entry. // This is meant to be used during shutdown to persist the snapshot without @@ -145,7 +145,7 @@ type snapshot interface { AccountIterator(seek common.Hash) AccountIterator // StorageIterator creates a storage iterator over an arbitrary layer. - StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) + StorageIterator(account common.Hash, seek common.Hash) StorageIterator } // Config includes the configurations for snapshots. @@ -335,7 +335,7 @@ func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot { // Update adds a new snapshot into the tree, if that can be linked to an existing // old parent. It is disallowed to insert a disk layer (the origin of all). -func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { +func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { // Reject noop updates to avoid self-loops in the snapshot tree. This is a // special case that can only happen for Clique networks where empty blocks // don't modify the state (0 block subsidy). @@ -350,7 +350,7 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m if parent == nil { return fmt.Errorf("parent [%#x] snapshot missing", parentRoot) } - snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage) + snap := parent.(snapshot).Update(blockRoot, accounts, storage) // Save the new snapshot for later t.lock.Lock() @@ -539,35 +539,6 @@ func diffToDisk(bottom *diffLayer) *diskLayer { base.stale = true base.lock.Unlock() - // Destroy all the destructed accounts from the database - for hash := range bottom.destructSet { - // Skip any account not covered yet by the snapshot - if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 { - continue - } - // Remove all storage slots - rawdb.DeleteAccountSnapshot(batch, hash) - base.cache.Set(hash[:], nil) - - it := rawdb.IterateStorageSnapshots(base.diskdb, hash) - for it.Next() { - key := it.Key() - batch.Delete(key) - base.cache.Del(key[1:]) - snapshotFlushStorageItemMeter.Mark(1) - - // Ensure we don't delete too much data blindly (contract can be - // huge). It's ok to flush, the root will go missing in case of a - // crash and we'll detect and regenerate the snapshot. - if batch.ValueSize() > 64*1024*1024 { - if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) - } - batch.Reset() - } - } - it.Release() - } // Push all updated accounts into the database for hash, data := range bottom.accountData { // Skip any account not covered yet by the snapshot @@ -575,10 +546,14 @@ func diffToDisk(bottom *diffLayer) *diskLayer { continue } // Push the account to disk - rawdb.WriteAccountSnapshot(batch, hash, data) - base.cache.Set(hash[:], data) - snapshotCleanAccountWriteMeter.Mark(int64(len(data))) - + if len(data) != 0 { + rawdb.WriteAccountSnapshot(batch, hash, data) + base.cache.Set(hash[:], data) + snapshotCleanAccountWriteMeter.Mark(int64(len(data))) + } else { + rawdb.DeleteAccountSnapshot(batch, hash) + base.cache.Set(hash[:], nil) + } snapshotFlushAccountItemMeter.Mark(1) snapshotFlushAccountSizeMeter.Mark(int64(len(data))) @@ -587,7 +562,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer { // the snapshot. if batch.ValueSize() > 64*1024*1024 { if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) + log.Crit("Failed to write state changes", "err", err) } batch.Reset() } @@ -616,6 +591,16 @@ func diffToDisk(bottom *diffLayer) *diskLayer { } snapshotFlushStorageItemMeter.Mark(1) snapshotFlushStorageSizeMeter.Mark(int64(len(data))) + + // Ensure we don't write too much data blindly. It's ok to flush, the + // root will go missing in case of a crash and we'll detect and regen + // the snapshot. + if batch.ValueSize() > 64*1024*1024 { + if err := batch.Write(); err != nil { + log.Crit("Failed to write state changes", "err", err) + } + batch.Reset() + } } } // Update the snapshot block marker and write any remainder data diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index a9ab3eaea3..34ef61e8d0 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -107,7 +107,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 2 { @@ -151,10 +151,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 3 { @@ -203,13 +203,13 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 4 { @@ -263,12 +263,12 @@ func TestPostCapBasicDataAccess(t *testing.T) { }, } // The lowest difflayer - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), setAccount("0xb2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) - snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), setAccount("0xb3"), nil) // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { @@ -363,7 +363,7 @@ func TestSnaphots(t *testing.T) { ) for i := 0; i < 129; i++ { head = makeRoot(uint64(i + 2)) - snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) + snaps.Update(head, last, setAccount(fmt.Sprintf("%d", i+2)), nil) last = head snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) } @@ -456,9 +456,9 @@ func TestReadStateDuringFlattening(t *testing.T) { }, } // 4 layers in total, 3 diff layers and 1 disk layers - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), setAccount("0xa3"), nil) // Obtain the topmost snapshot handler for state accessing snap := snaps.Snapshot(common.HexToHash("0xa3")) diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index 62f073d2e1..c35c82f67a 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -75,7 +75,7 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { func checkDanglingMemStorage(db ethdb.KeyValueStore) error { start := time.Now() log.Info("Checking dangling journalled storage") - err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + err := iterateJournal(db, func(pRoot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { for accHash := range storage { if _, ok := accounts[accHash]; !ok { log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) @@ -119,12 +119,11 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { } var depth = 0 - return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + return iterateJournal(db, func(pRoot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { _, a := accounts[hash] - _, b := destructs[hash] - _, c := storage[hash] + _, b := storage[hash] depth++ - if !a && !b && !c { + if !a && !b { return nil } fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) @@ -138,9 +137,6 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { fmt.Printf("\taccount.root: %x\n", account.Root) fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) } - if _, ok := destructs[hash]; ok { - fmt.Printf("\t Destructed!") - } if data, ok := storage[hash]; ok { fmt.Printf("\tStorage\n") for k, v := range data { diff --git a/core/state/statedb.go b/core/state/statedb.go index d855e5626d..9cc91c9332 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -932,16 +932,17 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { iter, err := snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer iter.Release() var ( - nodes = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) + nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) + storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) + storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { nodes.AddNode(path, trienode.NewDeleted()) @@ -949,42 +950,47 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, for iter.Next() { slot := common.CopyBytes(iter.Slot()) if err := iter.Error(); err != nil { // error might occur after Slot function - return nil, nil, err + return nil, nil, nil, err } - slots[iter.Hash()] = slot + key := iter.Hash() + storages[key] = nil + storageOrigins[key] = slot - if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { - return nil, nil, err + if err := stack.Update(key.Bytes(), slot); err != nil { + return nil, nil, nil, err } } if err := iter.Error(); err != nil { // error might occur during iteration - return nil, nil, err + return nil, nil, nil, err } if stack.Hash() != root { - return nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + return nil, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { - return nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return nil, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return nil, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( - nodes = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) + nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) + storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) + storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) for it.Next(true) { if it.Leaf() { - slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) + key := common.BytesToHash(it.LeafKey()) + storages[key] = nil + storageOrigins[key] = common.CopyBytes(it.LeafBlob()) continue } if it.Hash() == (common.Hash{}) { @@ -993,35 +999,36 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { - return nil, nil, err + return nil, nil, nil, err } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // deleteStorage is designed to delete the storage trie of a designated account. // The function will make an attempt to utilize an efficient strategy if the // associated state snapshot is reachable; otherwise, it will resort to a less // efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { var ( - err error - slots map[common.Hash][]byte - nodes *trienode.NodeSet + err error + nodes *trienode.NodeSet // the set for trie node mutations (value is nil) + storages map[common.Hash][]byte // the set for storage mutations (value is nil) + storageOrigins map[common.Hash][]byte // the set for tracking the original value of slot ) // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. snaps := s.db.Snapshot() if snaps != nil { - slots, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) + storages, storageOrigins, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) } if snaps == nil || err != nil { - slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + storages, storageOrigins, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1068,16 +1075,16 @@ func (s *StateDB) handleDestruction() (map[common.Hash]*accountDelete, []*trieno deletes[addrHash] = op // Short circuit if the origin storage was empty. - if prev.Root == types.EmptyRootHash || s.db.TrieDB().IsVerkle() { continue } // Remove storage slots belonging to the account. - slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + storages, storagesOrigin, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { return nil, nil, fmt.Errorf("failed to delete storage, err: %w", err) } - op.storagesOrigin = slots + op.storages = storages + op.storagesOrigin = storagesOrigin // Aggregate the associated trie node changes. nodes = append(nodes, set) @@ -1267,7 +1274,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { start := time.Now() - if err := snap.Update(ret.root, ret.originRoot, ret.destructs, ret.accounts, ret.storages); err != nil { + if err := snap.Update(ret.root, ret.originRoot, ret.accounts, ret.storages); err != nil { log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 90250819e3..7cbfd9b9d7 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "maps" "math" "math/rand" "reflect" @@ -176,24 +177,16 @@ func (test *stateTest) String() string { func (test *stateTest) run() bool { var ( - roots []common.Hash - accountList []map[common.Address][]byte - storageList []map[common.Address]map[common.Hash][]byte - copyUpdate = func(update *stateUpdate) { - accounts := make(map[common.Address][]byte, len(update.accountsOrigin)) - for key, val := range update.accountsOrigin { - accounts[key] = common.CopyBytes(val) - } - accountList = append(accountList, accounts) - - storages := make(map[common.Address]map[common.Hash][]byte, len(update.storagesOrigin)) - for addr, subset := range update.storagesOrigin { - storages[addr] = make(map[common.Hash][]byte, len(subset)) - for key, val := range subset { - storages[addr][key] = common.CopyBytes(val) - } - } - storageList = append(storageList, storages) + roots []common.Hash + accounts []map[common.Hash][]byte + accountOrigin []map[common.Address][]byte + storages []map[common.Hash]map[common.Hash][]byte + storageOrigin []map[common.Address]map[common.Hash][]byte + copyUpdate = func(update *stateUpdate) { + accounts = append(accounts, maps.Clone(update.accounts)) + accountOrigin = append(accountOrigin, maps.Clone(update.accountsOrigin)) + storages = append(storages, maps.Clone(update.storages)) + storageOrigin = append(storageOrigin, maps.Clone(update.storagesOrigin)) } disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) @@ -250,7 +243,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[i-1] } - test.err = test.verify(root, roots[i], tdb, accountList[i], storageList[i]) + test.err = test.verify(root, roots[i], tdb, accounts[i], accountOrigin[i], storages[i], storageOrigin[i]) if test.err != nil { return false } @@ -265,7 +258,7 @@ func (test *stateTest) run() bool { // - the account was indeed not present in trie // - the account is present in new trie, nil->nil is regarded as invalid // - the slots transition is correct -func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, account []byte, storages map[common.Hash][]byte, storagesOrigin map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -282,6 +275,13 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa if len(nBlob) == 0 { return fmt.Errorf("missing account in new trie, %x", addrHash) } + full, err := types.FullAccountRLP(account) + if err != nil { + return err + } + if !bytes.Equal(nBlob, full) { + return fmt.Errorf("unexpected account data, want: %v, got: %v", full, nBlob) + } // Verify storage changes var nAcct types.StateAccount @@ -290,7 +290,10 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa } // Account has no slot, empty slot set is expected if nAcct.Root == types.EmptyRootHash { - if len(slots) != 0 { + if len(storagesOrigin) != 0 { + return fmt.Errorf("unexpected slot changes %x", addrHash) + } + if len(storages) != 0 { return fmt.Errorf("unexpected slot changes %x", addrHash) } return nil @@ -300,9 +303,22 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa if err != nil { return err } - for key, val := range slots { + for key, val := range storagesOrigin { + if _, exist := storages[key]; !exist { + return errors.New("storage data is not found") + } + got, err := st.Get(key.Bytes()) + if err != nil { + return err + } + if !bytes.Equal(got, storages[key]) { + return fmt.Errorf("unexpected storage data, want: %v, got: %v", storages[key], got) + } st.Update(key.Bytes(), val) } + if len(storagesOrigin) != len(storages) { + return fmt.Errorf("extra storage found, want: %d, got: %d", len(storagesOrigin), len(storages)) + } if st.Hash() != types.EmptyRootHash { return errors.New("invalid slot changes") } @@ -316,7 +332,7 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa // - the account was indeed present in trie // - the account in old trie matches the provided value // - the slots transition is correct -func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, account []byte, accountOrigin []byte, storages map[common.Hash][]byte, storageOrigin map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -330,14 +346,23 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database if len(oBlob) == 0 { return fmt.Errorf("missing account in old trie, %x", addrHash) } - full, err := types.FullAccountRLP(origin) + full, err := types.FullAccountRLP(accountOrigin) if err != nil { return err } if !bytes.Equal(full, oBlob) { return fmt.Errorf("account value is not matched, %x", addrHash) } - + if len(nBlob) == 0 { + if len(account) != 0 { + return errors.New("unexpected account data") + } + } else { + full, _ = types.FullAccountRLP(account) + if !bytes.Equal(full, nBlob) { + return fmt.Errorf("unexpected account data, %x, want %v, got: %v", addrHash, full, nBlob) + } + } // Decode accounts var ( oAcct types.StateAccount @@ -361,16 +386,29 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database if err != nil { return err } - for key, val := range slots { + for key, val := range storageOrigin { + if _, exist := storages[key]; !exist { + return errors.New("storage data is not found") + } + got, err := st.Get(key.Bytes()) + if err != nil { + return err + } + if !bytes.Equal(got, storages[key]) { + return fmt.Errorf("unexpected storage data, want: %v, got: %v", storages[key], got) + } st.Update(key.Bytes(), val) } + if len(storageOrigin) != len(storages) { + return fmt.Errorf("extra storage found, want: %d, got: %d", len(storageOrigin), len(storages)) + } if st.Hash() != oAcct.Root { return errors.New("invalid slot changes") } return nil } -func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { +func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accounts map[common.Hash][]byte, accountsOrigin map[common.Address][]byte, storages map[common.Hash]map[common.Hash][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { otr, err := trie.New(trie.StateTrieID(root), db) if err != nil { return err @@ -379,12 +417,15 @@ func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Dat if err != nil { return err } - for addr, account := range accountsOrigin { - var err error - if len(account) == 0 { - err = test.verifyAccountCreation(next, db, otr, ntr, addr, storagesOrigin[addr]) + for addr, accountOrigin := range accountsOrigin { + var ( + err error + addrHash = crypto.Keccak256Hash(addr.Bytes()) + ) + if len(accountOrigin) == 0 { + err = test.verifyAccountCreation(next, db, otr, ntr, addr, accounts[addrHash], storages[addrHash], storagesOrigin[addr]) } else { - err = test.verifyAccountUpdate(next, db, otr, ntr, addr, accountsOrigin[addr], storagesOrigin[addr]) + err = test.verifyAccountUpdate(next, db, otr, ntr, addr, accounts[addrHash], accountsOrigin[addr], storages[addrHash], storagesOrigin[addr]) } if err != nil { return err diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 3647397df6..57886e6e03 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1305,12 +1305,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index c9231f0526..a320b72f11 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -17,6 +17,8 @@ package state import ( + "maps" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/trie/trienode" @@ -33,6 +35,7 @@ type contractCode struct { type accountDelete struct { address common.Address // address is the unique account identifier origin []byte // origin is the original value of account data in slim-RLP encoding. + storages map[common.Hash][]byte // storages stores mutated slots, the value should be nil. storagesOrigin map[common.Hash][]byte // storagesOrigin stores the original values of mutated slots in prefix-zero-trimmed RLP format. } @@ -52,7 +55,6 @@ type accountUpdate struct { type stateUpdate struct { originRoot common.Hash // hash of the state before applying mutation root common.Hash // hash of the state after applying mutation - destructs map[common.Hash]struct{} // destructs contains the list of destructed accounts accounts map[common.Hash][]byte // accounts stores mutated accounts in 'slim RLP' encoding accountsOrigin map[common.Address][]byte // accountsOrigin stores the original values of mutated accounts in 'slim RLP' encoding storages map[common.Hash]map[common.Hash][]byte // storages stores mutated slots in 'prefix-zero-trimmed' RLP format @@ -71,7 +73,6 @@ func (sc *stateUpdate) empty() bool { // account deletions and account updates to form a comprehensive state update. func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common.Hash]*accountDelete, updates map[common.Hash]*accountUpdate, nodes *trienode.MergedNodeSet) *stateUpdate { var ( - destructs = make(map[common.Hash]struct{}) accounts = make(map[common.Hash][]byte) accountsOrigin = make(map[common.Address][]byte) storages = make(map[common.Hash]map[common.Hash][]byte) @@ -82,8 +83,12 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common // within the same block, the deletions must be aggregated first. for addrHash, op := range deletes { addr := op.address - destructs[addrHash] = struct{}{} + accounts[addrHash] = nil accountsOrigin[addr] = op.origin + + if len(op.storages) > 0 { + storages[addrHash] = op.storages + } if len(op.storagesOrigin) > 0 { storagesOrigin[addr] = op.storagesOrigin } @@ -95,35 +100,41 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common if op.code != nil { codes[addr] = *op.code } - // Aggregate the account changes. The original account value will only - // be tracked if it's not present yet. accounts[addrHash] = op.data + + // Aggregate the account original value. If the account is already + // present in the aggregated accountsOrigin set, skip it. if _, found := accountsOrigin[addr]; !found { accountsOrigin[addr] = op.origin } - // Aggregate the storage changes. The original storage slot value will - // only be tracked if it's not present yet. + // Aggregate the storage mutation list. If a slot in op.storages is + // already present in aggregated storages set, the value will be + // overwritten. if len(op.storages) > 0 { - storages[addrHash] = op.storages - } - if len(op.storagesOrigin) > 0 { - origin := storagesOrigin[addr] - if origin == nil { - storagesOrigin[addr] = op.storagesOrigin - continue + if _, exist := storages[addrHash]; !exist { + storages[addrHash] = op.storages + } else { + maps.Copy(storages[addrHash], op.storages) } - for key, slot := range op.storagesOrigin { - if _, found := origin[key]; !found { - origin[key] = slot + } + // Aggregate the storage original values. If the slot is already present + // in aggregated storagesOrigin set, skip it. + if len(op.storagesOrigin) > 0 { + origin, exist := storagesOrigin[addr] + if !exist { + storagesOrigin[addr] = op.storagesOrigin + } else { + for key, slot := range op.storagesOrigin { + if _, found := origin[key]; !found { + origin[key] = slot + } } } - storagesOrigin[addr] = origin } } return &stateUpdate{ originRoot: types.TrieRootHash(originRoot), root: types.TrieRootHash(root), - destructs: destructs, accounts: accounts, accountsOrigin: accountsOrigin, storages: storages, @@ -139,7 +150,6 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common // package. func (sc *stateUpdate) stateSet() *triedb.StateSet { return &triedb.StateSet{ - Destructs: sc.destructs, Accounts: sc.accounts, AccountsOrigin: sc.accountsOrigin, Storages: sc.storages, diff --git a/triedb/states.go b/triedb/states.go index 1f9a0de522..0b03f2b9f3 100644 --- a/triedb/states.go +++ b/triedb/states.go @@ -23,7 +23,6 @@ import ( // StateSet represents a collection of mutated states during a state transition. type StateSet struct { - Destructs map[common.Hash]struct{} // Destructed accounts Accounts map[common.Hash][]byte // Mutated accounts in 'slim RLP' encoding AccountsOrigin map[common.Address][]byte // Original values of mutated accounts in 'slim RLP' encoding Storages map[common.Hash]map[common.Hash][]byte // Mutated storage slots in 'prefix-zero-trimmed' RLP format @@ -33,7 +32,6 @@ type StateSet struct { // NewStateSet initializes an empty state set. func NewStateSet() *StateSet { return &StateSet{ - Destructs: make(map[common.Hash]struct{}), Accounts: make(map[common.Hash][]byte), AccountsOrigin: make(map[common.Address][]byte), Storages: make(map[common.Hash]map[common.Hash][]byte), From 5e1a39d67f642990b9b98f8391ceca39a5fdb849 Mon Sep 17 00:00:00 2001 From: Daniel Liu <139250065@qq.com> Date: Mon, 25 Nov 2024 03:09:38 +0800 Subject: [PATCH 06/16] internal/flags: fix "flag redefined" bug for alias on custom flags (#30796) This change fixes a bug on the `DirectoryFlag` and the `BigFlag`, which would trigger a `panic` with the message "flag redefined" in case an alias was added to such a flag. --- internal/flags/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 1f6be3d3d3..b858e73d04 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -90,7 +90,7 @@ func (f *DirectoryFlag) Apply(set *flag.FlagSet) error { } } eachName(f, func(name string) { - set.Var(&f.Value, f.Name, f.Usage) + set.Var(&f.Value, name, f.Usage) }) return nil } @@ -172,7 +172,7 @@ func (f *BigFlag) Apply(set *flag.FlagSet) error { } eachName(f, func(name string) { f.Value = new(big.Int) - set.Var((*bigValue)(f.Value), f.Name, f.Usage) + set.Var((*bigValue)(f.Value), name, f.Usage) }) return nil } From ab4a1cc01f3897b3bd9918c1f7cddc5730baefa9 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 25 Nov 2024 10:07:50 +0100 Subject: [PATCH 07/16] eth/tracers/logger: fix json-logger output missing (#30804) Fixes a flaw introduced in https://github.com/ethereum/go-ethereum/pull/29795 , discovered while reviewing https://github.com/ethereum/go-ethereum/pull/30633 . --- eth/tracers/logger/logger_json.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/eth/tracers/logger/logger_json.go b/eth/tracers/logger/logger_json.go index 797f7ac658..52ac3945d4 100644 --- a/eth/tracers/logger/logger_json.go +++ b/eth/tracers/logger/logger_json.go @@ -71,7 +71,7 @@ func NewJSONLogger(cfg *Config, writer io.Writer) *tracing.Hooks { l.hooks = &tracing.Hooks{ OnTxStart: l.OnTxStart, OnSystemCallStart: l.onSystemCallStart, - OnExit: l.OnEnd, + OnExit: l.OnExit, OnOpcode: l.OnOpcode, OnFault: l.OnFault, } @@ -152,13 +152,6 @@ func (l *jsonLogger) OnEnter(depth int, typ byte, from common.Address, to common l.encoder.Encode(frame) } -func (l *jsonLogger) OnEnd(depth int, output []byte, gasUsed uint64, err error, reverted bool) { - if depth > 0 { - return - } - l.OnExit(depth, output, gasUsed, err, false) -} - func (l *jsonLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { type endLog struct { Output string `json:"output"` From 02159d553f8cc02baaa344edd8cc78f0306d6dbd Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 25 Nov 2024 13:29:27 +0100 Subject: [PATCH 08/16] eth/tracers/logger: improve markdown logger (#30805) This PR improves the output of the markdown logger a bit. - Remove `RStack` field, - Move `Stack` last, since it may have very large vertical expansion - Make the pre- and post-exec metadata structured into a bullet-list --- eth/tracers/logger/logger.go | 37 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index f918ce154b..bdd4e07055 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -363,26 +363,35 @@ func (t *mdLogger) OnEnter(depth int, typ byte, from common.Address, to common.A if depth != 0 { return } - create := vm.OpCode(typ) == vm.CREATE - if !create { - fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", - from.String(), to.String(), - input, gas, value) + if create := vm.OpCode(typ) == vm.CREATE; !create { + fmt.Fprintf(t.out, "Pre-execution info:\n"+ + " - from: `%v`\n"+ + " - to: `%v`\n"+ + " - data: `%#x`\n"+ + " - gas: `%d`\n"+ + " - value: `%v` wei\n", + from.String(), to.String(), input, gas, value) } else { - fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", - from.String(), to.String(), - input, gas, value) + fmt.Fprintf(t.out, "Pre-execution info:\n"+ + " - from: `%v`\n"+ + " - create: `%v`\n"+ + " - data: `%#x`\n"+ + " - gas: `%d`\n"+ + " - value: `%v` wei\n", + from.String(), to.String(), input, gas, value) } - fmt.Fprintf(t.out, ` -| Pc | Op | Cost | Stack | RStack | Refund | -|-------|-------------|------|-----------|-----------|---------| +| Pc | Op | Cost | Refund | Stack | +|-------|-------------|------|-----------|-----------| `) } func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { if depth == 0 { - fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", + fmt.Fprintf(t.out, "\nPost-execution info:\n"+ + " - output: `%#x`\n"+ + " - consumed gas: `%d`\n"+ + " - error: `%v`\n", output, gasUsed, err) } } @@ -390,7 +399,8 @@ func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, r // OnOpcode also tracks SLOAD/SSTORE ops to track storage change. func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { stack := scope.StackData() - fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, vm.OpCode(op).String(), cost) + fmt.Fprintf(t.out, "| %4d | %10v | %3d |%10v |", pc, vm.OpCode(op).String(), + cost, t.env.StateDB.GetRefund()) if !t.cfg.DisableStack { // format stack @@ -401,7 +411,6 @@ func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing. b := fmt.Sprintf("[%v]", strings.Join(a, ",")) fmt.Fprintf(t.out, "%10v |", b) } - fmt.Fprintf(t.out, "%10v |", t.env.StateDB.GetRefund()) fmt.Fprintln(t.out, "") if err != nil { fmt.Fprintf(t.out, "Error: %v\n", err) From 19fa71b917dfba9a2f7692aeef524bc541a710aa Mon Sep 17 00:00:00 2001 From: "Hyunsoo Shin (Lake)" Date: Mon, 25 Nov 2024 21:33:28 +0900 Subject: [PATCH 09/16] internal/ethapi: remove double map-clone (#30803) Similar to https://github.com/ethereum/go-ethereum/pull/30788 --- internal/ethapi/simulate.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 0b58a27096..2161e1d5f4 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "math/big" "time" @@ -186,7 +185,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, Tracer: tracer.Hooks(), } ) - var tracingStateDB = vm.StateDB(sim.state) + tracingStateDB := vm.StateDB(sim.state) if hooks := tracer.Hooks(); hooks != nil { tracingStateDB = state.NewHookedState(sim.state, hooks) } @@ -289,7 +288,7 @@ func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContra isMerge = (base.Difficulty.Sign() == 0) rules = sim.chainConfig.Rules(base.Number, isMerge, base.Time) ) - return maps.Clone(vm.ActivePrecompiledContracts(rules)) + return vm.ActivePrecompiledContracts(rules) } // sanitizeChain checks the chain integrity. Specifically it checks that From 3c754e2a09d112cd62f4d51f221637f9a147dcc6 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Mon, 25 Nov 2024 06:34:50 -0600 Subject: [PATCH 10/16] accounts/abi: fix MakeTopics mutation of big.Int inputs (#30785) #28764 updated `func MakeTopics` to support negative `*big.Int`s. However, it also changed the behavior of the function from just _reading_ the input `*big.Int` via `Bytes()`, to leveraging `big.U256Bytes` which is documented as being _destructive_: This change updates `MakeTopics` to not mutate the original, and also applies the same change in signer/core/apitypes. --- accounts/abi/topics.go | 2 +- accounts/abi/topics_test.go | 17 +++++++++++++++++ signer/core/apitypes/types.go | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/accounts/abi/topics.go b/accounts/abi/topics.go index 7ce9b7273c..4819334ae6 100644 --- a/accounts/abi/topics.go +++ b/accounts/abi/topics.go @@ -42,7 +42,7 @@ func MakeTopics(query ...[]interface{}) ([][]common.Hash, error) { case common.Address: copy(topic[common.HashLength-common.AddressLength:], rule[:]) case *big.Int: - copy(topic[:], math.U256Bytes(rule)) + copy(topic[:], math.U256Bytes(new(big.Int).Set(rule))) case bool: if rule { topic[common.HashLength-1] = 1 diff --git a/accounts/abi/topics_test.go b/accounts/abi/topics_test.go index 6a4c50078a..161867e2d9 100644 --- a/accounts/abi/topics_test.go +++ b/accounts/abi/topics_test.go @@ -149,6 +149,23 @@ func TestMakeTopics(t *testing.T) { } }) } + + t.Run("does not mutate big.Int", func(t *testing.T) { + t.Parallel() + want := [][]common.Hash{{common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")}} + + in := big.NewInt(-1) + got, err := MakeTopics([]interface{}{in}) + if err != nil { + t.Fatalf("makeTopics() error = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("makeTopics() = %v, want %v", got, want) + } + if orig := big.NewInt(-1); in.Cmp(orig) != 0 { + t.Fatalf("makeTopics() mutated an input parameter from %v to %v", orig, in) + } + }) } type args struct { diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 2ae182279a..b56931c1d1 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -676,7 +676,7 @@ func (typedData *TypedData) EncodePrimitiveValue(encType string, encValue interf if err != nil { return nil, err } - return math.U256Bytes(b), nil + return math.U256Bytes(new(big.Int).Set(b)), nil } return nil, fmt.Errorf("unrecognized type '%s'", encType) } From 23800122b37695be50565f8221858a16ce1763db Mon Sep 17 00:00:00 2001 From: Arran Schlosberg <519948+ARR4N@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:43:23 +0000 Subject: [PATCH 11/16] core/state/snapshot: simplify snapshot rebuild (#30772) This PR is purely for improved readability; I was doing work involving the file and think this may help others who are trying to understand what's going on. 1. `snapshot.Tree.Rebuild()` now returns a function that blocks until regeneration is complete, allowing `Tree.waitBuild()` to be removed entirely as all it did was search for the `done` channel behind this new function. 2. Its usage inside `New()` is also simplified by (a) only waiting if `!AsyncBuild`; and (b) avoiding the double negative of `if !NoBuild`. --------- Co-authored-by: Martin HS --- core/state/snapshot/snapshot.go | 48 ++++++++++----------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 7466f12351..181dc6b746 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -206,47 +206,24 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, roo log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil } - // Create the building waiter iff the background generation is allowed - if !config.NoBuild && !config.AsyncBuild { - defer snap.waitBuild() - } if err != nil { log.Warn("Failed to load snapshot", "err", err) - if !config.NoBuild { - snap.Rebuild(root) - return snap, nil + if config.NoBuild { + return nil, err } - return nil, err // Bail out the error, don't rebuild automatically. + wait := snap.Rebuild(root) + if !config.AsyncBuild { + wait() + } + return snap, nil } // Existing snapshot loaded, seed all the layers - for head != nil { + for ; head != nil; head = head.Parent() { snap.layers[head.Root()] = head - head = head.Parent() } return snap, nil } -// waitBuild blocks until the snapshot finishes rebuilding. This method is meant -// to be used by tests to ensure we're testing what we believe we are. -func (t *Tree) waitBuild() { - // Find the rebuild termination channel - var done chan struct{} - - t.lock.RLock() - for _, layer := range t.layers { - if layer, ok := layer.(*diskLayer); ok { - done = layer.genPending - break - } - } - t.lock.RUnlock() - - // Wait until the snapshot is generated - if done != nil { - <-done - } -} - // Disable interrupts any pending snapshot generator, deletes all the snapshot // layers in memory and marks snapshots disabled globally. In order to resume // the snapshot functionality, the caller must invoke Rebuild. @@ -688,8 +665,9 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { // Rebuild wipes all available snapshot data from the persistent database and // discard all caches and diff layers. Afterwards, it starts a new snapshot -// generator with the given root hash. -func (t *Tree) Rebuild(root common.Hash) { +// generator with the given root hash. The returned function blocks until +// regeneration is complete. +func (t *Tree) Rebuild(root common.Hash) (wait func()) { t.lock.Lock() defer t.lock.Unlock() @@ -721,9 +699,11 @@ func (t *Tree) Rebuild(root common.Hash) { // Start generating a new snapshot from scratch on a background thread. The // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") + disk := generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root) t.layers = map[common.Hash]snapshot{ - root: generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root), + root: disk, } + return func() { <-disk.genPending } } // AccountIterator creates a new account iterator for the specified root hash and From b4d99e39176f89bb0ffbb32635e9cf17b5fd7367 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 26 Nov 2024 02:49:12 +0100 Subject: [PATCH 12/16] eth/ethconfig: improve error message if TTD missing (#30807) This updates the message you get when trying to initialize Geth with genesis.json that doesn't have `terminalTotalDifficulty`. The previous message was a bit obscure, I had to check the code to find out what the problem was. --- eth/ethconfig/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f1a815e6d3..8542bc97c4 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" ) @@ -160,7 +161,8 @@ type Config struct { // only exist on already merged networks. func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (consensus.Engine, error) { if config.TerminalTotalDifficulty == nil { - return nil, fmt.Errorf("only PoS networks are supported, please transition old ones with Geth v1.13.x") + log.Error("Geth only supports PoS networks. Please transition legacy networks using Geth v1.13.x.") + return nil, fmt.Errorf("'terminalTotalDifficulty' is not set in genesis block") } // Wrap previously supported consensus engines into their post-merge counterpart if config.Clique != nil { From d7e7b541907b3c0a8e1898937f411d5c219759bd Mon Sep 17 00:00:00 2001 From: Nebojsa Urosevic Date: Mon, 25 Nov 2024 23:16:00 -0800 Subject: [PATCH 13/16] core/tracing: add GetCodeHash to StateDB (#30784) This PR extends the tracing.StateDB interface by adding a GetCodeHash function. --- core/tracing/hooks.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index a21bb1577b..0c5cae354b 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -42,6 +42,7 @@ type StateDB interface { GetBalance(common.Address) *uint256.Int GetNonce(common.Address) uint64 GetCode(common.Address) []byte + GetCodeHash(common.Address) common.Hash GetState(common.Address, common.Hash) common.Hash GetTransientState(common.Address, common.Hash) common.Hash Exist(common.Address) bool From a11b4bebcb4aacf36ad4ce2712c6696bdff5d143 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 26 Nov 2024 18:33:59 +0800 Subject: [PATCH 14/16] Revert "core/state/snapshot: simplify snapshot rebuild (#30772)" (#30810) This reverts commit 23800122b37695be50565f8221858a16ce1763db. The original pull request introduces a bug and some flaky tests are detected because of this flaw. ``` --- FAIL: TestRecoverSnapshotFromWipingCrash (0.27s) blockchain_snapshot_test.go:158: The disk layer is not integrated snapshot is not constructed {"pc":0,"op":88,"gas":"0x7148","gasCost":"0x2","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PC"} {"pc":1,"op":255,"gas":"0x7146","gasCost":"0x1db0","memSize":0,"stack":["0x0"],"depth":1,"refund":0,"opName":"SELFDESTRUCT"} {"output":"","gasUsed":"0x0"} {"output":"","gasUsed":"0x1db2"} {"pc":0,"op":116,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH21"} ``` Before the original PR, the snapshot would block the function until the disk layer was fully generated under the following conditions: (a) explicitly required by users with `AsyncBuild = false`. (b) the snapshot was being fully rebuilt or *the disk layer generation had resumed*. Unfortunately, with the changes introduced in that PR, the snapshot no longer waits for disk layer generation to complete if the generation is resumed. It brings lots of uncertainty and breaks this tiny debug feature. --- core/state/snapshot/snapshot.go | 48 +++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 181dc6b746..7466f12351 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -206,24 +206,47 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, roo log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil } + // Create the building waiter iff the background generation is allowed + if !config.NoBuild && !config.AsyncBuild { + defer snap.waitBuild() + } if err != nil { log.Warn("Failed to load snapshot", "err", err) - if config.NoBuild { - return nil, err + if !config.NoBuild { + snap.Rebuild(root) + return snap, nil } - wait := snap.Rebuild(root) - if !config.AsyncBuild { - wait() - } - return snap, nil + return nil, err // Bail out the error, don't rebuild automatically. } // Existing snapshot loaded, seed all the layers - for ; head != nil; head = head.Parent() { + for head != nil { snap.layers[head.Root()] = head + head = head.Parent() } return snap, nil } +// waitBuild blocks until the snapshot finishes rebuilding. This method is meant +// to be used by tests to ensure we're testing what we believe we are. +func (t *Tree) waitBuild() { + // Find the rebuild termination channel + var done chan struct{} + + t.lock.RLock() + for _, layer := range t.layers { + if layer, ok := layer.(*diskLayer); ok { + done = layer.genPending + break + } + } + t.lock.RUnlock() + + // Wait until the snapshot is generated + if done != nil { + <-done + } +} + // Disable interrupts any pending snapshot generator, deletes all the snapshot // layers in memory and marks snapshots disabled globally. In order to resume // the snapshot functionality, the caller must invoke Rebuild. @@ -665,9 +688,8 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { // Rebuild wipes all available snapshot data from the persistent database and // discard all caches and diff layers. Afterwards, it starts a new snapshot -// generator with the given root hash. The returned function blocks until -// regeneration is complete. -func (t *Tree) Rebuild(root common.Hash) (wait func()) { +// generator with the given root hash. +func (t *Tree) Rebuild(root common.Hash) { t.lock.Lock() defer t.lock.Unlock() @@ -699,11 +721,9 @@ func (t *Tree) Rebuild(root common.Hash) (wait func()) { // Start generating a new snapshot from scratch on a background thread. The // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") - disk := generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root) t.layers = map[common.Hash]snapshot{ - root: disk, + root: generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root), } - return func() { <-disk.genPending } } // AccountIterator creates a new account iterator for the specified root hash and From 915248cd6bf14b9d1b29db182375bc456d4ef868 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 26 Nov 2024 22:12:38 +0700 Subject: [PATCH 15/16] cmd/evm: don't reuse state between iterations, show errors (#30780) Reusing state between benchmark iterations resulted in inconsistent results across runs, which surfaced in https://github.com/ethereum/go-ethereum/issues/30778 . If these errors are triggered again, they will now trigger panic. --------- Co-authored-by: Martin HS --- cmd/evm/runner.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 8c382e6257..2884487faa 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -84,19 +84,20 @@ type execStats struct { func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, execStats, error) { if bench { + testing.Init() // Do one warm-up run output, gasUsed, err := execFunc() result := testing.Benchmark(func(b *testing.B) { for i := 0; i < b.N; i++ { haveOutput, haveGasUsed, haveErr := execFunc() if !bytes.Equal(haveOutput, output) { - b.Fatalf("output differs, have\n%x\nwant%x\n", haveOutput, output) + panic(fmt.Sprintf("output differs\nhave %x\nwant %x\n", haveOutput, output)) } if haveGasUsed != gasUsed { - b.Fatalf("gas differs, have %v want%v", haveGasUsed, gasUsed) + panic(fmt.Sprintf("gas differs, have %v want %v", haveGasUsed, gasUsed)) } if haveErr != err { - b.Fatalf("err differs, have %v want%v", haveErr, err) + panic(fmt.Sprintf("err differs, have %v want %v", haveErr, err)) } } }) @@ -137,7 +138,7 @@ func runCmd(ctx *cli.Context) error { var ( tracer *tracing.Hooks debugLogger *logger.StructLogger - statedb *state.StateDB + prestate *state.StateDB chainConfig *params.ChainConfig sender = common.BytesToAddress([]byte("sender")) receiver = common.BytesToAddress([]byte("receiver")) @@ -174,7 +175,7 @@ func runCmd(ctx *cli.Context) error { defer triedb.Close() genesis := genesisConfig.MustCommit(db, triedb) sdb := state.NewDatabase(triedb, nil) - statedb, _ = state.New(genesis.Root(), sdb) + prestate, _ = state.New(genesis.Root(), sdb) chainConfig = genesisConfig.Config if ctx.String(SenderFlag.Name) != "" { @@ -231,7 +232,7 @@ func runCmd(ctx *cli.Context) error { } runtimeConfig := runtime.Config{ Origin: sender, - State: statedb, + State: prestate, GasLimit: initialGas, GasPrice: flags.GlobalBig(ctx, PriceFlag.Name), Value: flags.GlobalBig(ctx, ValueFlag.Name), @@ -274,14 +275,18 @@ func runCmd(ctx *cli.Context) error { if ctx.Bool(CreateFlag.Name) { input = append(code, input...) execFunc = func() ([]byte, uint64, error) { + // don't mutate the state! + runtimeConfig.State = prestate.Copy() output, _, gasLeft, err := runtime.Create(input, &runtimeConfig) return output, gasLeft, err } } else { if len(code) > 0 { - statedb.SetCode(receiver, code) + prestate.SetCode(receiver, code) } execFunc = func() ([]byte, uint64, error) { + // don't mutate the state! + runtimeConfig.State = prestate.Copy() output, gasLeft, err := runtime.Call(receiver, input, &runtimeConfig) return output, initialGas - gasLeft, err } @@ -291,7 +296,7 @@ func runCmd(ctx *cli.Context) error { output, stats, err := timedExec(bench, execFunc) if ctx.Bool(DumpFlag.Name) { - root, err := statedb.Commit(genesisConfig.Number, true) + root, err := runtimeConfig.State.Commit(genesisConfig.Number, true) if err != nil { fmt.Printf("Failed to commit changes %v\n", err) return err @@ -310,7 +315,7 @@ func runCmd(ctx *cli.Context) error { logger.WriteTrace(os.Stderr, debugLogger.StructLogs()) } fmt.Fprintln(os.Stderr, "#### LOGS ####") - logger.WriteLogs(os.Stderr, statedb.Logs()) + logger.WriteLogs(os.Stderr, runtimeConfig.State.Logs()) } if bench || ctx.Bool(StatDumpFlag.Name) { From e0deac7f6f8dd3cb59529e25ccdd548dd003ed4d Mon Sep 17 00:00:00 2001 From: wangjingcun Date: Wed, 27 Nov 2024 14:17:03 +0800 Subject: [PATCH 16/16] core: better document reason for dropping error on return (#30811) Add a comment for error return of nil Signed-off-by: wangjingcun --- core/blockchain_reader.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 19c1b17f36..8f1da82a2b 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -277,6 +277,13 @@ func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLoo if tx == nil { progress, err := bc.TxIndexProgress() if err != nil { + // No error is returned if the transaction indexing progress is unreachable + // due to unexpected internal errors. In such cases, it is impossible to + // determine whether the transaction does not exist or has simply not been + // indexed yet without a progress marker. + // + // In such scenarios, the transaction is treated as unreachable, though + // this is clearly an unintended and unexpected situation. return nil, nil, nil } // The transaction indexing is not finished yet, returning an